pyerrors.input.sfcf

  1import os
  2import fnmatch
  3import re
  4import numpy as np  # Thinly-wrapped numpy
  5from ..obs import Obs
  6from .utils import sort_names, check_idl
  7import itertools
  8
  9
 10sep = "/"
 11
 12
 13def read_sfcf(path, prefix, name, quarks='.*', corr_type="bi", noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", silent=False, **kwargs):
 14    """Read sfcf files from given folder structure.
 15
 16    Parameters
 17    ----------
 18    path : str
 19        Path to the sfcf files.
 20    prefix : str
 21        Prefix of the sfcf files.
 22    name : str
 23        Name of the correlation function to read.
 24    quarks : str
 25        Label of the quarks used in the sfcf input file. e.g. "quark quark"
 26        for version 0.0 this does NOT need to be given with the typical " - "
 27        that is present in the output file,
 28        this is done automatically for this version
 29    corr_type : str
 30        Type of correlation function to read. Can be
 31        - 'bi' for boundary-inner
 32        - 'bb' for boundary-boundary
 33        - 'bib' for boundary-inner-boundary
 34    noffset : int
 35        Offset of the source (only relevant when wavefunctions are used)
 36    wf : int
 37        ID of wave function
 38    wf2 : int
 39        ID of the second wavefunction
 40        (only relevant for boundary-to-boundary correlation functions)
 41    im : bool
 42        if True, read imaginary instead of real part
 43        of the correlation function.
 44    names : list
 45        Alternative labeling for replicas/ensembles.
 46        Has to have the appropriate length
 47    ens_name : str
 48        replaces the name of the ensemble
 49    version: str
 50        version of SFCF, with which the measurement was done.
 51        if the compact output option (-c) was specified,
 52        append a "c" to the version (e.g. "1.0c")
 53        if the append output option (-a) was specified,
 54        append an "a" to the version
 55    cfg_separator : str
 56        String that separates the ensemble identifier from the configuration number (default 'n').
 57    replica: list
 58        list of replica to be read, default is all
 59    files: list
 60        list of files to be read per replica, default is all.
 61        for non-compact output format, hand the folders to be read here.
 62    check_configs: list[list[int]]
 63        list of list of supposed configs, eg. [range(1,1000)]
 64        for one replicum with 1000 configs
 65
 66    Returns
 67    -------
 68    result: list[Obs]
 69        list of Observables with length T, observable per timeslice.
 70        bb-type correlators have length 1.
 71    """
 72    ret = read_sfcf_multi(path, prefix, [name], quarks_list=[quarks], corr_type_list=[corr_type],
 73                          noffset_list=[noffset], wf_list=[wf], wf2_list=[wf2], version=version,
 74                          cfg_separator=cfg_separator, silent=silent, **kwargs)
 75    return ret[name][quarks][str(noffset)][str(wf)][str(wf2)]
 76
 77
 78def read_sfcf_multi(path, prefix, name_list, quarks_list=['.*'], corr_type_list=['bi'], noffset_list=[0], wf_list=[0], wf2_list=[0], version="1.0c", cfg_separator="n", silent=False, keyed_out=False, **kwargs):
 79    """Read sfcf files from given folder structure.
 80
 81    Parameters
 82    ----------
 83    path : str
 84        Path to the sfcf files.
 85    prefix : str
 86        Prefix of the sfcf files.
 87    name : str
 88        Name of the correlation function to read.
 89    quarks_list : list[str]
 90        Label of the quarks used in the sfcf input file. e.g. "quark quark"
 91        for version 0.0 this does NOT need to be given with the typical " - "
 92        that is present in the output file,
 93        this is done automatically for this version
 94    corr_type_list : list[str]
 95        Type of correlation function to read. Can be
 96        - 'bi' for boundary-inner
 97        - 'bb' for boundary-boundary
 98        - 'bib' for boundary-inner-boundary
 99    noffset_list : list[int]
100        Offset of the source (only relevant when wavefunctions are used)
101    wf_list : int
102        ID of wave function
103    wf2_list : list[int]
104        ID of the second wavefunction
105        (only relevant for boundary-to-boundary correlation functions)
106    im : bool
107        if True, read imaginary instead of real part
108        of the correlation function.
109    names : list
110        Alternative labeling for replicas/ensembles.
111        Has to have the appropriate length
112    ens_name : str
113        replaces the name of the ensemble
114    version: str
115        version of SFCF, with which the measurement was done.
116        if the compact output option (-c) was specified,
117        append a "c" to the version (e.g. "1.0c")
118        if the append output option (-a) was specified,
119        append an "a" to the version
120    cfg_separator : str
121        String that separates the ensemble identifier from the configuration number (default 'n').
122    replica: list
123        list of replica to be read, default is all
124    files: list[list[int]]
125        list of files to be read per replica, default is all.
126        for non-compact output format, hand the folders to be read here.
127    check_configs: list[list[int]]
128        list of list of supposed configs, eg. [range(1,1000)]
129        for one replicum with 1000 configs
130
131    Returns
132    -------
133    result: dict[list[Obs]]
134        dict with one of the following properties:
135        if keyed_out:
136            dict[key] = list[Obs]
137            where key has the form name/quarks/offset/wf/wf2
138        if not keyed_out:
139            dict[name][quarks][offset][wf][wf2] = list[Obs]
140    """
141
142    if kwargs.get('im'):
143        im = 1
144        part = 'imaginary'
145    else:
146        im = 0
147        part = 'real'
148
149    known_versions = ["0.0", "1.0", "2.0", "1.0c", "2.0c", "1.0a", "2.0a"]
150
151    if version not in known_versions:
152        raise Exception("This version is not known!")
153    if (version[-1] == "c"):
154        appended = False
155        compact = True
156        version = version[:-1]
157    elif (version[-1] == "a"):
158        appended = True
159        compact = False
160        version = version[:-1]
161    else:
162        compact = False
163        appended = False
164    ls = []
165    if "replica" in kwargs:
166        ls = kwargs.get("replica")
167    else:
168        for (dirpath, dirnames, filenames) in os.walk(path):
169            if not appended:
170                ls.extend(dirnames)
171            else:
172                ls.extend(filenames)
173            break
174        if not ls:
175            raise Exception('Error, directory not found')
176        # Exclude folders with different names
177        for exc in ls:
178            if not fnmatch.fnmatch(exc, prefix + '*'):
179                ls = list(set(ls) - set([exc]))
180
181    if not appended:
182        ls = sort_names(ls)
183        replica = len(ls)
184
185    else:
186        replica = len([file.split(".")[-1] for file in ls]) // len(set([file.split(".")[-1] for file in ls]))
187    if replica == 0:
188        raise Exception('No replica found in directory')
189    if not silent:
190        print('Read', part, 'part of', name_list, 'from', prefix[:-1], ',', replica, 'replica')
191
192    if 'names' in kwargs:
193        new_names = kwargs.get('names')
194        if len(new_names) != len(set(new_names)):
195            raise Exception("names are not unique!")
196        if len(new_names) != replica:
197            raise Exception('names should have the length', replica)
198
199    else:
200        ens_name = kwargs.get("ens_name")
201        if not appended:
202            new_names = _get_rep_names(ls, ens_name)
203        else:
204            new_names = _get_appended_rep_names(ls, prefix, name_list[0], ens_name)
205        new_names = sort_names(new_names)
206
207    idl = []
208
209    noffset_list = [str(x) for x in noffset_list]
210    wf_list = [str(x) for x in wf_list]
211    wf2_list = [str(x) for x in wf2_list]
212
213    # setup dict structures
214    intern = {}
215    for name, corr_type in zip(name_list, corr_type_list):
216        intern[name] = {}
217        b2b, single = _extract_corr_type(corr_type)
218        intern[name]["b2b"] = b2b
219        intern[name]["single"] = single
220        intern[name]["spec"] = {}
221        for quarks in quarks_list:
222            intern[name]["spec"][quarks] = {}
223            for off in noffset_list:
224                intern[name]["spec"][quarks][off] = {}
225                for w in wf_list:
226                    intern[name]["spec"][quarks][off][w] = {}
227                    if b2b:
228                        for w2 in wf2_list:
229                            intern[name]["spec"][quarks][off][w][w2] = {}
230                            intern[name]["spec"][quarks][off][w][w2]["pattern"] = _make_pattern(version, name, off, w, w2, intern[name]['b2b'], quarks)
231                    else:
232                        intern[name]["spec"][quarks][off][w]["0"] = {}
233                        intern[name]["spec"][quarks][off][w]["0"]["pattern"] = _make_pattern(version, name, off, w, 0, intern[name]['b2b'], quarks)
234
235    internal_ret_dict = {}
236    needed_keys = []
237    for name, corr_type in zip(name_list, corr_type_list):
238        b2b, single = _extract_corr_type(corr_type)
239        if b2b:
240            needed_keys.extend(_lists2key([name], quarks_list, noffset_list, wf_list, wf2_list))
241        else:
242            needed_keys.extend(_lists2key([name], quarks_list, noffset_list, wf_list, ["0"]))
243
244    for key in needed_keys:
245        internal_ret_dict[key] = []
246
247    if not appended:
248        for i, item in enumerate(ls):
249            rep_path = path + '/' + item
250            if "files" in kwargs:
251                files = kwargs.get("files")
252                if isinstance(files, list):
253                    if all(isinstance(f, list) for f in files):
254                        files = files[i]
255                    elif all(isinstance(f, str) for f in files):
256                        files = files
257                    else:
258                        raise TypeError("files has to be of type list[list[str]] or list[str]!")
259                else:
260                    raise TypeError("files has to be of type list[list[str]] or list[str]!")
261
262            else:
263                files = []
264            sub_ls = _find_files(rep_path, prefix, compact, files)
265            rep_idl = []
266            no_cfg = len(sub_ls)
267            for cfg in sub_ls:
268                try:
269                    if compact:
270                        rep_idl.append(int(cfg.split(cfg_separator)[-1]))
271                    else:
272                        rep_idl.append(int(cfg[3:]))
273                except Exception:
274                    raise Exception("Couldn't parse idl from directory, problem with file " + cfg)
275            rep_idl.sort()
276            # maybe there is a better way to print the idls
277            if not silent:
278                print(item, ':', no_cfg, ' configurations')
279            idl.append(rep_idl)
280            # here we have found all the files we need to look into.
281            if i == 0:
282                if version != "0.0" and compact:
283                    file = path + '/' + item + '/' + sub_ls[0]
284                for name_index, name in enumerate(name_list):
285                    if version == "0.0" or not compact:
286                        file = path + '/' + item + '/' + sub_ls[0] + '/' + name
287                    if corr_type_list[name_index] == 'bi':
288                        name_keys = _lists2key(quarks_list, noffset_list, wf_list, ["0"])
289                    else:
290                        name_keys = _lists2key(quarks_list, noffset_list, wf_list, wf2_list)
291                    for key in name_keys:
292                        specs = _key2specs(key)
293                        quarks = specs[0]
294                        off = specs[1]
295                        w = specs[2]
296                        w2 = specs[3]
297                        # here, we want to find the place within the file,
298                        # where the correlator we need is stored.
299                        # to do so, the pattern needed is put together
300                        # from the input values
301                        start_read, T = _find_correlator(file, version, intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["pattern"], intern[name]['b2b'], silent=silent)
302                        intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["start"] = start_read
303                        intern[name]["T"] = T
304                        # preparing the datastructure
305                        # the correlators get parsed into...
306                        deltas = []
307                        for j in range(intern[name]["T"]):
308                            deltas.append([])
309                        internal_ret_dict[sep.join([name, key])] = deltas
310
311            if compact:
312                rep_deltas = _read_compact_rep(path, item, sub_ls, intern, needed_keys, im)
313                for key in needed_keys:
314                    name = _key2specs(key)[0]
315                    for t in range(intern[name]["T"]):
316                        internal_ret_dict[key][t].append(rep_deltas[key][t])
317            else:
318                for key in needed_keys:
319                    rep_data = []
320                    name = _key2specs(key)[0]
321                    for subitem in sub_ls:
322                        cfg_path = path + '/' + item + '/' + subitem
323                        file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im)
324                        rep_data.append(file_data)
325                    for t in range(intern[name]["T"]):
326                        internal_ret_dict[key][t].append([])
327                        for cfg in range(no_cfg):
328                            internal_ret_dict[key][t][i].append(rep_data[cfg][key][t])
329    else:
330        for key in needed_keys:
331            specs = _key2specs(key)
332            name = specs[0]
333            quarks = specs[1]
334            off = specs[2]
335            w = specs[3]
336            w2 = specs[4]
337            if "files" in kwargs:
338                if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")):
339                    name_ls = kwargs.get("files")
340                else:
341                    raise TypeError("In append mode, files has to be of type list[str]!")
342            else:
343                name_ls = ls
344                for exc in name_ls:
345                    if not fnmatch.fnmatch(exc, prefix + '*.' + name):
346                        name_ls = list(set(name_ls) - set([exc]))
347            name_ls = sort_names(name_ls)
348            pattern = intern[name]['spec'][quarks][off][w][w2]['pattern']
349            deltas = []
350            for rep, file in enumerate(name_ls):
351                rep_idl = []
352                filename = path + '/' + file
353                T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single'])
354                if rep == 0:
355                    intern[name]['T'] = T
356                    for t in range(intern[name]['T']):
357                        deltas.append([])
358                for t in range(intern[name]['T']):
359                    deltas[t].append(rep_data[t])
360                internal_ret_dict[key] = deltas
361                if name == name_list[0]:
362                    idl.append(rep_idl)
363
364    if kwargs.get("check_configs") is True:
365        if not silent:
366            print("Checking for missing configs...")
367        che = kwargs.get("check_configs")
368        if not (len(che) == len(idl)):
369            raise Exception("check_configs has to be the same length as replica!")
370        for r in range(len(idl)):
371            if not silent:
372                print("checking " + new_names[r])
373            check_idl(idl[r], che[r])
374        if not silent:
375            print("Done")
376
377    result_dict = {}
378    if keyed_out:
379        for key in needed_keys:
380            name = _key2specs(key)[0]
381            result = []
382            for t in range(intern[name]["T"]):
383                result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl))
384            result_dict[key] = result
385    else:
386        for name, corr_type in zip(name_list, corr_type_list):
387            result_dict[name] = {}
388            for quarks in quarks_list:
389                result_dict[name][quarks] = {}
390                for off in noffset_list:
391                    result_dict[name][quarks][off] = {}
392                    for w in wf_list:
393                        result_dict[name][quarks][off][w] = {}
394                        if corr_type != 'bi':
395                            for w2 in wf2_list:
396                                key = _specs2key(name, quarks, off, w, w2)
397                                result = []
398                                for t in range(intern[name]["T"]):
399                                    result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl))
400                                result_dict[name][quarks][str(off)][str(w)][str(w2)] = result
401                        else:
402                            key = _specs2key(name, quarks, off, w, "0")
403                            result = []
404                            for t in range(intern[name]["T"]):
405                                result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl))
406                            result_dict[name][quarks][str(off)][str(w)][str(0)] = result
407    return result_dict
408
409
410def _lists2key(*lists):
411    keys = []
412    for tup in itertools.product(*lists):
413        keys.append(sep.join(tup))
414    return keys
415
416
417def _key2specs(key):
418    return key.split(sep)
419
420
421def _specs2key(*specs):
422    return sep.join(specs)
423
424
425def _read_o_file(cfg_path, name, needed_keys, intern, version, im):
426    return_vals = {}
427    for key in needed_keys:
428        file = cfg_path + '/' + name
429        specs = _key2specs(key)
430        if specs[0] == name:
431            with open(file) as fp:
432                lines = fp.readlines()
433                quarks = specs[1]
434                off = specs[2]
435                w = specs[3]
436                w2 = specs[4]
437                T = intern[name]["T"]
438                start_read = intern[name]["spec"][quarks][off][w][w2]["start"]
439                deltas = []
440                for line in lines[start_read:start_read + T]:
441                    floats = list(map(float, line.split()))
442                    if version == "0.0":
443                        deltas.append(floats[im - intern[name]["single"]])
444                    else:
445                        deltas.append(floats[1 + im - intern[name]["single"]])
446                return_vals[key] = deltas
447    return return_vals
448
449
450def _extract_corr_type(corr_type):
451    if corr_type == 'bb':
452        b2b = True
453        single = True
454    elif corr_type == 'bib':
455        b2b = True
456        single = False
457    else:
458        b2b = False
459        single = False
460    return b2b, single
461
462
463def _find_files(rep_path, prefix, compact, files=[]):
464    sub_ls = []
465    if not files == []:
466        files.sort(key=lambda x: int(re.findall(r'\d+', x)[-1]))
467    else:
468        for (dirpath, dirnames, filenames) in os.walk(rep_path):
469            if compact:
470                sub_ls.extend(filenames)
471            else:
472                sub_ls.extend(dirnames)
473            break
474        if compact:
475            for exc in sub_ls:
476                if not fnmatch.fnmatch(exc, prefix + '*'):
477                    sub_ls = list(set(sub_ls) - set([exc]))
478            sub_ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1]))
479        else:
480            for exc in sub_ls:
481                if not fnmatch.fnmatch(exc, 'cfg*'):
482                    sub_ls = list(set(sub_ls) - set([exc]))
483            sub_ls.sort(key=lambda x: int(x[3:]))
484        files = sub_ls
485    if len(files) == 0:
486        raise FileNotFoundError("Did not find files in", rep_path, "with prefix", prefix, "and the given structure.")
487    return files
488
489
490def _make_pattern(version, name, noffset, wf, wf2, b2b, quarks):
491    if version == "0.0":
492        pattern = "# " + name + " : offset " + str(noffset) + ", wf " + str(wf)
493        if b2b:
494            pattern += ", wf_2 " + str(wf2)
495        qs = quarks.split(" ")
496        pattern += " : " + qs[0] + " - " + qs[1]
497    else:
498        pattern = 'name      ' + name + '\nquarks    ' + quarks + '\noffset    ' + str(noffset) + '\nwf        ' + str(wf)
499        if b2b:
500            pattern += '\nwf_2      ' + str(wf2)
501    return pattern
502
503
504def _find_correlator(file_name, version, pattern, b2b, silent=False):
505    T = 0
506
507    with open(file_name, "r") as my_file:
508
509        content = my_file.read()
510        match = re.search(pattern, content)
511        if match:
512            if version == "0.0":
513                start_read = content.count('\n', 0, match.start()) + 1
514                T = content.count('\n', start_read)
515            else:
516                start_read = content.count('\n', 0, match.start()) + 5 + b2b
517                end_match = re.search(r'\n\s*\n', content[match.start():])
518                T = content[match.start():].count('\n', 0, end_match.start()) - 4 - b2b
519            if not T > 0:
520                raise ValueError("Correlator with pattern\n" + pattern + "\nis empty!")
521            if not silent:
522                print(T, 'entries, starting to read in line', start_read)
523
524        else:
525            raise ValueError('Correlator with pattern\n' + pattern + '\nnot found.')
526
527    return start_read, T
528
529
530def _read_compact_file(rep_path, cfg_file, intern, needed_keys, im):
531    return_vals = {}
532    with open(rep_path + cfg_file) as fp:
533        lines = fp.readlines()
534        for key in needed_keys:
535            keys = _key2specs(key)
536            name = keys[0]
537            quarks = keys[1]
538            off = keys[2]
539            w = keys[3]
540            w2 = keys[4]
541
542            T = intern[name]["T"]
543            start_read = intern[name]["spec"][quarks][off][w][w2]["start"]
544            # check, if the correlator is in fact
545            # printed completely
546            if (start_read + T + 1 > len(lines)):
547                raise Exception("EOF before end of correlator data! Maybe " + rep_path + cfg_file + " is corrupted?")
548            corr_lines = lines[start_read - 6: start_read + T]
549            t_vals = []
550
551            if corr_lines[1 - intern[name]["b2b"]].strip() != 'name      ' + name:
552                raise Exception('Wrong format in file', cfg_file)
553
554            for k in range(6, T + 6):
555                floats = list(map(float, corr_lines[k].split()))
556                t_vals.append(floats[-2:][im])
557            return_vals[key] = t_vals
558    return return_vals
559
560
561def _read_compact_rep(path, rep, sub_ls, intern, needed_keys, im):
562    rep_path = path + '/' + rep + '/'
563    no_cfg = len(sub_ls)
564
565    return_vals = {}
566    for key in needed_keys:
567        name = _key2specs(key)[0]
568        deltas = []
569        for t in range(intern[name]["T"]):
570            deltas.append(np.zeros(no_cfg))
571        return_vals[key] = deltas
572
573    for cfg in range(no_cfg):
574        cfg_file = sub_ls[cfg]
575        cfg_data = _read_compact_file(rep_path, cfg_file, intern, needed_keys, im)
576        for key in needed_keys:
577            name = _key2specs(key)[0]
578            for t in range(intern[name]["T"]):
579                return_vals[key][t][cfg] = cfg_data[key][t]
580    return return_vals
581
582
583def _read_chunk(chunk, gauge_line, cfg_sep, start_read, T, corr_line, b2b, pattern, im, single):
584    try:
585        idl = int(chunk[gauge_line].split(cfg_sep)[-1])
586    except Exception:
587        raise Exception("Couldn't parse idl from directory, problem with chunk around line ", gauge_line)
588
589    found_pat = ""
590    data = []
591    for li in chunk[corr_line + 1:corr_line + 6 + b2b]:
592        found_pat += li
593    if re.search(pattern, found_pat):
594        for t, line in enumerate(chunk[start_read:start_read + T]):
595            floats = list(map(float, line.split()))
596            data.append(floats[im + 1 - single])
597    return idl, data
598
599
600def _read_append_rep(filename, pattern, b2b, cfg_separator, im, single):
601    with open(filename, 'r') as fp:
602        content = fp.readlines()
603        data_starts = []
604        for linenumber, line in enumerate(content):
605            if "[run]" in line:
606                data_starts.append(linenumber)
607        if len(set([data_starts[i] - data_starts[i - 1] for i in range(1, len(data_starts))])) > 1:
608            raise Exception("Irregularities in file structure found, not all runs have the same output length")
609        chunk = content[:data_starts[1]]
610        for linenumber, line in enumerate(chunk):
611            if line.startswith("gauge_name"):
612                gauge_line = linenumber
613            elif line.startswith("[correlator]"):
614                corr_line = linenumber
615                found_pat = ""
616                for li in chunk[corr_line + 1: corr_line + 6 + b2b]:
617                    found_pat += li
618                if re.search(pattern, found_pat):
619                    start_read = corr_line + 7 + b2b
620                    break
621                else:
622                    raise ValueError("Did not find pattern\n", pattern, "\nin\n", filename)
623        endline = corr_line + 6 + b2b
624        while not chunk[endline] == "\n":
625            endline += 1
626        T = endline - start_read
627
628        # all other chunks should follow the same structure
629        rep_idl = []
630        rep_data = []
631
632        for cnfg in range(len(data_starts)):
633            start = data_starts[cnfg]
634            stop = start + data_starts[1]
635            chunk = content[start:stop]
636            idl, data = _read_chunk(chunk, gauge_line, cfg_separator, start_read, T, corr_line, b2b, pattern, im, single)
637            rep_idl.append(idl)
638            rep_data.append(data)
639
640        data = []
641
642        for t in range(T):
643            data.append([])
644            for c in range(len(rep_data)):
645                data[t].append(rep_data[c][t])
646        return T, rep_idl, data
647
648
649def _get_rep_names(ls, ens_name=None):
650    new_names = []
651    for entry in ls:
652        try:
653            idx = entry.index('r')
654        except Exception:
655            raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.")
656
657        if ens_name:
658            new_names.append('ens_name' + '|' + entry[idx:])
659        else:
660            new_names.append(entry[:idx] + '|' + entry[idx:])
661    return new_names
662
663
664def _get_appended_rep_names(ls, prefix, name, ens_name=None):
665    new_names = []
666    for exc in ls:
667        if not fnmatch.fnmatch(exc, prefix + '*.' + name):
668            ls = list(set(ls) - set([exc]))
669    ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1]))
670    for entry in ls:
671        myentry = entry[:-len(name) - 1]
672        try:
673            idx = myentry.index('r')
674        except Exception:
675            raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.")
676
677        if ens_name:
678            new_names.append('ens_name' + '|' + entry[idx:])
679        else:
680            new_names.append(myentry[:idx] + '|' + myentry[idx:])
681    return new_names
sep = '/'
def read_sfcf( path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0, wf2=0, version='1.0c', cfg_separator='n', silent=False, **kwargs):
14def read_sfcf(path, prefix, name, quarks='.*', corr_type="bi", noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", silent=False, **kwargs):
15    """Read sfcf files from given folder structure.
16
17    Parameters
18    ----------
19    path : str
20        Path to the sfcf files.
21    prefix : str
22        Prefix of the sfcf files.
23    name : str
24        Name of the correlation function to read.
25    quarks : str
26        Label of the quarks used in the sfcf input file. e.g. "quark quark"
27        for version 0.0 this does NOT need to be given with the typical " - "
28        that is present in the output file,
29        this is done automatically for this version
30    corr_type : str
31        Type of correlation function to read. Can be
32        - 'bi' for boundary-inner
33        - 'bb' for boundary-boundary
34        - 'bib' for boundary-inner-boundary
35    noffset : int
36        Offset of the source (only relevant when wavefunctions are used)
37    wf : int
38        ID of wave function
39    wf2 : int
40        ID of the second wavefunction
41        (only relevant for boundary-to-boundary correlation functions)
42    im : bool
43        if True, read imaginary instead of real part
44        of the correlation function.
45    names : list
46        Alternative labeling for replicas/ensembles.
47        Has to have the appropriate length
48    ens_name : str
49        replaces the name of the ensemble
50    version: str
51        version of SFCF, with which the measurement was done.
52        if the compact output option (-c) was specified,
53        append a "c" to the version (e.g. "1.0c")
54        if the append output option (-a) was specified,
55        append an "a" to the version
56    cfg_separator : str
57        String that separates the ensemble identifier from the configuration number (default 'n').
58    replica: list
59        list of replica to be read, default is all
60    files: list
61        list of files to be read per replica, default is all.
62        for non-compact output format, hand the folders to be read here.
63    check_configs: list[list[int]]
64        list of list of supposed configs, eg. [range(1,1000)]
65        for one replicum with 1000 configs
66
67    Returns
68    -------
69    result: list[Obs]
70        list of Observables with length T, observable per timeslice.
71        bb-type correlators have length 1.
72    """
73    ret = read_sfcf_multi(path, prefix, [name], quarks_list=[quarks], corr_type_list=[corr_type],
74                          noffset_list=[noffset], wf_list=[wf], wf2_list=[wf2], version=version,
75                          cfg_separator=cfg_separator, silent=silent, **kwargs)
76    return ret[name][quarks][str(noffset)][str(wf)][str(wf2)]

Read sfcf files from given folder structure.

Parameters
  • path (str): Path to the sfcf files.
  • prefix (str): Prefix of the sfcf files.
  • name (str): Name of the correlation function to read.
  • quarks (str): Label of the quarks used in the sfcf input file. e.g. "quark quark" for version 0.0 this does NOT need to be given with the typical " - " that is present in the output file, this is done automatically for this version
  • corr_type (str): Type of correlation function to read. Can be
    • 'bi' for boundary-inner
    • 'bb' for boundary-boundary
    • 'bib' for boundary-inner-boundary
  • noffset (int): Offset of the source (only relevant when wavefunctions are used)
  • wf (int): ID of wave function
  • wf2 (int): ID of the second wavefunction (only relevant for boundary-to-boundary correlation functions)
  • im (bool): if True, read imaginary instead of real part of the correlation function.
  • names (list): Alternative labeling for replicas/ensembles. Has to have the appropriate length
  • ens_name (str): replaces the name of the ensemble
  • version (str): version of SFCF, with which the measurement was done. if the compact output option (-c) was specified, append a "c" to the version (e.g. "1.0c") if the append output option (-a) was specified, append an "a" to the version
  • cfg_separator (str): String that separates the ensemble identifier from the configuration number (default 'n').
  • replica (list): list of replica to be read, default is all
  • files (list): list of files to be read per replica, default is all. for non-compact output format, hand the folders to be read here.
  • check_configs (list[list[int]]): list of list of supposed configs, eg. [range(1,1000)] for one replicum with 1000 configs
Returns
  • result (list[Obs]): list of Observables with length T, observable per timeslice. bb-type correlators have length 1.
def read_sfcf_multi( path, prefix, name_list, quarks_list=['.*'], corr_type_list=['bi'], noffset_list=[0], wf_list=[0], wf2_list=[0], version='1.0c', cfg_separator='n', silent=False, keyed_out=False, **kwargs):
 79def read_sfcf_multi(path, prefix, name_list, quarks_list=['.*'], corr_type_list=['bi'], noffset_list=[0], wf_list=[0], wf2_list=[0], version="1.0c", cfg_separator="n", silent=False, keyed_out=False, **kwargs):
 80    """Read sfcf files from given folder structure.
 81
 82    Parameters
 83    ----------
 84    path : str
 85        Path to the sfcf files.
 86    prefix : str
 87        Prefix of the sfcf files.
 88    name : str
 89        Name of the correlation function to read.
 90    quarks_list : list[str]
 91        Label of the quarks used in the sfcf input file. e.g. "quark quark"
 92        for version 0.0 this does NOT need to be given with the typical " - "
 93        that is present in the output file,
 94        this is done automatically for this version
 95    corr_type_list : list[str]
 96        Type of correlation function to read. Can be
 97        - 'bi' for boundary-inner
 98        - 'bb' for boundary-boundary
 99        - 'bib' for boundary-inner-boundary
100    noffset_list : list[int]
101        Offset of the source (only relevant when wavefunctions are used)
102    wf_list : int
103        ID of wave function
104    wf2_list : list[int]
105        ID of the second wavefunction
106        (only relevant for boundary-to-boundary correlation functions)
107    im : bool
108        if True, read imaginary instead of real part
109        of the correlation function.
110    names : list
111        Alternative labeling for replicas/ensembles.
112        Has to have the appropriate length
113    ens_name : str
114        replaces the name of the ensemble
115    version: str
116        version of SFCF, with which the measurement was done.
117        if the compact output option (-c) was specified,
118        append a "c" to the version (e.g. "1.0c")
119        if the append output option (-a) was specified,
120        append an "a" to the version
121    cfg_separator : str
122        String that separates the ensemble identifier from the configuration number (default 'n').
123    replica: list
124        list of replica to be read, default is all
125    files: list[list[int]]
126        list of files to be read per replica, default is all.
127        for non-compact output format, hand the folders to be read here.
128    check_configs: list[list[int]]
129        list of list of supposed configs, eg. [range(1,1000)]
130        for one replicum with 1000 configs
131
132    Returns
133    -------
134    result: dict[list[Obs]]
135        dict with one of the following properties:
136        if keyed_out:
137            dict[key] = list[Obs]
138            where key has the form name/quarks/offset/wf/wf2
139        if not keyed_out:
140            dict[name][quarks][offset][wf][wf2] = list[Obs]
141    """
142
143    if kwargs.get('im'):
144        im = 1
145        part = 'imaginary'
146    else:
147        im = 0
148        part = 'real'
149
150    known_versions = ["0.0", "1.0", "2.0", "1.0c", "2.0c", "1.0a", "2.0a"]
151
152    if version not in known_versions:
153        raise Exception("This version is not known!")
154    if (version[-1] == "c"):
155        appended = False
156        compact = True
157        version = version[:-1]
158    elif (version[-1] == "a"):
159        appended = True
160        compact = False
161        version = version[:-1]
162    else:
163        compact = False
164        appended = False
165    ls = []
166    if "replica" in kwargs:
167        ls = kwargs.get("replica")
168    else:
169        for (dirpath, dirnames, filenames) in os.walk(path):
170            if not appended:
171                ls.extend(dirnames)
172            else:
173                ls.extend(filenames)
174            break
175        if not ls:
176            raise Exception('Error, directory not found')
177        # Exclude folders with different names
178        for exc in ls:
179            if not fnmatch.fnmatch(exc, prefix + '*'):
180                ls = list(set(ls) - set([exc]))
181
182    if not appended:
183        ls = sort_names(ls)
184        replica = len(ls)
185
186    else:
187        replica = len([file.split(".")[-1] for file in ls]) // len(set([file.split(".")[-1] for file in ls]))
188    if replica == 0:
189        raise Exception('No replica found in directory')
190    if not silent:
191        print('Read', part, 'part of', name_list, 'from', prefix[:-1], ',', replica, 'replica')
192
193    if 'names' in kwargs:
194        new_names = kwargs.get('names')
195        if len(new_names) != len(set(new_names)):
196            raise Exception("names are not unique!")
197        if len(new_names) != replica:
198            raise Exception('names should have the length', replica)
199
200    else:
201        ens_name = kwargs.get("ens_name")
202        if not appended:
203            new_names = _get_rep_names(ls, ens_name)
204        else:
205            new_names = _get_appended_rep_names(ls, prefix, name_list[0], ens_name)
206        new_names = sort_names(new_names)
207
208    idl = []
209
210    noffset_list = [str(x) for x in noffset_list]
211    wf_list = [str(x) for x in wf_list]
212    wf2_list = [str(x) for x in wf2_list]
213
214    # setup dict structures
215    intern = {}
216    for name, corr_type in zip(name_list, corr_type_list):
217        intern[name] = {}
218        b2b, single = _extract_corr_type(corr_type)
219        intern[name]["b2b"] = b2b
220        intern[name]["single"] = single
221        intern[name]["spec"] = {}
222        for quarks in quarks_list:
223            intern[name]["spec"][quarks] = {}
224            for off in noffset_list:
225                intern[name]["spec"][quarks][off] = {}
226                for w in wf_list:
227                    intern[name]["spec"][quarks][off][w] = {}
228                    if b2b:
229                        for w2 in wf2_list:
230                            intern[name]["spec"][quarks][off][w][w2] = {}
231                            intern[name]["spec"][quarks][off][w][w2]["pattern"] = _make_pattern(version, name, off, w, w2, intern[name]['b2b'], quarks)
232                    else:
233                        intern[name]["spec"][quarks][off][w]["0"] = {}
234                        intern[name]["spec"][quarks][off][w]["0"]["pattern"] = _make_pattern(version, name, off, w, 0, intern[name]['b2b'], quarks)
235
236    internal_ret_dict = {}
237    needed_keys = []
238    for name, corr_type in zip(name_list, corr_type_list):
239        b2b, single = _extract_corr_type(corr_type)
240        if b2b:
241            needed_keys.extend(_lists2key([name], quarks_list, noffset_list, wf_list, wf2_list))
242        else:
243            needed_keys.extend(_lists2key([name], quarks_list, noffset_list, wf_list, ["0"]))
244
245    for key in needed_keys:
246        internal_ret_dict[key] = []
247
248    if not appended:
249        for i, item in enumerate(ls):
250            rep_path = path + '/' + item
251            if "files" in kwargs:
252                files = kwargs.get("files")
253                if isinstance(files, list):
254                    if all(isinstance(f, list) for f in files):
255                        files = files[i]
256                    elif all(isinstance(f, str) for f in files):
257                        files = files
258                    else:
259                        raise TypeError("files has to be of type list[list[str]] or list[str]!")
260                else:
261                    raise TypeError("files has to be of type list[list[str]] or list[str]!")
262
263            else:
264                files = []
265            sub_ls = _find_files(rep_path, prefix, compact, files)
266            rep_idl = []
267            no_cfg = len(sub_ls)
268            for cfg in sub_ls:
269                try:
270                    if compact:
271                        rep_idl.append(int(cfg.split(cfg_separator)[-1]))
272                    else:
273                        rep_idl.append(int(cfg[3:]))
274                except Exception:
275                    raise Exception("Couldn't parse idl from directory, problem with file " + cfg)
276            rep_idl.sort()
277            # maybe there is a better way to print the idls
278            if not silent:
279                print(item, ':', no_cfg, ' configurations')
280            idl.append(rep_idl)
281            # here we have found all the files we need to look into.
282            if i == 0:
283                if version != "0.0" and compact:
284                    file = path + '/' + item + '/' + sub_ls[0]
285                for name_index, name in enumerate(name_list):
286                    if version == "0.0" or not compact:
287                        file = path + '/' + item + '/' + sub_ls[0] + '/' + name
288                    if corr_type_list[name_index] == 'bi':
289                        name_keys = _lists2key(quarks_list, noffset_list, wf_list, ["0"])
290                    else:
291                        name_keys = _lists2key(quarks_list, noffset_list, wf_list, wf2_list)
292                    for key in name_keys:
293                        specs = _key2specs(key)
294                        quarks = specs[0]
295                        off = specs[1]
296                        w = specs[2]
297                        w2 = specs[3]
298                        # here, we want to find the place within the file,
299                        # where the correlator we need is stored.
300                        # to do so, the pattern needed is put together
301                        # from the input values
302                        start_read, T = _find_correlator(file, version, intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["pattern"], intern[name]['b2b'], silent=silent)
303                        intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["start"] = start_read
304                        intern[name]["T"] = T
305                        # preparing the datastructure
306                        # the correlators get parsed into...
307                        deltas = []
308                        for j in range(intern[name]["T"]):
309                            deltas.append([])
310                        internal_ret_dict[sep.join([name, key])] = deltas
311
312            if compact:
313                rep_deltas = _read_compact_rep(path, item, sub_ls, intern, needed_keys, im)
314                for key in needed_keys:
315                    name = _key2specs(key)[0]
316                    for t in range(intern[name]["T"]):
317                        internal_ret_dict[key][t].append(rep_deltas[key][t])
318            else:
319                for key in needed_keys:
320                    rep_data = []
321                    name = _key2specs(key)[0]
322                    for subitem in sub_ls:
323                        cfg_path = path + '/' + item + '/' + subitem
324                        file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im)
325                        rep_data.append(file_data)
326                    for t in range(intern[name]["T"]):
327                        internal_ret_dict[key][t].append([])
328                        for cfg in range(no_cfg):
329                            internal_ret_dict[key][t][i].append(rep_data[cfg][key][t])
330    else:
331        for key in needed_keys:
332            specs = _key2specs(key)
333            name = specs[0]
334            quarks = specs[1]
335            off = specs[2]
336            w = specs[3]
337            w2 = specs[4]
338            if "files" in kwargs:
339                if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")):
340                    name_ls = kwargs.get("files")
341                else:
342                    raise TypeError("In append mode, files has to be of type list[str]!")
343            else:
344                name_ls = ls
345                for exc in name_ls:
346                    if not fnmatch.fnmatch(exc, prefix + '*.' + name):
347                        name_ls = list(set(name_ls) - set([exc]))
348            name_ls = sort_names(name_ls)
349            pattern = intern[name]['spec'][quarks][off][w][w2]['pattern']
350            deltas = []
351            for rep, file in enumerate(name_ls):
352                rep_idl = []
353                filename = path + '/' + file
354                T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single'])
355                if rep == 0:
356                    intern[name]['T'] = T
357                    for t in range(intern[name]['T']):
358                        deltas.append([])
359                for t in range(intern[name]['T']):
360                    deltas[t].append(rep_data[t])
361                internal_ret_dict[key] = deltas
362                if name == name_list[0]:
363                    idl.append(rep_idl)
364
365    if kwargs.get("check_configs") is True:
366        if not silent:
367            print("Checking for missing configs...")
368        che = kwargs.get("check_configs")
369        if not (len(che) == len(idl)):
370            raise Exception("check_configs has to be the same length as replica!")
371        for r in range(len(idl)):
372            if not silent:
373                print("checking " + new_names[r])
374            check_idl(idl[r], che[r])
375        if not silent:
376            print("Done")
377
378    result_dict = {}
379    if keyed_out:
380        for key in needed_keys:
381            name = _key2specs(key)[0]
382            result = []
383            for t in range(intern[name]["T"]):
384                result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl))
385            result_dict[key] = result
386    else:
387        for name, corr_type in zip(name_list, corr_type_list):
388            result_dict[name] = {}
389            for quarks in quarks_list:
390                result_dict[name][quarks] = {}
391                for off in noffset_list:
392                    result_dict[name][quarks][off] = {}
393                    for w in wf_list:
394                        result_dict[name][quarks][off][w] = {}
395                        if corr_type != 'bi':
396                            for w2 in wf2_list:
397                                key = _specs2key(name, quarks, off, w, w2)
398                                result = []
399                                for t in range(intern[name]["T"]):
400                                    result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl))
401                                result_dict[name][quarks][str(off)][str(w)][str(w2)] = result
402                        else:
403                            key = _specs2key(name, quarks, off, w, "0")
404                            result = []
405                            for t in range(intern[name]["T"]):
406                                result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl))
407                            result_dict[name][quarks][str(off)][str(w)][str(0)] = result
408    return result_dict

Read sfcf files from given folder structure.

Parameters
  • path (str): Path to the sfcf files.
  • prefix (str): Prefix of the sfcf files.
  • name (str): Name of the correlation function to read.
  • quarks_list (list[str]): Label of the quarks used in the sfcf input file. e.g. "quark quark" for version 0.0 this does NOT need to be given with the typical " - " that is present in the output file, this is done automatically for this version
  • corr_type_list (list[str]): Type of correlation function to read. Can be
    • 'bi' for boundary-inner
    • 'bb' for boundary-boundary
    • 'bib' for boundary-inner-boundary
  • noffset_list (list[int]): Offset of the source (only relevant when wavefunctions are used)
  • wf_list (int): ID of wave function
  • wf2_list (list[int]): ID of the second wavefunction (only relevant for boundary-to-boundary correlation functions)
  • im (bool): if True, read imaginary instead of real part of the correlation function.
  • names (list): Alternative labeling for replicas/ensembles. Has to have the appropriate length
  • ens_name (str): replaces the name of the ensemble
  • version (str): version of SFCF, with which the measurement was done. if the compact output option (-c) was specified, append a "c" to the version (e.g. "1.0c") if the append output option (-a) was specified, append an "a" to the version
  • cfg_separator (str): String that separates the ensemble identifier from the configuration number (default 'n').
  • replica (list): list of replica to be read, default is all
  • files (list[list[int]]): list of files to be read per replica, default is all. for non-compact output format, hand the folders to be read here.
  • check_configs (list[list[int]]): list of list of supposed configs, eg. [range(1,1000)] for one replicum with 1000 configs
Returns
  • result (dict[list[Obs]]): dict with one of the following properties: if keyed_out: dict[key] = list[Obs] where key has the form name/quarks/offset/wf/wf2 if not keyed_out: dict[name][quarks][offset][wf][wf2] = list[Obs]