pyerrors.input.sfcf

  0import os
  1import fnmatch
  2import re
  3import numpy as np  # Thinly-wrapped numpy
  4from ..obs import Obs
  5from . import utils
  6
  7
  8def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", **kwargs):
  9    """Read sfcf c format from given folder structure.
 10
 11    Parameters
 12    ----------
 13    quarks: str
 14        Label of the quarks used in the sfcf input file. e.g. "quark quark"
 15        for version 0.0 this does NOT need to be given with the typical " - "
 16        that is present in the output file,
 17        this is done automatically for this version
 18    noffset: int
 19        Offset of the source (only relevant when wavefunctions are used)
 20    wf: int
 21        ID of wave function
 22    wf2: int
 23        ID of the second wavefunction
 24        (only relevant for boundary-to-boundary correlation functions)
 25    im: bool
 26        if True, read imaginary instead of real part
 27        of the correlation function.
 28    corr_type : str
 29        change between bi (boundary - inner) (default) bib (boundary - inner - boundary) and bb (boundary - boundary)
 30        correlator types
 31    names : list
 32        Alternative labeling for replicas/ensembles.
 33        Has to have the appropriate length
 34    ens_name : str
 35        replaces the name of the ensemble
 36    version: str
 37        version of SFCF, with which the measurement was done.
 38        if the compact output option (-c) was specified,
 39        append a "c" to the version (e.g. "1.0c")
 40        if the append output option (-a) was specified,
 41        append an "a" to the version
 42    cfg_separator : str
 43        String that separates the ensemble identifier from the configuration number (default 'n').
 44    replica: list
 45        list of replica to be read, default is all
 46    files: list
 47        list of files to be read per replica, default is all.
 48        for non-compact output format, hand the folders to be read here.
 49    check_configs:
 50        list of list of supposed configs, eg. [range(1,1000)]
 51        for one replicum with 1000 configs
 52    """
 53    if kwargs.get('im'):
 54        im = 1
 55        part = 'imaginary'
 56    else:
 57        im = 0
 58        part = 'real'
 59    if "replica" in kwargs:
 60        reps = kwargs.get("replica")
 61    if corr_type == 'bb':
 62        b2b = True
 63        single = True
 64    elif corr_type == 'bib':
 65        b2b = True
 66        single = False
 67    else:
 68        b2b = False
 69        single = False
 70    compact = True
 71    appended = False
 72    known_versions = ["0.0", "1.0", "2.0", "1.0c", "2.0c", "1.0a", "2.0a"]
 73
 74    if version not in known_versions:
 75        raise Exception("This version is not known!")
 76    if(version[-1] == "c"):
 77        appended = False
 78        compact = True
 79        version = version[:-1]
 80    elif(version[-1] == "a"):
 81        appended = True
 82        compact = False
 83        version = version[:-1]
 84    else:
 85        compact = False
 86        appended = False
 87    read = 0
 88    T = 0
 89    start = 0
 90    ls = []
 91    if "replica" in kwargs:
 92        ls = reps
 93    else:
 94        for (dirpath, dirnames, filenames) in os.walk(path):
 95            if not appended:
 96                ls.extend(dirnames)
 97            else:
 98                ls.extend(filenames)
 99            break
100        if not ls:
101            raise Exception('Error, directory not found')
102        # Exclude folders with different names
103        for exc in ls:
104            if not fnmatch.fnmatch(exc, prefix + '*'):
105                ls = list(set(ls) - set([exc]))
106
107    if not appended:
108        if len(ls) > 1:
109            # New version, to cope with ids, etc.
110            ls.sort(key=lambda x: int(re.findall(r'\d+', x[len(prefix):])[0]))
111        replica = len(ls)
112    else:
113        replica = len([file.split(".")[-1] for file in ls]) // len(set([file.split(".")[-1] for file in ls]))
114    print('Read', part, 'part of', name, 'from', prefix[:-1],
115          ',', replica, 'replica')
116    if 'names' in kwargs:
117        new_names = kwargs.get('names')
118        if len(new_names) != len(set(new_names)):
119            raise Exception("names are not unique!")
120        if len(new_names) != replica:
121            raise Exception('Names does not have the required length', replica)
122    else:
123        new_names = []
124        if not appended:
125            for entry in ls:
126                try:
127                    idx = entry.index('r')
128                except Exception:
129                    raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.")
130
131                if 'ens_name' in kwargs:
132                    new_names.append(kwargs.get('ens_name') + '|' + entry[idx:])
133                else:
134                    new_names.append(entry[:idx] + '|' + entry[idx:])
135        else:
136
137            for exc in ls:
138                if not fnmatch.fnmatch(exc, prefix + '*.' + name):
139                    ls = list(set(ls) - set([exc]))
140            ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1]))
141            for entry in ls:
142                myentry = entry[:-len(name) - 1]
143                try:
144                    idx = myentry.index('r')
145                except Exception:
146                    raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.")
147
148                if 'ens_name' in kwargs:
149                    new_names.append(kwargs.get('ens_name') + '|' + myentry[idx:])
150                else:
151                    new_names.append(myentry[:idx] + '|' + myentry[idx:])
152    idl = []
153    if not appended:
154        for i, item in enumerate(ls):
155            sub_ls = []
156            if "files" in kwargs:
157                sub_ls = kwargs.get("files")
158                sub_ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1]))
159            else:
160                for (dirpath, dirnames, filenames) in os.walk(path + '/' + item):
161                    if compact:
162                        sub_ls.extend(filenames)
163                    else:
164                        sub_ls.extend(dirnames)
165                    break
166                if compact:
167                    for exc in sub_ls:
168                        if not fnmatch.fnmatch(exc, prefix + '*'):
169                            sub_ls = list(set(sub_ls) - set([exc]))
170                    sub_ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1]))
171                else:
172                    for exc in sub_ls:
173                        if not fnmatch.fnmatch(exc, 'cfg*'):
174                            sub_ls = list(set(sub_ls) - set([exc]))
175                    sub_ls.sort(key=lambda x: int(x[3:]))
176            rep_idl = []
177            no_cfg = len(sub_ls)
178            for cfg in sub_ls:
179                try:
180                    if compact:
181                        rep_idl.append(int(cfg.split(cfg_separator)[-1]))
182                    else:
183                        rep_idl.append(int(cfg[3:]))
184                except Exception:
185                    raise Exception("Couldn't parse idl from directroy, problem with file " + cfg)
186            rep_idl.sort()
187            # maybe there is a better way to print the idls
188            print(item, ':', no_cfg, ' configurations')
189            idl.append(rep_idl)
190            # here we have found all the files we need to look into.
191            if i == 0:
192                # here, we want to find the place within the file,
193                # where the correlator we need is stored.
194                # to do so, the pattern needed is put together
195                # from the input values
196                if version == "0.0":
197                    pattern = "# " + name + " : offset " + str(noffset) + ", wf " + str(wf)
198                    # if b2b, a second wf is needed
199                    if b2b:
200                        pattern += ", wf_2 " + str(wf2)
201                    qs = quarks.split(" ")
202                    pattern += " : " + qs[0] + " - " + qs[1]
203                    file = open(path + '/' + item + '/' + sub_ls[0] + '/' + name, "r")
204                    for k, line in enumerate(file):
205                        if read == 1 and not line.strip() and k > start + 1:
206                            break
207                        if read == 1 and k >= start:
208                            T += 1
209                        if pattern in line:
210                            read = 1
211                            start = k + 1
212                    print(str(T) + " entries found.")
213                    file.close()
214                else:
215                    pattern = 'name      ' + name + '\nquarks    ' + quarks + '\noffset    ' + str(noffset) + '\nwf        ' + str(wf)
216                    if b2b:
217                        pattern += '\nwf_2      ' + str(wf2)
218                    # and the file is parsed through to find the pattern
219                    if compact:
220                        file = open(path + '/' + item + '/' + sub_ls[0], "r")
221                    else:
222                        # for non-compactified versions of the files
223                        file = open(path + '/' + item + '/' + sub_ls[0] + '/' + name, "r")
224
225                    content = file.read()
226                    match = re.search(pattern, content)
227                    if match:
228                        start_read = content.count('\n', 0, match.start()) + 5 + b2b
229                        end_match = re.search(r'\n\s*\n', content[match.start():])
230                        T = content[match.start():].count('\n', 0, end_match.start()) - 4 - b2b
231                        assert T > 0
232                        print(T, 'entries, starting to read in line', start_read)
233                        file.close()
234                    else:
235                        file.close()
236                        raise Exception('Correlator with pattern\n' + pattern + '\nnot found.')
237
238                # we found where the correlator
239                # that is to be read is in the files
240                # after preparing the datastructure
241                # the correlators get parsed into...
242                deltas = []
243                for j in range(T):
244                    deltas.append([])
245
246            for t in range(T):
247                deltas[t].append(np.zeros(no_cfg))
248            if compact:
249                for cfg in range(no_cfg):
250                    with open(path + '/' + item + '/' + sub_ls[cfg]) as fp:
251                        lines = fp.readlines()
252                        # check, if the correlator is in fact
253                        # printed completely
254                        if(start_read + T > len(lines)):
255                            raise Exception("EOF before end of correlator data! Maybe " + path + '/' + item + '/' + sub_ls[cfg] + " is corrupted?")
256                        # and start to read the correlator.
257                        # the range here is chosen like this,
258                        # since this allows for implementing
259                        # a security check for every read correlator later...
260                        for k in range(start_read - 6, start_read + T):
261                            if k == start_read - 5 - b2b:
262                                if lines[k].strip() != 'name      ' + name:
263                                    raise Exception('Wrong format', sub_ls[cfg])
264                            if(k >= start_read and k < start_read + T):
265                                floats = list(map(float, lines[k].split()))
266                                deltas[k - start_read][i][cfg] = floats[-2:][im]
267            else:
268                for cnfg, subitem in enumerate(sub_ls):
269                    with open(path + '/' + item + '/' + subitem + '/' + name) as fp:
270                        # since the non-compatified files
271                        # are typically not so long,
272                        # we can iterate over the whole file.
273                        # here one can also implement the chekc from above.
274                        for k, line in enumerate(fp):
275                            if(k >= start_read and k < start_read + T):
276                                floats = list(map(float, line.split()))
277                                if version == "0.0":
278                                    deltas[k - start][i][cnfg] = floats[im - single]
279                                else:
280                                    deltas[k - start_read][i][cnfg] = floats[1 + im - single]
281
282    else:
283        if "files" in kwargs:
284            ls = kwargs.get("files")
285        else:
286            for exc in ls:
287                if not fnmatch.fnmatch(exc, prefix + '*.' + name):
288                    ls = list(set(ls) - set([exc]))
289                ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1]))
290        pattern = 'name      ' + name + '\nquarks    ' + quarks + '\noffset    ' + str(noffset) + '\nwf        ' + str(wf)
291        if b2b:
292            pattern += '\nwf_2      ' + str(wf2)
293        for rep, file in enumerate(ls):
294            rep_idl = []
295            with open(path + '/' + file, 'r') as fp:
296                content = fp.readlines()
297                data_starts = []
298                for linenumber, line in enumerate(content):
299                    if "[run]" in line:
300                        data_starts.append(linenumber)
301                if len(set([data_starts[i] - data_starts[i - 1] for i in range(1, len(data_starts))])) > 1:
302                    raise Exception("Irregularities in file structure found, not all runs have the same output length")
303                chunk = content[:data_starts[1]]
304                for linenumber, line in enumerate(chunk):
305                    if line.startswith("gauge_name"):
306                        gauge_line = linenumber
307                    elif line.startswith("[correlator]"):
308                        corr_line = linenumber
309                        found_pat = ""
310                        for li in chunk[corr_line + 1: corr_line + 6 + b2b]:
311                            found_pat += li
312                        if re.search(pattern, found_pat):
313                            start_read = corr_line + 7 + b2b
314                            break
315                endline = corr_line + 6 + b2b
316                while not chunk[endline] == "\n":
317                    endline += 1
318                T = endline - start_read
319                if rep == 0:
320                    deltas = []
321                    for t in range(T):
322                        deltas.append([])
323                for t in range(T):
324                    deltas[t].append(np.zeros(len(data_starts)))
325                # all other chunks should follow the same structure
326                for cnfg in range(len(data_starts)):
327                    start = data_starts[cnfg]
328                    stop = start + data_starts[1]
329                    chunk = content[start:stop]
330                    try:
331                        rep_idl.append(int(chunk[gauge_line].split(cfg_separator)[-1]))
332                    except Exception:
333                        raise Exception("Couldn't parse idl from directory, problem with chunk around line ", gauge_line)
334
335                    found_pat = ""
336                    for li in chunk[corr_line + 1:corr_line + 6 + b2b]:
337                        found_pat += li
338                    if re.search(pattern, found_pat):
339                        for t, line in enumerate(chunk[start_read:start_read + T]):
340                            floats = list(map(float, line.split()))
341                            deltas[t][rep][cnfg] = floats[im + 1 - single]
342            idl.append(rep_idl)
343
344    if "check_configs" in kwargs:
345        print("Checking for missing configs...")
346        che = kwargs.get("check_configs")
347        if not (len(che) == len(idl)):
348            raise Exception("check_configs has to be the same length as replica!")
349        for r in range(len(idl)):
350            print("checking " + new_names[r])
351            utils.check_idl(idl[r], che[r])
352        print("Done")
353    result = []
354    for t in range(T):
355        result.append(Obs(deltas[t], new_names, idl=idl))
356    return result