diff --git a/pyerrors/input/openQCD.py b/pyerrors/input/openQCD.py index 6b11fb0b..c89f40e0 100644 --- a/pyerrors/input/openQCD.py +++ b/pyerrors/input/openQCD.py @@ -1,6 +1,5 @@ import os import fnmatch -import re import struct import warnings import numpy as np # Thinly-wrapped numpy @@ -10,75 +9,7 @@ from ..obs import Obs from ..fits import fit_lin from ..obs import CObs from ..correlators import Corr - - -def _find_files(path, prefix, postfix, ext, known_files=[]): - found = [] - files = [] - - if postfix != "": - if postfix[-1] != ".": - postfix = postfix + "." - if postfix[0] != ".": - postfix = "." + postfix - - if ext[0] == ".": - ext = ext[1:] - - pattern = prefix + "*" + postfix + ext - - for (dirpath, dirnames, filenames) in os.walk(path + "/"): - found.extend(filenames) - break - - if known_files != []: - for kf in known_files: - if kf not in found: - raise FileNotFoundError("Given file " + kf + " does not exist!") - - return known_files - - if not found: - raise FileNotFoundError(f"Error, directory '{path}' not found") - - for f in found: - if fnmatch.fnmatch(f, pattern): - files.append(f) - - if files == []: - raise Exception("No files found after pattern filter!") - - files = _sort_names(files) - return files - - -def _sort_names(ll): - r_pattern = r'r(\d+)' - id_pattern = r'id(\d+)' - - # sort list by id first - if all([re.search(id_pattern, entry) for entry in ll]): - ll.sort(key=lambda x: int(re.findall(id_pattern, x)[0])) - # then by replikum - if all([re.search(r_pattern, entry) for entry in ll]): - ll.sort(key=lambda x: int(re.findall(r_pattern, x)[0])) - # as the rearrangements by one key let the other key untouched, the list is sorted now - - else: - # fallback - sames = '' - if len(ll) > 1: - for i in range(len(ll[0])): - checking = ll[0][i] - for rn in ll[1:]: - is_same = (rn[i] == checking) - if is_same: - sames += checking - else: - break - print(ll[0][len(sames):]) - ll.sort(key=lambda x: int(re.findall(r'\d+', x[len(sames):])[0])) - return ll +from .utils import sort_names def read_rwms(path, prefix, version='2.0', names=None, **kwargs): @@ -171,7 +102,7 @@ def read_rwms(path, prefix, version='2.0', names=None, **kwargs): else: rep_names = names - rep_names = _sort_names(rep_names) + rep_names = sort_names(rep_names) print_err = 0 if 'print_err' in kwargs: @@ -561,6 +492,46 @@ def _parse_array_openQCD2(d, n, size, wa, quadrupel=False): return arr +def _find_files(path, prefix, postfix, ext, known_files=[]): + found = [] + files = [] + + if postfix != "": + if postfix[-1] != ".": + postfix = postfix + "." + if postfix[0] != ".": + postfix = "." + postfix + + if ext[0] == ".": + ext = ext[1:] + + pattern = prefix + "*" + postfix + ext + + for (dirpath, dirnames, filenames) in os.walk(path + "/"): + found.extend(filenames) + break + + if known_files != []: + for kf in known_files: + if kf not in found: + raise FileNotFoundError("Given file " + kf + " does not exist!") + + return known_files + + if not found: + raise FileNotFoundError(f"Error, directory '{path}' not found") + + for f in found: + if fnmatch.fnmatch(f, pattern): + files.append(f) + + if files == []: + raise Exception("No files found after pattern filter!") + + files = sort_names(files) + return files + + def _read_array_openQCD2(fp): t = fp.read(4) d = struct.unpack('i', t)[0] @@ -974,7 +945,7 @@ def _read_flow_obs(path, prefix, c, dtr_cnfg=1, version="openQCD", obspos=0, sum deltas.append(Q_top) - rep_names = _sort_names(rep_names) + rep_names = sort_names(rep_names) idl = [range(int(configlist[rep][r_start_index[rep]]), int(configlist[rep][r_stop_index[rep]]) + 1, 1) for rep in range(len(deltas))] deltas = [deltas[nrep][r_start_index[nrep]:r_stop_index[nrep] + 1] for nrep in range(len(deltas))] diff --git a/pyerrors/input/sfcf.py b/pyerrors/input/sfcf.py index c64c8ffd..f069d2bf 100644 --- a/pyerrors/input/sfcf.py +++ b/pyerrors/input/sfcf.py @@ -3,11 +3,11 @@ import fnmatch import re import numpy as np # Thinly-wrapped numpy from ..obs import Obs -from . import utils +from .utils import sort_names, check_idl -def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", **kwargs): - """Read sfcf c format from given folder structure. +def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", silent=False, **kwargs): + """Read sfcf files from given folder structure. Parameters ---------- @@ -71,8 +71,7 @@ def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0, else: im = 0 part = 'real' - if "replica" in kwargs: - reps = kwargs.get("replica") + if corr_type == 'bb': b2b = True single = True @@ -82,8 +81,7 @@ def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0, else: b2b = False single = False - compact = True - appended = False + known_versions = ["0.0", "1.0", "2.0", "1.0c", "2.0c", "1.0a", "2.0a"] if version not in known_versions: @@ -99,12 +97,9 @@ def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0, else: compact = False appended = False - read = 0 - T = 0 - start = 0 ls = [] if "replica" in kwargs: - ls = reps + ls = kwargs.get("replica") else: for (dirpath, dirnames, filenames) in os.walk(path): if not appended: @@ -120,74 +115,38 @@ def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0, ls = list(set(ls) - set([exc])) if not appended: - if len(ls) > 1: - # New version, to cope with ids, etc. - ls.sort(key=lambda x: int(re.findall(r'\d+', x[len(prefix):])[0])) + ls = sort_names(ls) replica = len(ls) + else: replica = len([file.split(".")[-1] for file in ls]) // len(set([file.split(".")[-1] for file in ls])) - print('Read', part, 'part of', name, 'from', prefix[:-1], - ',', replica, 'replica') + if not silent: + print('Read', part, 'part of', name, 'from', prefix[:-1], ',', replica, 'replica') + if 'names' in kwargs: new_names = kwargs.get('names') if len(new_names) != len(set(new_names)): raise Exception("names are not unique!") if len(new_names) != replica: - raise Exception('Names does not have the required length', replica) + raise Exception('names should have the length', replica) + else: - new_names = [] + ens_name = kwargs.get("ens_name") if not appended: - for entry in ls: - try: - idx = entry.index('r') - except Exception: - raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") - - if 'ens_name' in kwargs: - new_names.append(kwargs.get('ens_name') + '|' + entry[idx:]) - else: - new_names.append(entry[:idx] + '|' + entry[idx:]) + new_names = _get_rep_names(ls, ens_name) else: + new_names = _get_appended_rep_names(ls, prefix, name, ens_name) + new_names = sort_names(new_names) - for exc in ls: - if not fnmatch.fnmatch(exc, prefix + '*.' + name): - ls = list(set(ls) - set([exc])) - ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) - for entry in ls: - myentry = entry[:-len(name) - 1] - try: - idx = myentry.index('r') - except Exception: - raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") - - if 'ens_name' in kwargs: - new_names.append(kwargs.get('ens_name') + '|' + myentry[idx:]) - else: - new_names.append(myentry[:idx] + '|' + myentry[idx:]) idl = [] if not appended: for i, item in enumerate(ls): - sub_ls = [] + rep_path = path + '/' + item if "files" in kwargs: - sub_ls = kwargs.get("files") - sub_ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) + files = kwargs.get("files") else: - for (dirpath, dirnames, filenames) in os.walk(path + '/' + item): - if compact: - sub_ls.extend(filenames) - else: - sub_ls.extend(dirnames) - break - if compact: - for exc in sub_ls: - if not fnmatch.fnmatch(exc, prefix + '*'): - sub_ls = list(set(sub_ls) - set([exc])) - sub_ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) - else: - for exc in sub_ls: - if not fnmatch.fnmatch(exc, 'cfg*'): - sub_ls = list(set(sub_ls) - set([exc])) - sub_ls.sort(key=lambda x: int(x[3:])) + files = [] + sub_ls = _find_files(rep_path, prefix, compact, files) rep_idl = [] no_cfg = len(sub_ls) for cfg in sub_ls: @@ -200,7 +159,8 @@ def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0, raise Exception("Couldn't parse idl from directroy, problem with file " + cfg) rep_idl.sort() # maybe there is a better way to print the idls - print(item, ':', no_cfg, ' configurations') + if not silent: + print(item, ':', no_cfg, ' configurations') idl.append(rep_idl) # here we have found all the files we need to look into. if i == 0: @@ -209,88 +169,37 @@ def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0, # to do so, the pattern needed is put together # from the input values if version == "0.0": - pattern = "# " + name + " : offset " + str(noffset) + ", wf " + str(wf) - # if b2b, a second wf is needed - if b2b: - pattern += ", wf_2 " + str(wf2) - qs = quarks.split(" ") - pattern += " : " + qs[0] + " - " + qs[1] - file = open(path + '/' + item + '/' + sub_ls[0] + '/' + name, "r") - for k, line in enumerate(file): - if read == 1 and not line.strip() and k > start + 1: - break - if read == 1 and k >= start: - T += 1 - if pattern in line: - read = 1 - start = k + 1 - print(str(T) + " entries found.") - file.close() + file = path + '/' + item + '/' + sub_ls[0] + '/' + name else: - pattern = 'name ' + name + '\nquarks ' + quarks + '\noffset ' + str(noffset) + '\nwf ' + str(wf) - if b2b: - pattern += '\nwf_2 ' + str(wf2) - # and the file is parsed through to find the pattern if compact: - file = open(path + '/' + item + '/' + sub_ls[0], "r") + file = path + '/' + item + '/' + sub_ls[0] else: - # for non-compactified versions of the files - file = open(path + '/' + item + '/' + sub_ls[0] + '/' + name, "r") + file = path + '/' + item + '/' + sub_ls[0] + '/' + name - content = file.read() - match = re.search(pattern, content) - if match: - start_read = content.count('\n', 0, match.start()) + 5 + b2b - end_match = re.search(r'\n\s*\n', content[match.start():]) - T = content[match.start():].count('\n', 0, end_match.start()) - 4 - b2b - assert T > 0 - print(T, 'entries, starting to read in line', start_read) - file.close() - else: - file.close() - raise Exception('Correlator with pattern\n' + pattern + '\nnot found.') + pattern = _make_pattern(version, name, noffset, wf, wf2, b2b, quarks) + start_read, T = _find_correlator(file, version, pattern, b2b, silent=silent) - # we found where the correlator - # that is to be read is in the files - # after preparing the datastructure + # preparing the datastructure # the correlators get parsed into... deltas = [] for j in range(T): deltas.append([]) - for t in range(T): - deltas[t].append(np.zeros(no_cfg)) if compact: - for cfg in range(no_cfg): - with open(path + '/' + item + '/' + sub_ls[cfg]) as fp: - lines = fp.readlines() - # check, if the correlator is in fact - # printed completely - if (start_read + T > len(lines)): - raise Exception("EOF before end of correlator data! Maybe " + path + '/' + item + '/' + sub_ls[cfg] + " is corrupted?") - # and start to read the correlator. - # the range here is chosen like this, - # since this allows for implementing - # a security check for every read correlator later... - for k in range(start_read - 6, start_read + T): - if k == start_read - 5 - b2b: - if lines[k].strip() != 'name ' + name: - raise Exception('Wrong format', sub_ls[cfg]) - if (k >= start_read and k < start_read + T): - floats = list(map(float, lines[k].split())) - deltas[k - start_read][i][cfg] = floats[-2:][im] + rep_deltas = _read_compact_rep(path, item, sub_ls, start_read, T, b2b, name, im) + + for t in range(T): + deltas[t].append(rep_deltas[t]) else: + for t in range(T): + deltas[t].append(np.zeros(no_cfg)) for cnfg, subitem in enumerate(sub_ls): with open(path + '/' + item + '/' + subitem + '/' + name) as fp: - # since the non-compatified files - # are typically not so long, - # we can iterate over the whole file. - # here one can also implement the chekc from above. for k, line in enumerate(fp): if (k >= start_read and k < start_read + T): floats = list(map(float, line.split())) if version == "0.0": - deltas[k - start][i][cnfg] = floats[im - single] + deltas[k - start_read][i][cnfg] = floats[im - single] else: deltas[k - start_read][i][cnfg] = floats[1 + im - single] @@ -301,71 +210,237 @@ def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0, for exc in ls: if not fnmatch.fnmatch(exc, prefix + '*.' + name): ls = list(set(ls) - set([exc])) - ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) - pattern = 'name ' + name + '\nquarks ' + quarks + '\noffset ' + str(noffset) + '\nwf ' + str(wf) - if b2b: - pattern += '\nwf_2 ' + str(wf2) + ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) + pattern = _make_pattern(version, name, noffset, wf, wf2, b2b, quarks) + deltas = [] for rep, file in enumerate(ls): rep_idl = [] - with open(path + '/' + file, 'r') as fp: - content = fp.readlines() - data_starts = [] - for linenumber, line in enumerate(content): - if "[run]" in line: - data_starts.append(linenumber) - if len(set([data_starts[i] - data_starts[i - 1] for i in range(1, len(data_starts))])) > 1: - raise Exception("Irregularities in file structure found, not all runs have the same output length") - chunk = content[:data_starts[1]] - for linenumber, line in enumerate(chunk): - if line.startswith("gauge_name"): - gauge_line = linenumber - elif line.startswith("[correlator]"): - corr_line = linenumber - found_pat = "" - for li in chunk[corr_line + 1: corr_line + 6 + b2b]: - found_pat += li - if re.search(pattern, found_pat): - start_read = corr_line + 7 + b2b - break - endline = corr_line + 6 + b2b - while not chunk[endline] == "\n": - endline += 1 - T = endline - start_read - if rep == 0: - deltas = [] - for t in range(T): - deltas.append([]) + filename = path + '/' + file + T, rep_idl, rep_data = _read_append_rep(filename, pattern, b2b, cfg_separator, im, single) + if rep == 0: for t in range(T): - deltas[t].append(np.zeros(len(data_starts))) - # all other chunks should follow the same structure - for cnfg in range(len(data_starts)): - start = data_starts[cnfg] - stop = start + data_starts[1] - chunk = content[start:stop] - try: - rep_idl.append(int(chunk[gauge_line].split(cfg_separator)[-1])) - except Exception: - raise Exception("Couldn't parse idl from directory, problem with chunk around line ", gauge_line) - - found_pat = "" - for li in chunk[corr_line + 1:corr_line + 6 + b2b]: - found_pat += li - if re.search(pattern, found_pat): - for t, line in enumerate(chunk[start_read:start_read + T]): - floats = list(map(float, line.split())) - deltas[t][rep][cnfg] = floats[im + 1 - single] + deltas.append([]) + for t in range(T): + deltas[t].append(rep_data[t]) idl.append(rep_idl) if "check_configs" in kwargs: - print("Checking for missing configs...") + if not silent: + print("Checking for missing configs...") che = kwargs.get("check_configs") if not (len(che) == len(idl)): raise Exception("check_configs has to be the same length as replica!") for r in range(len(idl)): - print("checking " + new_names[r]) - utils.check_idl(idl[r], che[r]) - print("Done") + if not silent: + print("checking " + new_names[r]) + check_idl(idl[r], che[r]) + if not silent: + print("Done") result = [] for t in range(T): result.append(Obs(deltas[t], new_names, idl=idl)) return result + + +def _find_files(rep_path, prefix, compact, files=[]): + sub_ls = [] + if not files == []: + files.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) + else: + for (dirpath, dirnames, filenames) in os.walk(rep_path): + if compact: + sub_ls.extend(filenames) + else: + sub_ls.extend(dirnames) + break + if compact: + for exc in sub_ls: + if not fnmatch.fnmatch(exc, prefix + '*'): + sub_ls = list(set(sub_ls) - set([exc])) + sub_ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) + else: + for exc in sub_ls: + if not fnmatch.fnmatch(exc, 'cfg*'): + sub_ls = list(set(sub_ls) - set([exc])) + sub_ls.sort(key=lambda x: int(x[3:])) + files = sub_ls + if len(files) == 0: + raise FileNotFoundError("Did not find files in", rep_path, "with prefix", prefix, "and the given structure.") + return files + + +def _make_pattern(version, name, noffset, wf, wf2, b2b, quarks): + if version == "0.0": + pattern = "# " + name + " : offset " + str(noffset) + ", wf " + str(wf) + if b2b: + pattern += ", wf_2 " + str(wf2) + qs = quarks.split(" ") + pattern += " : " + qs[0] + " - " + qs[1] + else: + pattern = 'name ' + name + '\nquarks ' + quarks + '\noffset ' + str(noffset) + '\nwf ' + str(wf) + if b2b: + pattern += '\nwf_2 ' + str(wf2) + return pattern + + +def _find_correlator(file_name, version, pattern, b2b, silent=False): + T = 0 + + file = open(file_name, "r") + + content = file.read() + match = re.search(pattern, content) + if match: + if version == "0.0": + start_read = content.count('\n', 0, match.start()) + 1 + T = content.count('\n', start_read) + else: + start_read = content.count('\n', 0, match.start()) + 5 + b2b + end_match = re.search(r'\n\s*\n', content[match.start():]) + T = content[match.start():].count('\n', 0, end_match.start()) - 4 - b2b + if not T > 0: + raise ValueError("Correlator with pattern\n" + pattern + "\nis empty!") + if not silent: + print(T, 'entries, starting to read in line', start_read) + + else: + file.close() + raise ValueError('Correlator with pattern\n' + pattern + '\nnot found.') + + file.close() + return start_read, T + + +def _read_compact_file(rep_path, config_file, start_read, T, b2b, name, im): + with open(rep_path + config_file) as fp: + lines = fp.readlines() + # check, if the correlator is in fact + # printed completely + if (start_read + T + 1 > len(lines)): + raise Exception("EOF before end of correlator data! Maybe " + rep_path + config_file + " is corrupted?") + corr_lines = lines[start_read - 6: start_read + T] + del lines + t_vals = [] + + if corr_lines[1 - b2b].strip() != 'name ' + name: + raise Exception('Wrong format in file', config_file) + + for k in range(6, T + 6): + floats = list(map(float, corr_lines[k].split())) + t_vals.append(floats[-2:][im]) + return t_vals + + +def _read_compact_rep(path, rep, sub_ls, start_read, T, b2b, name, im): + rep_path = path + '/' + rep + '/' + no_cfg = len(sub_ls) + deltas = [] + for t in range(T): + deltas.append(np.zeros(no_cfg)) + for cfg in range(no_cfg): + cfg_file = sub_ls[cfg] + cfg_data = _read_compact_file(rep_path, cfg_file, start_read, T, b2b, name, im) + for t in range(T): + deltas[t][cfg] = cfg_data[t] + return deltas + + +def _read_chunk(chunk, gauge_line, cfg_sep, start_read, T, corr_line, b2b, pattern, im, single): + try: + idl = int(chunk[gauge_line].split(cfg_sep)[-1]) + except Exception: + raise Exception("Couldn't parse idl from directory, problem with chunk around line ", gauge_line) + + found_pat = "" + data = [] + for li in chunk[corr_line + 1:corr_line + 6 + b2b]: + found_pat += li + if re.search(pattern, found_pat): + for t, line in enumerate(chunk[start_read:start_read + T]): + floats = list(map(float, line.split())) + data.append(floats[im + 1 - single]) + return idl, data + + +def _read_append_rep(filename, pattern, b2b, cfg_separator, im, single): + with open(filename, 'r') as fp: + content = fp.readlines() + data_starts = [] + for linenumber, line in enumerate(content): + if "[run]" in line: + data_starts.append(linenumber) + if len(set([data_starts[i] - data_starts[i - 1] for i in range(1, len(data_starts))])) > 1: + raise Exception("Irregularities in file structure found, not all runs have the same output length") + chunk = content[:data_starts[1]] + for linenumber, line in enumerate(chunk): + if line.startswith("gauge_name"): + gauge_line = linenumber + elif line.startswith("[correlator]"): + corr_line = linenumber + found_pat = "" + for li in chunk[corr_line + 1: corr_line + 6 + b2b]: + found_pat += li + if re.search(pattern, found_pat): + start_read = corr_line + 7 + b2b + break + else: + raise ValueError("Did not find pattern\n", pattern, "\nin\n", filename) + endline = corr_line + 6 + b2b + while not chunk[endline] == "\n": + endline += 1 + T = endline - start_read + + # all other chunks should follow the same structure + rep_idl = [] + rep_data = [] + + for cnfg in range(len(data_starts)): + start = data_starts[cnfg] + stop = start + data_starts[1] + chunk = content[start:stop] + idl, data = _read_chunk(chunk, gauge_line, cfg_separator, start_read, T, corr_line, b2b, pattern, im, single) + rep_idl.append(idl) + rep_data.append(data) + + data = [] + + for t in range(T): + data.append([]) + for c in range(len(rep_data)): + data[t].append(rep_data[c][t]) + return T, rep_idl, data + + +def _get_rep_names(ls, ens_name=None): + new_names = [] + for entry in ls: + try: + idx = entry.index('r') + except Exception: + raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") + + if ens_name: + new_names.append('ens_name' + '|' + entry[idx:]) + else: + new_names.append(entry[:idx] + '|' + entry[idx:]) + return new_names + + +def _get_appended_rep_names(ls, prefix, name, ens_name=None): + new_names = [] + for exc in ls: + if not fnmatch.fnmatch(exc, prefix + '*.' + name): + ls = list(set(ls) - set([exc])) + ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) + for entry in ls: + myentry = entry[:-len(name) - 1] + try: + idx = myentry.index('r') + except Exception: + raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") + + if ens_name: + new_names.append('ens_name' + '|' + entry[idx:]) + else: + new_names.append(myentry[:idx] + '|' + myentry[idx:]) + return new_names diff --git a/pyerrors/input/utils.py b/pyerrors/input/utils.py index be8436a6..6cabe0b5 100644 --- a/pyerrors/input/utils.py +++ b/pyerrors/input/utils.py @@ -1,6 +1,51 @@ +import re """Utilities for the input""" +def sort_names(ll): + """Sorts a list of names of replika with searches for `r` and `id` in the replikum string. + If this search fails, a fallback method is used, + where the strings are simply compared and the first diffeing numeral is used for differentiation. + + Parameters + ---------- + ll: list + list to sort + + Returns + ------- + ll: list + sorted list + """ + if len(ll) > 1: + r_pattern = r'r(\d+)' + id_pattern = r'id(\d+)' + + # sort list by id first + if all([re.search(id_pattern, entry) for entry in ll]): + ll.sort(key=lambda x: int(re.findall(id_pattern, x)[0])) + # then by replikum + if all([re.search(r_pattern, entry) for entry in ll]): + ll.sort(key=lambda x: int(re.findall(r_pattern, x)[0])) + # as the rearrangements by one key let the other key untouched, the list is sorted now + + else: + # fallback + sames = '' + if len(ll) > 1: + for i in range(len(ll[0])): + checking = ll[0][i] + for rn in ll[1:]: + is_same = (rn[i] == checking) + if is_same: + sames += checking + else: + break + print("Using prefix:", ll[0][len(sames):]) + ll.sort(key=lambda x: int(re.findall(r'\d+', x[len(sames):])[0])) + return ll + + def check_idl(idl, che): """Checks if list of configurations is contained in an idl diff --git a/tests/data/sfcf_test/broken_data_c/data_c_r0/data_c_r0_n1 b/tests/data/sfcf_test/broken_data_c/data_c_r0/data_c_r0_n1 new file mode 100644 index 00000000..9b9d694d --- /dev/null +++ b/tests/data/sfcf_test/broken_data_c/data_c_r0/data_c_r0_n1 @@ -0,0 +1,472 @@ +[run] + +version 2.1 +date 2022-01-19 11:03:58 +0100 +host r04n07.palma.wwu +dir /scratch/tmp/j_kuhl19 +user j_kuhl19 +gauge_name /unity +gauge_md5 1ea28326e4090996111a320b8372811d +param_name sfcf_unity_test.in +param_md5 d881e90d41188a33b8b0f1bd0bc53ea5 +param_hash 686af5e712ee2902180f5428af94c6e7 +data_name ./output_10519905/data_c + +[correlator] + +name f_A +quarks lquark lquark +offset 0 +wf 0 +corr_t + +[correlator] + +name f_1 +quarks lquark lquark +offset 0 +wf 0 +wf_2 0 +corr ++3.5119415254545021e+02 +6.7620978057264750e-15 + +[correlator] + +name f_1 +quarks lquark lquark +offset 0 +wf 0 +wf_2 1 +corr ++3.5120703575855339e+02 +6.5026340956203663e-15 + +[correlator] + +name f_1 +quarks lquark lquark +offset 0 +wf 0 +wf_2 2 +corr ++3.5120808902177868e+02 +6.5443496235264788e-15 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 0 +wf 0 +wf_2 0 +corr_t + 1 +6.8367760900851147e+02 +3.0531839956225539e-10 + 2 +6.6131885855823339e+02 +3.9736225045852382e-12 + 3 +6.8367760900810049e+02 -2.5611665964422843e-10 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 0 +wf 0 +wf_2 1 +corr_t + 1 +6.8370283168793060e+02 +3.0532966356282939e-10 + 2 +6.6134325636407561e+02 +3.9737690976212336e-12 + 3 +6.8370283168751973e+02 -2.5612610847134760e-10 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 0 +wf 0 +wf_2 2 +corr_t + 1 +6.8370484437212463e+02 +3.0533056232915147e-10 + 2 +6.6134520322615822e+02 +3.9737807346122766e-12 + 3 +6.8370484437171353e+02 -2.5612686251836130e-10 + +[correlator] + +name f_A +quarks lquark lquark +offset 0 +wf 1 +corr_t + 1 +6.5551520722862705e+01 +2.0963356863957609e-13 + 2 +1.0542820240851569e+00 +2.3989756974599379e-15 + 3 -4.1024441815729936e+01 -5.7107484666182308e-15 + +[correlator] + +name f_1 +quarks lquark lquark +offset 0 +wf 1 +wf_2 0 +corr ++3.5120703575855515e+02 +6.9706500417651470e-15 + +[correlator] + +name f_1 +quarks lquark lquark +offset 0 +wf 1 +wf_2 1 +corr ++3.5122001235609065e+02 +6.9516150897757419e-15 + +[correlator] + +name f_1 +quarks lquark lquark +offset 0 +wf 1 +wf_2 2 +corr ++3.5122104108046199e+02 +6.9232860455434941e-15 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 0 +wf 1 +wf_2 0 +corr_t + 1 +6.8370283168792889e+02 +3.0532890521977295e-10 + 2 +6.6134325636407402e+02 +3.9730355551484655e-12 + 3 +6.8370283168751791e+02 -2.5612686681440218e-10 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 0 +wf 1 +wf_2 1 +corr_t + 1 +6.8372805529787934e+02 +3.0534016954586185e-10 + 2 +6.6136765507001564e+02 +3.9731820664935325e-12 + 3 +6.8372805529746825e+02 -2.5613631608015786e-10 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 0 +wf 1 +wf_2 2 +corr_t + 1 +6.8373006805632656e+02 +3.0534106842445933e-10 + 2 +6.6136960200392332e+02 +3.9731937804440792e-12 + 3 +6.8373006805591558e+02 -2.5613707007587266e-10 + +[correlator] + +name f_A +quarks lquark lquark +offset 0 +wf 2 +corr_t + 1 +6.5529951269442847e+01 -6.6512260271334321e-14 + 2 +1.0516822345055969e+00 -2.2935262162529075e-15 + 3 -4.1025142768037746e+01 +3.7566377680004518e-16 + +[correlator] + +name f_1 +quarks lquark lquark +offset 0 +wf 2 +wf_2 0 +corr ++3.5120808902177447e+02 +1.0849949614595719e-14 + +[correlator] + +name f_1 +quarks lquark lquark +offset 0 +wf 2 +wf_2 1 +corr ++3.5122104108046182e+02 +1.0866063643253473e-14 + +[correlator] + +name f_1 +quarks lquark lquark +offset 0 +wf 2 +wf_2 2 +corr ++3.5122207631098047e+02 +1.0827277318679030e-14 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 0 +wf 2 +wf_2 0 +corr_t + 1 +6.8370484437212156e+02 +3.0532220084664646e-10 + 2 +6.6134520322615526e+02 +3.9656927030717790e-12 + 3 +6.8370484437171069e+02 -2.5613522400086377e-10 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 0 +wf 2 +wf_2 1 +corr_t + 1 +6.8373006805632599e+02 +3.0533346499198999e-10 + 2 +6.6136960200392275e+02 +3.9658390079382195e-12 + 3 +6.8373006805591490e+02 -2.5614467350834153e-10 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 0 +wf 2 +wf_2 2 +corr_t + 1 +6.8373208082069789e+02 +3.0533436384459901e-10 + 2 +6.6137154894356127e+02 +3.9658506942251639e-12 + 3 +6.8373208082028680e+02 -2.5614542753491032e-10 + +[correlator] + +name f_A +quarks lquark lquark +offset 1 +wf 0 +corr_t + 1 +6.5471188727965909e+01 -1.6112786177915427e-11 + 2 +1.0447210337411881e+00 -7.0387528705692678e-13 + 3 -4.1025094911167137e+01 +4.6509152745618223e-13 + +[correlator] + +name f_1 +quarks lquark lquark +offset 1 +wf 0 +wf_2 0 +corr ++3.5119415254545038e+02 +3.0143306723935508e-15 + +[correlator] + +name f_1 +quarks lquark lquark +offset 1 +wf 0 +wf_2 1 +corr ++3.5120703575855367e+02 +4.3340379505972648e-15 + +[correlator] + +name f_1 +quarks lquark lquark +offset 1 +wf 0 +wf_2 2 +corr ++3.5120808902177902e+02 +3.9652247575094006e-15 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 1 +wf 0 +wf_2 0 +corr_t + 1 +6.8367760900918211e+02 -9.5149222536505804e-10 + 2 +6.6131885855810310e+02 +3.2960434859595585e-10 + 3 +6.8367760900806934e+02 -2.3744430846347533e-10 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 1 +wf 0 +wf_2 1 +corr_t + 1 +6.8370283168860135e+02 -9.5152732859532533e-10 + 2 +6.6134325636394544e+02 +3.2961650841969937e-10 + 3 +6.8370283168748858e+02 -2.3745306857315358e-10 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 1 +wf 0 +wf_2 2 +corr_t + 1 +6.8370484437279526e+02 -9.5153012965274573e-10 + 2 +6.6134520322602793e+02 +3.2961747879154288e-10 + 3 +6.8370484437168250e+02 -2.3745376753897864e-10 + +[correlator] + +name f_A +quarks lquark lquark +offset 1 +wf 1 +corr_t + 1 +6.5551520722842213e+01 -8.1976426690345305e-13 + 2 +1.0542820240843382e+00 +2.1626370477046812e-13 + 3 -4.1024441815730086e+01 -2.4147931196409923e-14 + +[correlator] + +name f_1 +quarks lquark lquark +offset 1 +wf 1 +wf_2 0 +corr ++3.5120703575855526e+02 -8.2540994138261318e-16 + +[correlator] + +name f_1 +quarks lquark lquark +offset 1 +wf 1 +wf_2 1 +corr ++3.5122001235609082e+02 -9.7121215247039609e-16 + +[correlator] + +name f_1 +quarks lquark lquark +offset 1 +wf 1 +wf_2 2 +corr ++3.5122104108046227e+02 -9.0872484903683497e-16 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 1 +wf 1 +wf_2 0 +corr_t + 1 +6.8370283168859953e+02 -9.5151770701795658e-10 + 2 +6.6134325636394351e+02 +3.2962581533640458e-10 + 3 +6.8370283168748665e+02 -2.3744344699578737e-10 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 1 +wf 1 +wf_2 1 +corr_t + 1 +6.8372805529855032e+02 -9.5155281099707234e-10 + 2 +6.6136765506988547e+02 +3.2963797613709602e-10 + 3 +6.8372805529743755e+02 -2.3745220688244645e-10 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 1 +wf 1 +wf_2 2 +corr_t + 1 +6.8373006805699742e+02 -9.5155561220425917e-10 + 2 +6.6136960200379315e+02 +3.2963894649982994e-10 + 3 +6.8373006805588454e+02 -2.3745290592048597e-10 + +[correlator] + +name f_A +quarks lquark lquark +offset 1 +wf 2 +corr_t + 1 +6.5529951269443117e+01 +7.9192560386479701e-14 + 2 +1.0516822345055870e+00 -1.2443038782429568e-14 + 3 -4.1025142768037739e+01 +5.9315333178954509e-17 + +[correlator] + +name f_1 +quarks lquark lquark +offset 1 +wf 2 +wf_2 0 +corr ++3.5120808902177453e+02 +5.1331372776616026e-15 + +[correlator] + +name f_1 +quarks lquark lquark +offset 1 +wf 2 +wf_2 1 +corr ++3.5122104108046193e+02 +5.0816653044831932e-15 + +[correlator] + +name f_1 +quarks lquark lquark +offset 1 +wf 2 +wf_2 2 +corr ++3.5122207631098064e+02 +5.1165649253001659e-15 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 1 +wf 2 +wf_2 0 +corr_t + 1 +6.8370484437279174e+02 -9.5153224647433932e-10 + 2 +6.6134520322602452e+02 +3.2961543119772646e-10 + 3 +6.8370484437167897e+02 -2.3745588436057620e-10 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 1 +wf 2 +wf_2 1 +corr_t + 1 +6.8373006805699617e+02 -9.5156735103669992e-10 + 2 +6.6136960200379178e+02 +3.2962759157000606e-10 + 3 +6.8373006805588329e+02 -2.3746464475292832e-10 + +[correlator] + +name F_V0 +quarks lquark lquark +offset 1 +wf 2 +wf_2 2 +corr_t + 1 +6.8373208082136819e+02 -9.5157015223999714e-10 + 2 +6.6137154894343041e+02 +3.2962856194733528e-10 + diff --git a/tests/openQCD_in_test.py b/tests/openQCD_in_test.py index 22ac514a..35da5817 100644 --- a/tests/openQCD_in_test.py +++ b/tests/openQCD_in_test.py @@ -1,4 +1,3 @@ -import os import numpy as np import pyerrors as pe import pytest @@ -136,7 +135,7 @@ def test_read_ms5_xsf(): def test_find_files(): - path = './tests//data/openqcd_test/' + path = './tests/data/openqcd_test/' prefix = "ms5_xsf_T24L16" qc = 'dd' @@ -166,10 +165,3 @@ def test_find_files(): with pytest.raises(Exception): pe.input.openQCD._find_files(path, fpre, "ms5_xsf_" + qc, "dat") - -def test_sort_names(): - my_list = ['sfqcd_r1_id5', 'sfqcd_r10_id5', 'sfqcd_r7_id5', 'sfqcd_r2_id5', 'sfqcd_r2_id9', 'sfqcd_r10_id4'] - presorted_list = ['sfqcd_r1_id5', 'sfqcd_r2_id5', 'sfqcd_r2_id9', 'sfqcd_r7_id5', 'sfqcd_r10_id4', 'sfqcd_r10_id5'] - - sorted_list = pe.input.openQCD._sort_names(my_list) - assert (all([sorted_list[i] == presorted_list[i] for i in range(len(sorted_list))])) diff --git a/tests/sfcf_in_test.py b/tests/sfcf_in_test.py index 86ff6cd6..e0327175 100644 --- a/tests/sfcf_in_test.py +++ b/tests/sfcf_in_test.py @@ -4,121 +4,122 @@ import inspect import pyerrors as pe import pyerrors.input.sfcf as sfin import shutil +import pytest current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) parent_dir = os.path.dirname(current_dir) sys.path.insert(0, parent_dir) -def build_test_environment(env_type, cfgs, reps): +def build_test_environment(path, env_type, cfgs, reps): + shutil.copytree("tests/data/sfcf_test/data_"+env_type, (path + "/data_" + env_type)) if env_type == "o": for i in range(2,cfgs+1): - shutil.copytree("tests/data/sfcf_test/data_o/test_r0/cfg1","tests/data/sfcf_test/data_o/test_r0/cfg"+str(i)) + shutil.copytree(path + "/data_o/test_r0/cfg1", path + "/data_o/test_r0/cfg"+str(i)) for i in range(1,reps): - shutil.copytree("tests/data/sfcf_test/data_o/test_r0", "tests/data/sfcf_test/data_o/test_r"+str(i)) + shutil.copytree(path + "/data_o/test_r0", path + "/data_o/test_r"+str(i)) elif env_type == "c": for i in range(2,cfgs+1): - shutil.copy("tests/data/sfcf_test/data_c/data_c_r0/data_c_r0_n1","tests/data/sfcf_test/data_c/data_c_r0/data_c_r0_n"+str(i)) + shutil.copy(path + "/data_c/data_c_r0/data_c_r0_n1", path + "/data_c/data_c_r0/data_c_r0_n"+str(i)) for i in range(1,reps): - os.mkdir("tests/data/sfcf_test/data_c/data_c_r"+str(i)) + os.mkdir(path + "/data_c/data_c_r"+str(i)) for j in range(1,cfgs+1): - shutil.copy("tests/data/sfcf_test/data_c/data_c_r0/data_c_r0_n1","tests/data/sfcf_test/data_c/data_c_r"+str(i)+"/data_c_r"+str(i)+"_n"+str(j)) + shutil.copy(path + "/data_c/data_c_r0/data_c_r0_n1",path + "/data_c/data_c_r"+str(i)+"/data_c_r"+str(i)+"_n"+str(j)) - -def clean_test_environment(env_type, cfgs, reps): - if env_type == "o": - for i in range(1,reps): - shutil.rmtree("tests/data/sfcf_test/data_o/test_r"+str(i)) - for i in range(2,cfgs+1): - shutil.rmtree("tests/data/sfcf_test/data_o/test_r0/cfg"+str(i)) - elif env_type == "c": - for i in range(1,reps): - shutil.rmtree("tests/data/sfcf_test/data_c/data_c_r"+str(i)) - for i in range(2,cfgs+1): - os.remove("tests/data/sfcf_test/data_c/data_c_r0/data_c_r0_n"+str(i)) - - -def test_o_bb(): - build_test_environment("o",5,3) - f_1 = sfin.read_sfcf("tests/data/sfcf_test/data_o", "test", "f_1",quarks="lquark lquark", wf = 0, wf2=0, version = "2.0", corr_type="bb") +def test_o_bb(tmp_path): + build_test_environment(str(tmp_path), "o",5,3) + f_1 = sfin.read_sfcf(str(tmp_path) + "/data_o", "test", "f_1",quarks="lquark lquark", wf = 0, wf2=0, version = "2.0", corr_type="bb") print(f_1) - clean_test_environment("o",5,3) assert len(f_1) == 1 assert f_1[0].value == 351.1941525454502 -def test_o_bi(): - build_test_environment("o",5,3) - f_A = sfin.read_sfcf("tests/data/sfcf_test/data_o", "test", "f_A",quarks="lquark lquark", wf = 0, version = "2.0") +def test_o_bi(tmp_path): + build_test_environment(str(tmp_path), "o",5,3) + f_A = sfin.read_sfcf(str(tmp_path) + "/data_o", "test", "f_A",quarks="lquark lquark", wf = 0, version = "2.0") print(f_A) - clean_test_environment("o",5,3) assert len(f_A) == 3 assert f_A[0].value == 65.4711887279723 assert f_A[1].value == 1.0447210336915187 assert f_A[2].value == -41.025094911185185 -def test_o_bib(): - build_test_environment("o",5,3) - f_V0 = sfin.read_sfcf("tests/data/sfcf_test/data_o", "test", "F_V0",quarks="lquark lquark", wf = 0, wf2 = 0, version = "2.0", corr_type="bib") +def test_o_bib(tmp_path): + build_test_environment(str(tmp_path), "o",5,3) + f_V0 = sfin.read_sfcf(str(tmp_path) + "/data_o", "test", "F_V0",quarks="lquark lquark", wf = 0, wf2 = 0, version = "2.0", corr_type="bib") print(f_V0) - clean_test_environment("o",5,3) assert len(f_V0) == 3 assert f_V0[0] == 683.6776090085115 assert f_V0[1] == 661.3188585582334 assert f_V0[2] == 683.6776090081005 -def test_c_bb(): - build_test_environment("c",5,3) - f_1 = sfin.read_sfcf("tests/data/sfcf_test/data_c", "data_c", "f_1", quarks="lquark lquark", wf = 0, wf2=0, version = "2.0c", corr_type="bb") +def test_c_bb(tmp_path): + build_test_environment(str(tmp_path), "c",5,3) + f_1 = sfin.read_sfcf(str(tmp_path) + "/data_c", "data_c", "f_1", quarks="lquark lquark", wf = 0, wf2=0, version = "2.0c", corr_type="bb") print(f_1) - clean_test_environment("c",5,3) assert len(f_1) == 1 assert f_1[0].value == 351.1941525454502 -def test_c_bi(): - build_test_environment("c",5,3) - f_A = sfin.read_sfcf("tests/data/sfcf_test/data_c", "data_c", "f_A", quarks="lquark lquark", wf = 0, version = "2.0c") +def test_c_bi(tmp_path): + build_test_environment(str(tmp_path), "c",5,3) + f_A = sfin.read_sfcf(str(tmp_path) + "/data_c", "data_c", "f_A", quarks="lquark lquark", wf = 0, version = "2.0c") print(f_A) - clean_test_environment("c",5,3) assert len(f_A) == 3 assert f_A[0].value == 65.4711887279723 assert f_A[1].value == 1.0447210336915187 assert f_A[2].value == -41.025094911185185 -def test_c_bib(): - build_test_environment("c",5,3) - f_V0 = sfin.read_sfcf("tests/data/sfcf_test/data_c", "data_c", "F_V0",quarks="lquark lquark", wf = 0, wf2 = 0, version = "2.0c", corr_type="bib") +def test_c_bib(tmp_path): + build_test_environment(str(tmp_path), "c",5,3) + f_V0 = sfin.read_sfcf(str(tmp_path) + "/data_c", "data_c", "F_V0",quarks="lquark lquark", wf = 0, wf2 = 0, version = "2.0c", corr_type="bib") print(f_V0) - clean_test_environment("c",5,3) assert len(f_V0) == 3 assert f_V0[0] == 683.6776090085115 assert f_V0[1] == 661.3188585582334 assert f_V0[2] == 683.6776090081005 -def test_a_bb(): - build_test_environment("a",5,3) +def test_a_bb(tmp_path): f_1 = sfin.read_sfcf("tests/data/sfcf_test/data_a", "data_a", "f_1", quarks="lquark lquark", wf = 0, wf2=0, version = "2.0a", corr_type="bb") print(f_1) - clean_test_environment("a",5,3) assert len(f_1) == 1 assert f_1[0].value == 351.1941525454502 -def test_a_bi(): - build_test_environment("a",5,3) +def test_a_bi(tmp_path): f_A = sfin.read_sfcf("tests/data/sfcf_test/data_a", "data_a", "f_A", quarks="lquark lquark", wf = 0, version = "2.0a") print(f_A) - clean_test_environment("a",5,3) assert len(f_A) == 3 assert f_A[0].value == 65.4711887279723 assert f_A[1].value == 1.0447210336915187 assert f_A[2].value == -41.02509491118518 -def test_a_bib(): - build_test_environment("a",5,3) +def test_a_bib(tmp_path): f_V0 = sfin.read_sfcf("tests/data/sfcf_test/data_a", "data_a", "F_V0",quarks="lquark lquark", wf = 0, wf2 = 0, version = "2.0a", corr_type="bib") print(f_V0) - clean_test_environment("a",5,3) assert len(f_V0) == 3 assert f_V0[0] == 683.6776090085115 assert f_V0[1] == 661.3188585582334 assert f_V0[2] == 683.6776090081005 + +def test_find_corr(tmp_path): + pattern = 'name ' + "f_A" + '\nquarks ' + "lquark lquark" + '\noffset ' + str(0) + '\nwf ' + str(0) + start_read, T = sfin._find_correlator("tests/data/sfcf_test/data_c/data_c_r0/data_c_r0_n1", "2.0c", pattern, False) + assert start_read == 21 + assert T == 3 + + pattern = 'name ' + "f_X" + '\nquarks ' + "lquark lquark" + '\noffset ' + str(0) + '\nwf ' + str(0) + with pytest.raises(ValueError): + sfin._find_correlator("tests/data/sfcf_test/data_c/data_c_r0/data_c_r0_n1", "2.0c", pattern, False) + + pattern = 'name ' + "f_A" + '\nquarks ' + "lquark lquark" + '\noffset ' + str(0) + '\nwf ' + str(0) + with pytest.raises(ValueError): + sfin._find_correlator("tests/data/sfcf_test/broken_data_c/data_c_r0/data_c_r0_n1", "2.0c", pattern, False) + +def test_read_compact_file(tmp_path): + rep_path = "tests/data/sfcf_test/broken_data_c/data_c_r0/" + config_file = "data_c_r0_n1" + start_read = 469 + T = 3 + b2b = False + name = "F_V0" + im = False + with pytest.raises(Exception): + sfin._read_compact_file(rep_path, config_file, start_read, T, b2b, name, im) diff --git a/tests/utils_in_test.py b/tests/utils_in_test.py new file mode 100644 index 00000000..2b02a4aa --- /dev/null +++ b/tests/utils_in_test.py @@ -0,0 +1,9 @@ +import pyerrors as pe + + +def test_sort_names(): + my_list = ['sfqcd_r1_id5', 'sfqcd_r10_id5', 'sfqcd_r7_id5', 'sfqcd_r2_id5', 'sfqcd_r2_id9', 'sfqcd_r10_id4'] + presorted_list = ['sfqcd_r1_id5', 'sfqcd_r2_id5', 'sfqcd_r2_id9', 'sfqcd_r7_id5', 'sfqcd_r10_id4', 'sfqcd_r10_id5'] + + sorted_list = pe.input.utils.sort_names(my_list) + assert (all([sorted_list[i] == presorted_list[i] for i in range(len(sorted_list))])) \ No newline at end of file