From d7d1cec2645333aa0d4f4b733d6654234ab1bab5 Mon Sep 17 00:00:00 2001 From: fjosw Date: Mon, 25 Mar 2024 14:51:05 +0000 Subject: [PATCH] Documentation updated --- docs/pyerrors/input/sfcf.html | 1330 +++++++++++++++++---------------- 1 file changed, 667 insertions(+), 663 deletions(-) diff --git a/docs/pyerrors/input/sfcf.html b/docs/pyerrors/input/sfcf.html index 5aeb91c6..ac65fd41 100644 --- a/docs/pyerrors/input/sfcf.html +++ b/docs/pyerrors/input/sfcf.html @@ -268,477 +268,479 @@ 184 185 else: 186 replica = len([file.split(".")[-1] for file in ls]) // len(set([file.split(".")[-1] for file in ls])) -187 if not silent: -188 print('Read', part, 'part of', name_list, 'from', prefix[:-1], ',', replica, 'replica') -189 -190 if 'names' in kwargs: -191 new_names = kwargs.get('names') -192 if len(new_names) != len(set(new_names)): -193 raise Exception("names are not unique!") -194 if len(new_names) != replica: -195 raise Exception('names should have the length', replica) -196 -197 else: -198 ens_name = kwargs.get("ens_name") -199 if not appended: -200 new_names = _get_rep_names(ls, ens_name) -201 else: -202 new_names = _get_appended_rep_names(ls, prefix, name_list[0], ens_name) -203 new_names = sort_names(new_names) -204 -205 idl = [] +187 if replica == 0: +188 raise Exception('No replica found in directory') +189 if not silent: +190 print('Read', part, 'part of', name_list, 'from', prefix[:-1], ',', replica, 'replica') +191 +192 if 'names' in kwargs: +193 new_names = kwargs.get('names') +194 if len(new_names) != len(set(new_names)): +195 raise Exception("names are not unique!") +196 if len(new_names) != replica: +197 raise Exception('names should have the length', replica) +198 +199 else: +200 ens_name = kwargs.get("ens_name") +201 if not appended: +202 new_names = _get_rep_names(ls, ens_name) +203 else: +204 new_names = _get_appended_rep_names(ls, prefix, name_list[0], ens_name) +205 new_names = sort_names(new_names) 206 -207 noffset_list = [str(x) for x in noffset_list] -208 wf_list = [str(x) for x in wf_list] -209 wf2_list = [str(x) for x in wf2_list] -210 -211 # setup dict structures -212 intern = {} -213 for name, corr_type in zip(name_list, corr_type_list): -214 intern[name] = {} -215 b2b, single = _extract_corr_type(corr_type) -216 intern[name]["b2b"] = b2b -217 intern[name]["single"] = single -218 intern[name]["spec"] = {} -219 for quarks in quarks_list: -220 intern[name]["spec"][quarks] = {} -221 for off in noffset_list: -222 intern[name]["spec"][quarks][off] = {} -223 for w in wf_list: -224 intern[name]["spec"][quarks][off][w] = {} -225 for w2 in wf2_list: -226 intern[name]["spec"][quarks][off][w][w2] = {} -227 intern[name]["spec"][quarks][off][w][w2]["pattern"] = _make_pattern(version, name, off, w, w2, intern[name]['b2b'], quarks) -228 -229 internal_ret_dict = {} -230 needed_keys = _lists2key(name_list, quarks_list, noffset_list, wf_list, wf2_list) -231 for key in needed_keys: -232 internal_ret_dict[key] = [] -233 -234 if not appended: -235 for i, item in enumerate(ls): -236 rep_path = path + '/' + item -237 if "files" in kwargs: -238 files = kwargs.get("files") -239 if isinstance(files, list): -240 if all(isinstance(f, list) for f in files): -241 files = files[i] -242 elif all(isinstance(f, str) for f in files): -243 files = files -244 else: -245 raise TypeError("files has to be of type list[list[str]] or list[str]!") -246 else: -247 raise TypeError("files has to be of type list[list[str]] or list[str]!") -248 -249 else: -250 files = [] -251 sub_ls = _find_files(rep_path, prefix, compact, files) -252 rep_idl = [] -253 no_cfg = len(sub_ls) -254 for cfg in sub_ls: -255 try: -256 if compact: -257 rep_idl.append(int(cfg.split(cfg_separator)[-1])) -258 else: -259 rep_idl.append(int(cfg[3:])) -260 except Exception: -261 raise Exception("Couldn't parse idl from directory, problem with file " + cfg) -262 rep_idl.sort() -263 # maybe there is a better way to print the idls -264 if not silent: -265 print(item, ':', no_cfg, ' configurations') -266 idl.append(rep_idl) -267 # here we have found all the files we need to look into. -268 if i == 0: -269 if version != "0.0" and compact: -270 file = path + '/' + item + '/' + sub_ls[0] -271 for name in name_list: -272 if version == "0.0" or not compact: -273 file = path + '/' + item + '/' + sub_ls[0] + '/' + name -274 for key in _lists2key(quarks_list, noffset_list, wf_list, wf2_list): -275 specs = _key2specs(key) -276 quarks = specs[0] -277 off = specs[1] -278 w = specs[2] -279 w2 = specs[3] -280 # here, we want to find the place within the file, -281 # where the correlator we need is stored. -282 # to do so, the pattern needed is put together -283 # from the input values -284 start_read, T = _find_correlator(file, version, intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["pattern"], intern[name]['b2b'], silent=silent) -285 intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["start"] = start_read -286 intern[name]["T"] = T -287 # preparing the datastructure -288 # the correlators get parsed into... -289 deltas = [] -290 for j in range(intern[name]["T"]): -291 deltas.append([]) -292 internal_ret_dict[sep.join([name, key])] = deltas -293 -294 if compact: -295 rep_deltas = _read_compact_rep(path, item, sub_ls, intern, needed_keys, im) -296 for key in needed_keys: -297 name = _key2specs(key)[0] -298 for t in range(intern[name]["T"]): -299 internal_ret_dict[key][t].append(rep_deltas[key][t]) -300 else: -301 for key in needed_keys: -302 rep_data = [] -303 name = _key2specs(key)[0] -304 for subitem in sub_ls: -305 cfg_path = path + '/' + item + '/' + subitem -306 file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im) -307 rep_data.append(file_data) -308 print(rep_data) -309 for t in range(intern[name]["T"]): -310 internal_ret_dict[key][t].append([]) -311 for cfg in range(no_cfg): -312 internal_ret_dict[key][t][i].append(rep_data[cfg][key][t]) -313 else: -314 for key in needed_keys: -315 specs = _key2specs(key) -316 name = specs[0] -317 quarks = specs[1] -318 off = specs[2] -319 w = specs[3] -320 w2 = specs[4] -321 if "files" in kwargs: -322 if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")): -323 name_ls = kwargs.get("files") -324 else: -325 raise TypeError("In append mode, files has to be of type list[str]!") -326 else: -327 name_ls = ls -328 for exc in name_ls: -329 if not fnmatch.fnmatch(exc, prefix + '*.' + name): -330 name_ls = list(set(name_ls) - set([exc])) -331 name_ls = sort_names(name_ls) -332 pattern = intern[name]['spec'][quarks][off][w][w2]['pattern'] -333 deltas = [] -334 for rep, file in enumerate(name_ls): -335 rep_idl = [] -336 filename = path + '/' + file -337 T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single']) -338 if rep == 0: -339 intern[name]['T'] = T -340 for t in range(intern[name]['T']): -341 deltas.append([]) -342 for t in range(intern[name]['T']): -343 deltas[t].append(rep_data[t]) -344 internal_ret_dict[key] = deltas -345 if name == name_list[0]: -346 idl.append(rep_idl) -347 -348 if kwargs.get("check_configs") is True: -349 if not silent: -350 print("Checking for missing configs...") -351 che = kwargs.get("check_configs") -352 if not (len(che) == len(idl)): -353 raise Exception("check_configs has to be the same length as replica!") -354 for r in range(len(idl)): -355 if not silent: -356 print("checking " + new_names[r]) -357 check_idl(idl[r], che[r]) -358 if not silent: -359 print("Done") -360 -361 result_dict = {} -362 if keyed_out: -363 for key in needed_keys: -364 result = [] -365 for t in range(intern[name]["T"]): -366 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) -367 result_dict[key] = result -368 else: -369 for name in name_list: -370 result_dict[name] = {} -371 for quarks in quarks_list: -372 result_dict[name][quarks] = {} -373 for off in noffset_list: -374 result_dict[name][quarks][off] = {} -375 for w in wf_list: -376 result_dict[name][quarks][off][w] = {} -377 for w2 in wf2_list: -378 key = _specs2key(name, quarks, off, w, w2) -379 result = [] -380 for t in range(intern[name]["T"]): -381 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) -382 result_dict[name][quarks][str(off)][str(w)][str(w2)] = result -383 return result_dict -384 -385 -386def _lists2key(*lists): -387 keys = [] -388 for tup in itertools.product(*lists): -389 keys.append(sep.join(tup)) -390 return keys -391 -392 -393def _key2specs(key): -394 return key.split(sep) -395 -396 -397def _specs2key(*specs): -398 return sep.join(specs) -399 -400 -401def _read_o_file(cfg_path, name, needed_keys, intern, version, im): -402 return_vals = {} -403 for key in needed_keys: -404 file = cfg_path + '/' + name -405 specs = _key2specs(key) -406 if specs[0] == name: -407 with open(file) as fp: -408 lines = fp.readlines() -409 quarks = specs[1] -410 off = specs[2] -411 w = specs[3] -412 w2 = specs[4] -413 T = intern[name]["T"] -414 start_read = intern[name]["spec"][quarks][off][w][w2]["start"] -415 deltas = [] -416 for line in lines[start_read:start_read + T]: -417 floats = list(map(float, line.split())) -418 if version == "0.0": -419 deltas.append(floats[im - intern[name]["single"]]) -420 else: -421 deltas.append(floats[1 + im - intern[name]["single"]]) -422 return_vals[key] = deltas -423 return return_vals -424 -425 -426def _extract_corr_type(corr_type): -427 if corr_type == 'bb': -428 b2b = True -429 single = True -430 elif corr_type == 'bib': -431 b2b = True -432 single = False -433 else: -434 b2b = False -435 single = False -436 return b2b, single -437 -438 -439def _find_files(rep_path, prefix, compact, files=[]): -440 sub_ls = [] -441 if not files == []: -442 files.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) -443 else: -444 for (dirpath, dirnames, filenames) in os.walk(rep_path): -445 if compact: -446 sub_ls.extend(filenames) -447 else: -448 sub_ls.extend(dirnames) -449 break -450 if compact: -451 for exc in sub_ls: -452 if not fnmatch.fnmatch(exc, prefix + '*'): -453 sub_ls = list(set(sub_ls) - set([exc])) -454 sub_ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) -455 else: -456 for exc in sub_ls: -457 if not fnmatch.fnmatch(exc, 'cfg*'): -458 sub_ls = list(set(sub_ls) - set([exc])) -459 sub_ls.sort(key=lambda x: int(x[3:])) -460 files = sub_ls -461 if len(files) == 0: -462 raise FileNotFoundError("Did not find files in", rep_path, "with prefix", prefix, "and the given structure.") -463 return files -464 -465 -466def _make_pattern(version, name, noffset, wf, wf2, b2b, quarks): -467 if version == "0.0": -468 pattern = "# " + name + " : offset " + str(noffset) + ", wf " + str(wf) -469 if b2b: -470 pattern += ", wf_2 " + str(wf2) -471 qs = quarks.split(" ") -472 pattern += " : " + qs[0] + " - " + qs[1] -473 else: -474 pattern = 'name ' + name + '\nquarks ' + quarks + '\noffset ' + str(noffset) + '\nwf ' + str(wf) -475 if b2b: -476 pattern += '\nwf_2 ' + str(wf2) -477 return pattern -478 -479 -480def _find_correlator(file_name, version, pattern, b2b, silent=False): -481 T = 0 -482 -483 with open(file_name, "r") as my_file: +207 idl = [] +208 +209 noffset_list = [str(x) for x in noffset_list] +210 wf_list = [str(x) for x in wf_list] +211 wf2_list = [str(x) for x in wf2_list] +212 +213 # setup dict structures +214 intern = {} +215 for name, corr_type in zip(name_list, corr_type_list): +216 intern[name] = {} +217 b2b, single = _extract_corr_type(corr_type) +218 intern[name]["b2b"] = b2b +219 intern[name]["single"] = single +220 intern[name]["spec"] = {} +221 for quarks in quarks_list: +222 intern[name]["spec"][quarks] = {} +223 for off in noffset_list: +224 intern[name]["spec"][quarks][off] = {} +225 for w in wf_list: +226 intern[name]["spec"][quarks][off][w] = {} +227 for w2 in wf2_list: +228 intern[name]["spec"][quarks][off][w][w2] = {} +229 intern[name]["spec"][quarks][off][w][w2]["pattern"] = _make_pattern(version, name, off, w, w2, intern[name]['b2b'], quarks) +230 +231 internal_ret_dict = {} +232 needed_keys = _lists2key(name_list, quarks_list, noffset_list, wf_list, wf2_list) +233 for key in needed_keys: +234 internal_ret_dict[key] = [] +235 +236 if not appended: +237 for i, item in enumerate(ls): +238 rep_path = path + '/' + item +239 if "files" in kwargs: +240 files = kwargs.get("files") +241 if isinstance(files, list): +242 if all(isinstance(f, list) for f in files): +243 files = files[i] +244 elif all(isinstance(f, str) for f in files): +245 files = files +246 else: +247 raise TypeError("files has to be of type list[list[str]] or list[str]!") +248 else: +249 raise TypeError("files has to be of type list[list[str]] or list[str]!") +250 +251 else: +252 files = [] +253 sub_ls = _find_files(rep_path, prefix, compact, files) +254 rep_idl = [] +255 no_cfg = len(sub_ls) +256 for cfg in sub_ls: +257 try: +258 if compact: +259 rep_idl.append(int(cfg.split(cfg_separator)[-1])) +260 else: +261 rep_idl.append(int(cfg[3:])) +262 except Exception: +263 raise Exception("Couldn't parse idl from directory, problem with file " + cfg) +264 rep_idl.sort() +265 # maybe there is a better way to print the idls +266 if not silent: +267 print(item, ':', no_cfg, ' configurations') +268 idl.append(rep_idl) +269 # here we have found all the files we need to look into. +270 if i == 0: +271 if version != "0.0" and compact: +272 file = path + '/' + item + '/' + sub_ls[0] +273 for name in name_list: +274 if version == "0.0" or not compact: +275 file = path + '/' + item + '/' + sub_ls[0] + '/' + name +276 for key in _lists2key(quarks_list, noffset_list, wf_list, wf2_list): +277 specs = _key2specs(key) +278 quarks = specs[0] +279 off = specs[1] +280 w = specs[2] +281 w2 = specs[3] +282 # here, we want to find the place within the file, +283 # where the correlator we need is stored. +284 # to do so, the pattern needed is put together +285 # from the input values +286 start_read, T = _find_correlator(file, version, intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["pattern"], intern[name]['b2b'], silent=silent) +287 intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["start"] = start_read +288 intern[name]["T"] = T +289 # preparing the datastructure +290 # the correlators get parsed into... +291 deltas = [] +292 for j in range(intern[name]["T"]): +293 deltas.append([]) +294 internal_ret_dict[sep.join([name, key])] = deltas +295 +296 if compact: +297 rep_deltas = _read_compact_rep(path, item, sub_ls, intern, needed_keys, im) +298 for key in needed_keys: +299 name = _key2specs(key)[0] +300 for t in range(intern[name]["T"]): +301 internal_ret_dict[key][t].append(rep_deltas[key][t]) +302 else: +303 for key in needed_keys: +304 rep_data = [] +305 name = _key2specs(key)[0] +306 for subitem in sub_ls: +307 cfg_path = path + '/' + item + '/' + subitem +308 file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im) +309 rep_data.append(file_data) +310 print(rep_data) +311 for t in range(intern[name]["T"]): +312 internal_ret_dict[key][t].append([]) +313 for cfg in range(no_cfg): +314 internal_ret_dict[key][t][i].append(rep_data[cfg][key][t]) +315 else: +316 for key in needed_keys: +317 specs = _key2specs(key) +318 name = specs[0] +319 quarks = specs[1] +320 off = specs[2] +321 w = specs[3] +322 w2 = specs[4] +323 if "files" in kwargs: +324 if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")): +325 name_ls = kwargs.get("files") +326 else: +327 raise TypeError("In append mode, files has to be of type list[str]!") +328 else: +329 name_ls = ls +330 for exc in name_ls: +331 if not fnmatch.fnmatch(exc, prefix + '*.' + name): +332 name_ls = list(set(name_ls) - set([exc])) +333 name_ls = sort_names(name_ls) +334 pattern = intern[name]['spec'][quarks][off][w][w2]['pattern'] +335 deltas = [] +336 for rep, file in enumerate(name_ls): +337 rep_idl = [] +338 filename = path + '/' + file +339 T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single']) +340 if rep == 0: +341 intern[name]['T'] = T +342 for t in range(intern[name]['T']): +343 deltas.append([]) +344 for t in range(intern[name]['T']): +345 deltas[t].append(rep_data[t]) +346 internal_ret_dict[key] = deltas +347 if name == name_list[0]: +348 idl.append(rep_idl) +349 +350 if kwargs.get("check_configs") is True: +351 if not silent: +352 print("Checking for missing configs...") +353 che = kwargs.get("check_configs") +354 if not (len(che) == len(idl)): +355 raise Exception("check_configs has to be the same length as replica!") +356 for r in range(len(idl)): +357 if not silent: +358 print("checking " + new_names[r]) +359 check_idl(idl[r], che[r]) +360 if not silent: +361 print("Done") +362 +363 result_dict = {} +364 if keyed_out: +365 for key in needed_keys: +366 result = [] +367 for t in range(intern[name]["T"]): +368 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) +369 result_dict[key] = result +370 else: +371 for name in name_list: +372 result_dict[name] = {} +373 for quarks in quarks_list: +374 result_dict[name][quarks] = {} +375 for off in noffset_list: +376 result_dict[name][quarks][off] = {} +377 for w in wf_list: +378 result_dict[name][quarks][off][w] = {} +379 for w2 in wf2_list: +380 key = _specs2key(name, quarks, off, w, w2) +381 result = [] +382 for t in range(intern[name]["T"]): +383 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) +384 result_dict[name][quarks][str(off)][str(w)][str(w2)] = result +385 return result_dict +386 +387 +388def _lists2key(*lists): +389 keys = [] +390 for tup in itertools.product(*lists): +391 keys.append(sep.join(tup)) +392 return keys +393 +394 +395def _key2specs(key): +396 return key.split(sep) +397 +398 +399def _specs2key(*specs): +400 return sep.join(specs) +401 +402 +403def _read_o_file(cfg_path, name, needed_keys, intern, version, im): +404 return_vals = {} +405 for key in needed_keys: +406 file = cfg_path + '/' + name +407 specs = _key2specs(key) +408 if specs[0] == name: +409 with open(file) as fp: +410 lines = fp.readlines() +411 quarks = specs[1] +412 off = specs[2] +413 w = specs[3] +414 w2 = specs[4] +415 T = intern[name]["T"] +416 start_read = intern[name]["spec"][quarks][off][w][w2]["start"] +417 deltas = [] +418 for line in lines[start_read:start_read + T]: +419 floats = list(map(float, line.split())) +420 if version == "0.0": +421 deltas.append(floats[im - intern[name]["single"]]) +422 else: +423 deltas.append(floats[1 + im - intern[name]["single"]]) +424 return_vals[key] = deltas +425 return return_vals +426 +427 +428def _extract_corr_type(corr_type): +429 if corr_type == 'bb': +430 b2b = True +431 single = True +432 elif corr_type == 'bib': +433 b2b = True +434 single = False +435 else: +436 b2b = False +437 single = False +438 return b2b, single +439 +440 +441def _find_files(rep_path, prefix, compact, files=[]): +442 sub_ls = [] +443 if not files == []: +444 files.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) +445 else: +446 for (dirpath, dirnames, filenames) in os.walk(rep_path): +447 if compact: +448 sub_ls.extend(filenames) +449 else: +450 sub_ls.extend(dirnames) +451 break +452 if compact: +453 for exc in sub_ls: +454 if not fnmatch.fnmatch(exc, prefix + '*'): +455 sub_ls = list(set(sub_ls) - set([exc])) +456 sub_ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) +457 else: +458 for exc in sub_ls: +459 if not fnmatch.fnmatch(exc, 'cfg*'): +460 sub_ls = list(set(sub_ls) - set([exc])) +461 sub_ls.sort(key=lambda x: int(x[3:])) +462 files = sub_ls +463 if len(files) == 0: +464 raise FileNotFoundError("Did not find files in", rep_path, "with prefix", prefix, "and the given structure.") +465 return files +466 +467 +468def _make_pattern(version, name, noffset, wf, wf2, b2b, quarks): +469 if version == "0.0": +470 pattern = "# " + name + " : offset " + str(noffset) + ", wf " + str(wf) +471 if b2b: +472 pattern += ", wf_2 " + str(wf2) +473 qs = quarks.split(" ") +474 pattern += " : " + qs[0] + " - " + qs[1] +475 else: +476 pattern = 'name ' + name + '\nquarks ' + quarks + '\noffset ' + str(noffset) + '\nwf ' + str(wf) +477 if b2b: +478 pattern += '\nwf_2 ' + str(wf2) +479 return pattern +480 +481 +482def _find_correlator(file_name, version, pattern, b2b, silent=False): +483 T = 0 484 -485 content = my_file.read() -486 match = re.search(pattern, content) -487 if match: -488 if version == "0.0": -489 start_read = content.count('\n', 0, match.start()) + 1 -490 T = content.count('\n', start_read) -491 else: -492 start_read = content.count('\n', 0, match.start()) + 5 + b2b -493 end_match = re.search(r'\n\s*\n', content[match.start():]) -494 T = content[match.start():].count('\n', 0, end_match.start()) - 4 - b2b -495 if not T > 0: -496 raise ValueError("Correlator with pattern\n" + pattern + "\nis empty!") -497 if not silent: -498 print(T, 'entries, starting to read in line', start_read) -499 -500 else: -501 raise ValueError('Correlator with pattern\n' + pattern + '\nnot found.') -502 -503 return start_read, T +485 with open(file_name, "r") as my_file: +486 +487 content = my_file.read() +488 match = re.search(pattern, content) +489 if match: +490 if version == "0.0": +491 start_read = content.count('\n', 0, match.start()) + 1 +492 T = content.count('\n', start_read) +493 else: +494 start_read = content.count('\n', 0, match.start()) + 5 + b2b +495 end_match = re.search(r'\n\s*\n', content[match.start():]) +496 T = content[match.start():].count('\n', 0, end_match.start()) - 4 - b2b +497 if not T > 0: +498 raise ValueError("Correlator with pattern\n" + pattern + "\nis empty!") +499 if not silent: +500 print(T, 'entries, starting to read in line', start_read) +501 +502 else: +503 raise ValueError('Correlator with pattern\n' + pattern + '\nnot found.') 504 -505 -506def _read_compact_file(rep_path, cfg_file, intern, needed_keys, im): -507 return_vals = {} -508 with open(rep_path + cfg_file) as fp: -509 lines = fp.readlines() -510 for key in needed_keys: -511 keys = _key2specs(key) -512 name = keys[0] -513 quarks = keys[1] -514 off = keys[2] -515 w = keys[3] -516 w2 = keys[4] -517 -518 T = intern[name]["T"] -519 start_read = intern[name]["spec"][quarks][off][w][w2]["start"] -520 # check, if the correlator is in fact -521 # printed completely -522 if (start_read + T + 1 > len(lines)): -523 raise Exception("EOF before end of correlator data! Maybe " + rep_path + cfg_file + " is corrupted?") -524 corr_lines = lines[start_read - 6: start_read + T] -525 t_vals = [] -526 -527 if corr_lines[1 - intern[name]["b2b"]].strip() != 'name ' + name: -528 raise Exception('Wrong format in file', cfg_file) -529 -530 for k in range(6, T + 6): -531 floats = list(map(float, corr_lines[k].split())) -532 t_vals.append(floats[-2:][im]) -533 return_vals[key] = t_vals -534 return return_vals -535 -536 -537def _read_compact_rep(path, rep, sub_ls, intern, needed_keys, im): -538 rep_path = path + '/' + rep + '/' -539 no_cfg = len(sub_ls) -540 -541 return_vals = {} -542 for key in needed_keys: -543 name = _key2specs(key)[0] -544 deltas = [] -545 for t in range(intern[name]["T"]): -546 deltas.append(np.zeros(no_cfg)) -547 return_vals[key] = deltas -548 -549 for cfg in range(no_cfg): -550 cfg_file = sub_ls[cfg] -551 cfg_data = _read_compact_file(rep_path, cfg_file, intern, needed_keys, im) -552 for key in needed_keys: -553 name = _key2specs(key)[0] -554 for t in range(intern[name]["T"]): -555 return_vals[key][t][cfg] = cfg_data[key][t] -556 return return_vals -557 -558 -559def _read_chunk(chunk, gauge_line, cfg_sep, start_read, T, corr_line, b2b, pattern, im, single): -560 try: -561 idl = int(chunk[gauge_line].split(cfg_sep)[-1]) -562 except Exception: -563 raise Exception("Couldn't parse idl from directory, problem with chunk around line ", gauge_line) -564 -565 found_pat = "" -566 data = [] -567 for li in chunk[corr_line + 1:corr_line + 6 + b2b]: -568 found_pat += li -569 if re.search(pattern, found_pat): -570 for t, line in enumerate(chunk[start_read:start_read + T]): -571 floats = list(map(float, line.split())) -572 data.append(floats[im + 1 - single]) -573 return idl, data -574 -575 -576def _read_append_rep(filename, pattern, b2b, cfg_separator, im, single): -577 with open(filename, 'r') as fp: -578 content = fp.readlines() -579 data_starts = [] -580 for linenumber, line in enumerate(content): -581 if "[run]" in line: -582 data_starts.append(linenumber) -583 if len(set([data_starts[i] - data_starts[i - 1] for i in range(1, len(data_starts))])) > 1: -584 raise Exception("Irregularities in file structure found, not all runs have the same output length") -585 chunk = content[:data_starts[1]] -586 for linenumber, line in enumerate(chunk): -587 if line.startswith("gauge_name"): -588 gauge_line = linenumber -589 elif line.startswith("[correlator]"): -590 corr_line = linenumber -591 found_pat = "" -592 for li in chunk[corr_line + 1: corr_line + 6 + b2b]: -593 found_pat += li -594 if re.search(pattern, found_pat): -595 start_read = corr_line + 7 + b2b -596 break -597 else: -598 raise ValueError("Did not find pattern\n", pattern, "\nin\n", filename) -599 endline = corr_line + 6 + b2b -600 while not chunk[endline] == "\n": -601 endline += 1 -602 T = endline - start_read -603 -604 # all other chunks should follow the same structure -605 rep_idl = [] -606 rep_data = [] -607 -608 for cnfg in range(len(data_starts)): -609 start = data_starts[cnfg] -610 stop = start + data_starts[1] -611 chunk = content[start:stop] -612 idl, data = _read_chunk(chunk, gauge_line, cfg_separator, start_read, T, corr_line, b2b, pattern, im, single) -613 rep_idl.append(idl) -614 rep_data.append(data) -615 -616 data = [] +505 return start_read, T +506 +507 +508def _read_compact_file(rep_path, cfg_file, intern, needed_keys, im): +509 return_vals = {} +510 with open(rep_path + cfg_file) as fp: +511 lines = fp.readlines() +512 for key in needed_keys: +513 keys = _key2specs(key) +514 name = keys[0] +515 quarks = keys[1] +516 off = keys[2] +517 w = keys[3] +518 w2 = keys[4] +519 +520 T = intern[name]["T"] +521 start_read = intern[name]["spec"][quarks][off][w][w2]["start"] +522 # check, if the correlator is in fact +523 # printed completely +524 if (start_read + T + 1 > len(lines)): +525 raise Exception("EOF before end of correlator data! Maybe " + rep_path + cfg_file + " is corrupted?") +526 corr_lines = lines[start_read - 6: start_read + T] +527 t_vals = [] +528 +529 if corr_lines[1 - intern[name]["b2b"]].strip() != 'name ' + name: +530 raise Exception('Wrong format in file', cfg_file) +531 +532 for k in range(6, T + 6): +533 floats = list(map(float, corr_lines[k].split())) +534 t_vals.append(floats[-2:][im]) +535 return_vals[key] = t_vals +536 return return_vals +537 +538 +539def _read_compact_rep(path, rep, sub_ls, intern, needed_keys, im): +540 rep_path = path + '/' + rep + '/' +541 no_cfg = len(sub_ls) +542 +543 return_vals = {} +544 for key in needed_keys: +545 name = _key2specs(key)[0] +546 deltas = [] +547 for t in range(intern[name]["T"]): +548 deltas.append(np.zeros(no_cfg)) +549 return_vals[key] = deltas +550 +551 for cfg in range(no_cfg): +552 cfg_file = sub_ls[cfg] +553 cfg_data = _read_compact_file(rep_path, cfg_file, intern, needed_keys, im) +554 for key in needed_keys: +555 name = _key2specs(key)[0] +556 for t in range(intern[name]["T"]): +557 return_vals[key][t][cfg] = cfg_data[key][t] +558 return return_vals +559 +560 +561def _read_chunk(chunk, gauge_line, cfg_sep, start_read, T, corr_line, b2b, pattern, im, single): +562 try: +563 idl = int(chunk[gauge_line].split(cfg_sep)[-1]) +564 except Exception: +565 raise Exception("Couldn't parse idl from directory, problem with chunk around line ", gauge_line) +566 +567 found_pat = "" +568 data = [] +569 for li in chunk[corr_line + 1:corr_line + 6 + b2b]: +570 found_pat += li +571 if re.search(pattern, found_pat): +572 for t, line in enumerate(chunk[start_read:start_read + T]): +573 floats = list(map(float, line.split())) +574 data.append(floats[im + 1 - single]) +575 return idl, data +576 +577 +578def _read_append_rep(filename, pattern, b2b, cfg_separator, im, single): +579 with open(filename, 'r') as fp: +580 content = fp.readlines() +581 data_starts = [] +582 for linenumber, line in enumerate(content): +583 if "[run]" in line: +584 data_starts.append(linenumber) +585 if len(set([data_starts[i] - data_starts[i - 1] for i in range(1, len(data_starts))])) > 1: +586 raise Exception("Irregularities in file structure found, not all runs have the same output length") +587 chunk = content[:data_starts[1]] +588 for linenumber, line in enumerate(chunk): +589 if line.startswith("gauge_name"): +590 gauge_line = linenumber +591 elif line.startswith("[correlator]"): +592 corr_line = linenumber +593 found_pat = "" +594 for li in chunk[corr_line + 1: corr_line + 6 + b2b]: +595 found_pat += li +596 if re.search(pattern, found_pat): +597 start_read = corr_line + 7 + b2b +598 break +599 else: +600 raise ValueError("Did not find pattern\n", pattern, "\nin\n", filename) +601 endline = corr_line + 6 + b2b +602 while not chunk[endline] == "\n": +603 endline += 1 +604 T = endline - start_read +605 +606 # all other chunks should follow the same structure +607 rep_idl = [] +608 rep_data = [] +609 +610 for cnfg in range(len(data_starts)): +611 start = data_starts[cnfg] +612 stop = start + data_starts[1] +613 chunk = content[start:stop] +614 idl, data = _read_chunk(chunk, gauge_line, cfg_separator, start_read, T, corr_line, b2b, pattern, im, single) +615 rep_idl.append(idl) +616 rep_data.append(data) 617 -618 for t in range(T): -619 data.append([]) -620 for c in range(len(rep_data)): -621 data[t].append(rep_data[c][t]) -622 return T, rep_idl, data -623 -624 -625def _get_rep_names(ls, ens_name=None): -626 new_names = [] -627 for entry in ls: -628 try: -629 idx = entry.index('r') -630 except Exception: -631 raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") -632 -633 if ens_name: -634 new_names.append('ens_name' + '|' + entry[idx:]) -635 else: -636 new_names.append(entry[:idx] + '|' + entry[idx:]) -637 return new_names -638 -639 -640def _get_appended_rep_names(ls, prefix, name, ens_name=None): -641 new_names = [] -642 for exc in ls: -643 if not fnmatch.fnmatch(exc, prefix + '*.' + name): -644 ls = list(set(ls) - set([exc])) -645 ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) -646 for entry in ls: -647 myentry = entry[:-len(name) - 1] -648 try: -649 idx = myentry.index('r') -650 except Exception: -651 raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") -652 -653 if ens_name: -654 new_names.append('ens_name' + '|' + entry[idx:]) -655 else: -656 new_names.append(myentry[:idx] + '|' + myentry[idx:]) -657 return new_names +618 data = [] +619 +620 for t in range(T): +621 data.append([]) +622 for c in range(len(rep_data)): +623 data[t].append(rep_data[c][t]) +624 return T, rep_idl, data +625 +626 +627def _get_rep_names(ls, ens_name=None): +628 new_names = [] +629 for entry in ls: +630 try: +631 idx = entry.index('r') +632 except Exception: +633 raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") +634 +635 if ens_name: +636 new_names.append('ens_name' + '|' + entry[idx:]) +637 else: +638 new_names.append(entry[:idx] + '|' + entry[idx:]) +639 return new_names +640 +641 +642def _get_appended_rep_names(ls, prefix, name, ens_name=None): +643 new_names = [] +644 for exc in ls: +645 if not fnmatch.fnmatch(exc, prefix + '*.' + name): +646 ls = list(set(ls) - set([exc])) +647 ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) +648 for entry in ls: +649 myentry = entry[:-len(name) - 1] +650 try: +651 idx = myentry.index('r') +652 except Exception: +653 raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") +654 +655 if ens_name: +656 new_names.append('ens_name' + '|' + entry[idx:]) +657 else: +658 new_names.append(myentry[:idx] + '|' + myentry[idx:]) +659 return new_names @@ -1019,203 +1021,205 @@ bb-type correlators have length 1. 185 186 else: 187 replica = len([file.split(".")[-1] for file in ls]) // len(set([file.split(".")[-1] for file in ls])) -188 if not silent: -189 print('Read', part, 'part of', name_list, 'from', prefix[:-1], ',', replica, 'replica') -190 -191 if 'names' in kwargs: -192 new_names = kwargs.get('names') -193 if len(new_names) != len(set(new_names)): -194 raise Exception("names are not unique!") -195 if len(new_names) != replica: -196 raise Exception('names should have the length', replica) -197 -198 else: -199 ens_name = kwargs.get("ens_name") -200 if not appended: -201 new_names = _get_rep_names(ls, ens_name) -202 else: -203 new_names = _get_appended_rep_names(ls, prefix, name_list[0], ens_name) -204 new_names = sort_names(new_names) -205 -206 idl = [] +188 if replica == 0: +189 raise Exception('No replica found in directory') +190 if not silent: +191 print('Read', part, 'part of', name_list, 'from', prefix[:-1], ',', replica, 'replica') +192 +193 if 'names' in kwargs: +194 new_names = kwargs.get('names') +195 if len(new_names) != len(set(new_names)): +196 raise Exception("names are not unique!") +197 if len(new_names) != replica: +198 raise Exception('names should have the length', replica) +199 +200 else: +201 ens_name = kwargs.get("ens_name") +202 if not appended: +203 new_names = _get_rep_names(ls, ens_name) +204 else: +205 new_names = _get_appended_rep_names(ls, prefix, name_list[0], ens_name) +206 new_names = sort_names(new_names) 207 -208 noffset_list = [str(x) for x in noffset_list] -209 wf_list = [str(x) for x in wf_list] -210 wf2_list = [str(x) for x in wf2_list] -211 -212 # setup dict structures -213 intern = {} -214 for name, corr_type in zip(name_list, corr_type_list): -215 intern[name] = {} -216 b2b, single = _extract_corr_type(corr_type) -217 intern[name]["b2b"] = b2b -218 intern[name]["single"] = single -219 intern[name]["spec"] = {} -220 for quarks in quarks_list: -221 intern[name]["spec"][quarks] = {} -222 for off in noffset_list: -223 intern[name]["spec"][quarks][off] = {} -224 for w in wf_list: -225 intern[name]["spec"][quarks][off][w] = {} -226 for w2 in wf2_list: -227 intern[name]["spec"][quarks][off][w][w2] = {} -228 intern[name]["spec"][quarks][off][w][w2]["pattern"] = _make_pattern(version, name, off, w, w2, intern[name]['b2b'], quarks) -229 -230 internal_ret_dict = {} -231 needed_keys = _lists2key(name_list, quarks_list, noffset_list, wf_list, wf2_list) -232 for key in needed_keys: -233 internal_ret_dict[key] = [] -234 -235 if not appended: -236 for i, item in enumerate(ls): -237 rep_path = path + '/' + item -238 if "files" in kwargs: -239 files = kwargs.get("files") -240 if isinstance(files, list): -241 if all(isinstance(f, list) for f in files): -242 files = files[i] -243 elif all(isinstance(f, str) for f in files): -244 files = files -245 else: -246 raise TypeError("files has to be of type list[list[str]] or list[str]!") -247 else: -248 raise TypeError("files has to be of type list[list[str]] or list[str]!") -249 -250 else: -251 files = [] -252 sub_ls = _find_files(rep_path, prefix, compact, files) -253 rep_idl = [] -254 no_cfg = len(sub_ls) -255 for cfg in sub_ls: -256 try: -257 if compact: -258 rep_idl.append(int(cfg.split(cfg_separator)[-1])) -259 else: -260 rep_idl.append(int(cfg[3:])) -261 except Exception: -262 raise Exception("Couldn't parse idl from directory, problem with file " + cfg) -263 rep_idl.sort() -264 # maybe there is a better way to print the idls -265 if not silent: -266 print(item, ':', no_cfg, ' configurations') -267 idl.append(rep_idl) -268 # here we have found all the files we need to look into. -269 if i == 0: -270 if version != "0.0" and compact: -271 file = path + '/' + item + '/' + sub_ls[0] -272 for name in name_list: -273 if version == "0.0" or not compact: -274 file = path + '/' + item + '/' + sub_ls[0] + '/' + name -275 for key in _lists2key(quarks_list, noffset_list, wf_list, wf2_list): -276 specs = _key2specs(key) -277 quarks = specs[0] -278 off = specs[1] -279 w = specs[2] -280 w2 = specs[3] -281 # here, we want to find the place within the file, -282 # where the correlator we need is stored. -283 # to do so, the pattern needed is put together -284 # from the input values -285 start_read, T = _find_correlator(file, version, intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["pattern"], intern[name]['b2b'], silent=silent) -286 intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["start"] = start_read -287 intern[name]["T"] = T -288 # preparing the datastructure -289 # the correlators get parsed into... -290 deltas = [] -291 for j in range(intern[name]["T"]): -292 deltas.append([]) -293 internal_ret_dict[sep.join([name, key])] = deltas -294 -295 if compact: -296 rep_deltas = _read_compact_rep(path, item, sub_ls, intern, needed_keys, im) -297 for key in needed_keys: -298 name = _key2specs(key)[0] -299 for t in range(intern[name]["T"]): -300 internal_ret_dict[key][t].append(rep_deltas[key][t]) -301 else: -302 for key in needed_keys: -303 rep_data = [] -304 name = _key2specs(key)[0] -305 for subitem in sub_ls: -306 cfg_path = path + '/' + item + '/' + subitem -307 file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im) -308 rep_data.append(file_data) -309 print(rep_data) -310 for t in range(intern[name]["T"]): -311 internal_ret_dict[key][t].append([]) -312 for cfg in range(no_cfg): -313 internal_ret_dict[key][t][i].append(rep_data[cfg][key][t]) -314 else: -315 for key in needed_keys: -316 specs = _key2specs(key) -317 name = specs[0] -318 quarks = specs[1] -319 off = specs[2] -320 w = specs[3] -321 w2 = specs[4] -322 if "files" in kwargs: -323 if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")): -324 name_ls = kwargs.get("files") -325 else: -326 raise TypeError("In append mode, files has to be of type list[str]!") -327 else: -328 name_ls = ls -329 for exc in name_ls: -330 if not fnmatch.fnmatch(exc, prefix + '*.' + name): -331 name_ls = list(set(name_ls) - set([exc])) -332 name_ls = sort_names(name_ls) -333 pattern = intern[name]['spec'][quarks][off][w][w2]['pattern'] -334 deltas = [] -335 for rep, file in enumerate(name_ls): -336 rep_idl = [] -337 filename = path + '/' + file -338 T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single']) -339 if rep == 0: -340 intern[name]['T'] = T -341 for t in range(intern[name]['T']): -342 deltas.append([]) -343 for t in range(intern[name]['T']): -344 deltas[t].append(rep_data[t]) -345 internal_ret_dict[key] = deltas -346 if name == name_list[0]: -347 idl.append(rep_idl) -348 -349 if kwargs.get("check_configs") is True: -350 if not silent: -351 print("Checking for missing configs...") -352 che = kwargs.get("check_configs") -353 if not (len(che) == len(idl)): -354 raise Exception("check_configs has to be the same length as replica!") -355 for r in range(len(idl)): -356 if not silent: -357 print("checking " + new_names[r]) -358 check_idl(idl[r], che[r]) -359 if not silent: -360 print("Done") -361 -362 result_dict = {} -363 if keyed_out: -364 for key in needed_keys: -365 result = [] -366 for t in range(intern[name]["T"]): -367 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) -368 result_dict[key] = result -369 else: -370 for name in name_list: -371 result_dict[name] = {} -372 for quarks in quarks_list: -373 result_dict[name][quarks] = {} -374 for off in noffset_list: -375 result_dict[name][quarks][off] = {} -376 for w in wf_list: -377 result_dict[name][quarks][off][w] = {} -378 for w2 in wf2_list: -379 key = _specs2key(name, quarks, off, w, w2) -380 result = [] -381 for t in range(intern[name]["T"]): -382 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) -383 result_dict[name][quarks][str(off)][str(w)][str(w2)] = result -384 return result_dict +208 idl = [] +209 +210 noffset_list = [str(x) for x in noffset_list] +211 wf_list = [str(x) for x in wf_list] +212 wf2_list = [str(x) for x in wf2_list] +213 +214 # setup dict structures +215 intern = {} +216 for name, corr_type in zip(name_list, corr_type_list): +217 intern[name] = {} +218 b2b, single = _extract_corr_type(corr_type) +219 intern[name]["b2b"] = b2b +220 intern[name]["single"] = single +221 intern[name]["spec"] = {} +222 for quarks in quarks_list: +223 intern[name]["spec"][quarks] = {} +224 for off in noffset_list: +225 intern[name]["spec"][quarks][off] = {} +226 for w in wf_list: +227 intern[name]["spec"][quarks][off][w] = {} +228 for w2 in wf2_list: +229 intern[name]["spec"][quarks][off][w][w2] = {} +230 intern[name]["spec"][quarks][off][w][w2]["pattern"] = _make_pattern(version, name, off, w, w2, intern[name]['b2b'], quarks) +231 +232 internal_ret_dict = {} +233 needed_keys = _lists2key(name_list, quarks_list, noffset_list, wf_list, wf2_list) +234 for key in needed_keys: +235 internal_ret_dict[key] = [] +236 +237 if not appended: +238 for i, item in enumerate(ls): +239 rep_path = path + '/' + item +240 if "files" in kwargs: +241 files = kwargs.get("files") +242 if isinstance(files, list): +243 if all(isinstance(f, list) for f in files): +244 files = files[i] +245 elif all(isinstance(f, str) for f in files): +246 files = files +247 else: +248 raise TypeError("files has to be of type list[list[str]] or list[str]!") +249 else: +250 raise TypeError("files has to be of type list[list[str]] or list[str]!") +251 +252 else: +253 files = [] +254 sub_ls = _find_files(rep_path, prefix, compact, files) +255 rep_idl = [] +256 no_cfg = len(sub_ls) +257 for cfg in sub_ls: +258 try: +259 if compact: +260 rep_idl.append(int(cfg.split(cfg_separator)[-1])) +261 else: +262 rep_idl.append(int(cfg[3:])) +263 except Exception: +264 raise Exception("Couldn't parse idl from directory, problem with file " + cfg) +265 rep_idl.sort() +266 # maybe there is a better way to print the idls +267 if not silent: +268 print(item, ':', no_cfg, ' configurations') +269 idl.append(rep_idl) +270 # here we have found all the files we need to look into. +271 if i == 0: +272 if version != "0.0" and compact: +273 file = path + '/' + item + '/' + sub_ls[0] +274 for name in name_list: +275 if version == "0.0" or not compact: +276 file = path + '/' + item + '/' + sub_ls[0] + '/' + name +277 for key in _lists2key(quarks_list, noffset_list, wf_list, wf2_list): +278 specs = _key2specs(key) +279 quarks = specs[0] +280 off = specs[1] +281 w = specs[2] +282 w2 = specs[3] +283 # here, we want to find the place within the file, +284 # where the correlator we need is stored. +285 # to do so, the pattern needed is put together +286 # from the input values +287 start_read, T = _find_correlator(file, version, intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["pattern"], intern[name]['b2b'], silent=silent) +288 intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["start"] = start_read +289 intern[name]["T"] = T +290 # preparing the datastructure +291 # the correlators get parsed into... +292 deltas = [] +293 for j in range(intern[name]["T"]): +294 deltas.append([]) +295 internal_ret_dict[sep.join([name, key])] = deltas +296 +297 if compact: +298 rep_deltas = _read_compact_rep(path, item, sub_ls, intern, needed_keys, im) +299 for key in needed_keys: +300 name = _key2specs(key)[0] +301 for t in range(intern[name]["T"]): +302 internal_ret_dict[key][t].append(rep_deltas[key][t]) +303 else: +304 for key in needed_keys: +305 rep_data = [] +306 name = _key2specs(key)[0] +307 for subitem in sub_ls: +308 cfg_path = path + '/' + item + '/' + subitem +309 file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im) +310 rep_data.append(file_data) +311 print(rep_data) +312 for t in range(intern[name]["T"]): +313 internal_ret_dict[key][t].append([]) +314 for cfg in range(no_cfg): +315 internal_ret_dict[key][t][i].append(rep_data[cfg][key][t]) +316 else: +317 for key in needed_keys: +318 specs = _key2specs(key) +319 name = specs[0] +320 quarks = specs[1] +321 off = specs[2] +322 w = specs[3] +323 w2 = specs[4] +324 if "files" in kwargs: +325 if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")): +326 name_ls = kwargs.get("files") +327 else: +328 raise TypeError("In append mode, files has to be of type list[str]!") +329 else: +330 name_ls = ls +331 for exc in name_ls: +332 if not fnmatch.fnmatch(exc, prefix + '*.' + name): +333 name_ls = list(set(name_ls) - set([exc])) +334 name_ls = sort_names(name_ls) +335 pattern = intern[name]['spec'][quarks][off][w][w2]['pattern'] +336 deltas = [] +337 for rep, file in enumerate(name_ls): +338 rep_idl = [] +339 filename = path + '/' + file +340 T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single']) +341 if rep == 0: +342 intern[name]['T'] = T +343 for t in range(intern[name]['T']): +344 deltas.append([]) +345 for t in range(intern[name]['T']): +346 deltas[t].append(rep_data[t]) +347 internal_ret_dict[key] = deltas +348 if name == name_list[0]: +349 idl.append(rep_idl) +350 +351 if kwargs.get("check_configs") is True: +352 if not silent: +353 print("Checking for missing configs...") +354 che = kwargs.get("check_configs") +355 if not (len(che) == len(idl)): +356 raise Exception("check_configs has to be the same length as replica!") +357 for r in range(len(idl)): +358 if not silent: +359 print("checking " + new_names[r]) +360 check_idl(idl[r], che[r]) +361 if not silent: +362 print("Done") +363 +364 result_dict = {} +365 if keyed_out: +366 for key in needed_keys: +367 result = [] +368 for t in range(intern[name]["T"]): +369 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) +370 result_dict[key] = result +371 else: +372 for name in name_list: +373 result_dict[name] = {} +374 for quarks in quarks_list: +375 result_dict[name][quarks] = {} +376 for off in noffset_list: +377 result_dict[name][quarks][off] = {} +378 for w in wf_list: +379 result_dict[name][quarks][off][w] = {} +380 for w2 in wf2_list: +381 key = _specs2key(name, quarks, off, w, w2) +382 result = [] +383 for t in range(intern[name]["T"]): +384 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) +385 result_dict[name][quarks][str(off)][str(w)][str(w2)] = result +386 return result_dict