pyerrors.input.sfcf
1import os 2import fnmatch 3import re 4import numpy as np # Thinly-wrapped numpy 5from ..obs import Obs 6from .utils import sort_names, check_idl 7import itertools 8 9 10sep = "/" 11 12 13def read_sfcf(path, prefix, name, quarks='.*', corr_type="bi", noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", silent=False, **kwargs): 14 """Read sfcf files from given folder structure. 15 16 Parameters 17 ---------- 18 path : str 19 Path to the sfcf files. 20 prefix : str 21 Prefix of the sfcf files. 22 name : str 23 Name of the correlation function to read. 24 quarks : str 25 Label of the quarks used in the sfcf input file. e.g. "quark quark" 26 for version 0.0 this does NOT need to be given with the typical " - " 27 that is present in the output file, 28 this is done automatically for this version 29 corr_type : str 30 Type of correlation function to read. Can be 31 - 'bi' for boundary-inner 32 - 'bb' for boundary-boundary 33 - 'bib' for boundary-inner-boundary 34 noffset : int 35 Offset of the source (only relevant when wavefunctions are used) 36 wf : int 37 ID of wave function 38 wf2 : int 39 ID of the second wavefunction 40 (only relevant for boundary-to-boundary correlation functions) 41 im : bool 42 if True, read imaginary instead of real part 43 of the correlation function. 44 names : list 45 Alternative labeling for replicas/ensembles. 46 Has to have the appropriate length 47 ens_name : str 48 replaces the name of the ensemble 49 version: str 50 version of SFCF, with which the measurement was done. 51 if the compact output option (-c) was specified, 52 append a "c" to the version (e.g. "1.0c") 53 if the append output option (-a) was specified, 54 append an "a" to the version 55 cfg_separator : str 56 String that separates the ensemble identifier from the configuration number (default 'n'). 57 replica: list 58 list of replica to be read, default is all 59 files: list 60 list of files to be read per replica, default is all. 61 for non-compact output format, hand the folders to be read here. 62 check_configs: list[list[int]] 63 list of list of supposed configs, eg. [range(1,1000)] 64 for one replicum with 1000 configs 65 66 Returns 67 ------- 68 result: list[Obs] 69 list of Observables with length T, observable per timeslice. 70 bb-type correlators have length 1. 71 """ 72 ret = read_sfcf_multi(path, prefix, [name], quarks_list=[quarks], corr_type_list=[corr_type], 73 noffset_list=[noffset], wf_list=[wf], wf2_list=[wf2], version=version, 74 cfg_separator=cfg_separator, silent=silent, **kwargs) 75 return ret[name][quarks][str(noffset)][str(wf)][str(wf2)] 76 77 78def read_sfcf_multi(path, prefix, name_list, quarks_list=['.*'], corr_type_list=['bi'], noffset_list=[0], wf_list=[0], wf2_list=[0], version="1.0c", cfg_separator="n", silent=False, keyed_out=False, **kwargs): 79 """Read sfcf files from given folder structure. 80 81 Parameters 82 ---------- 83 path : str 84 Path to the sfcf files. 85 prefix : str 86 Prefix of the sfcf files. 87 name : str 88 Name of the correlation function to read. 89 quarks_list : list[str] 90 Label of the quarks used in the sfcf input file. e.g. "quark quark" 91 for version 0.0 this does NOT need to be given with the typical " - " 92 that is present in the output file, 93 this is done automatically for this version 94 corr_type_list : list[str] 95 Type of correlation function to read. Can be 96 - 'bi' for boundary-inner 97 - 'bb' for boundary-boundary 98 - 'bib' for boundary-inner-boundary 99 noffset_list : list[int] 100 Offset of the source (only relevant when wavefunctions are used) 101 wf_list : int 102 ID of wave function 103 wf2_list : list[int] 104 ID of the second wavefunction 105 (only relevant for boundary-to-boundary correlation functions) 106 im : bool 107 if True, read imaginary instead of real part 108 of the correlation function. 109 names : list 110 Alternative labeling for replicas/ensembles. 111 Has to have the appropriate length 112 ens_name : str 113 replaces the name of the ensemble 114 version: str 115 version of SFCF, with which the measurement was done. 116 if the compact output option (-c) was specified, 117 append a "c" to the version (e.g. "1.0c") 118 if the append output option (-a) was specified, 119 append an "a" to the version 120 cfg_separator : str 121 String that separates the ensemble identifier from the configuration number (default 'n'). 122 replica: list 123 list of replica to be read, default is all 124 files: list[list[int]] 125 list of files to be read per replica, default is all. 126 for non-compact output format, hand the folders to be read here. 127 check_configs: list[list[int]] 128 list of list of supposed configs, eg. [range(1,1000)] 129 for one replicum with 1000 configs 130 131 Returns 132 ------- 133 result: dict[list[Obs]] 134 dict with one of the following properties: 135 if keyed_out: 136 dict[key] = list[Obs] 137 where key has the form name/quarks/offset/wf/wf2 138 if not keyed_out: 139 dict[name][quarks][offset][wf][wf2] = list[Obs] 140 """ 141 142 if kwargs.get('im'): 143 im = 1 144 part = 'imaginary' 145 else: 146 im = 0 147 part = 'real' 148 149 known_versions = ["0.0", "1.0", "2.0", "1.0c", "2.0c", "1.0a", "2.0a"] 150 151 if version not in known_versions: 152 raise Exception("This version is not known!") 153 if (version[-1] == "c"): 154 appended = False 155 compact = True 156 version = version[:-1] 157 elif (version[-1] == "a"): 158 appended = True 159 compact = False 160 version = version[:-1] 161 else: 162 compact = False 163 appended = False 164 ls = [] 165 if "replica" in kwargs: 166 ls = kwargs.get("replica") 167 else: 168 for (dirpath, dirnames, filenames) in os.walk(path): 169 if not appended: 170 ls.extend(dirnames) 171 else: 172 ls.extend(filenames) 173 break 174 if not ls: 175 raise Exception('Error, directory not found') 176 # Exclude folders with different names 177 for exc in ls: 178 if not fnmatch.fnmatch(exc, prefix + '*'): 179 ls = list(set(ls) - set([exc])) 180 181 if not appended: 182 ls = sort_names(ls) 183 replica = len(ls) 184 185 else: 186 replica = len([file.split(".")[-1] for file in ls]) // len(set([file.split(".")[-1] for file in ls])) 187 if replica == 0: 188 raise Exception('No replica found in directory') 189 if not silent: 190 print('Read', part, 'part of', name_list, 'from', prefix[:-1], ',', replica, 'replica') 191 192 if 'names' in kwargs: 193 new_names = kwargs.get('names') 194 if len(new_names) != len(set(new_names)): 195 raise Exception("names are not unique!") 196 if len(new_names) != replica: 197 raise Exception('names should have the length', replica) 198 199 else: 200 ens_name = kwargs.get("ens_name") 201 if not appended: 202 new_names = _get_rep_names(ls, ens_name) 203 else: 204 new_names = _get_appended_rep_names(ls, prefix, name_list[0], ens_name) 205 new_names = sort_names(new_names) 206 207 idl = [] 208 209 noffset_list = [str(x) for x in noffset_list] 210 wf_list = [str(x) for x in wf_list] 211 wf2_list = [str(x) for x in wf2_list] 212 213 # setup dict structures 214 intern = {} 215 for name, corr_type in zip(name_list, corr_type_list): 216 intern[name] = {} 217 b2b, single = _extract_corr_type(corr_type) 218 intern[name]["b2b"] = b2b 219 intern[name]["single"] = single 220 intern[name]["spec"] = {} 221 for quarks in quarks_list: 222 intern[name]["spec"][quarks] = {} 223 for off in noffset_list: 224 intern[name]["spec"][quarks][off] = {} 225 for w in wf_list: 226 intern[name]["spec"][quarks][off][w] = {} 227 if b2b: 228 for w2 in wf2_list: 229 intern[name]["spec"][quarks][off][w][w2] = {} 230 intern[name]["spec"][quarks][off][w][w2]["pattern"] = _make_pattern(version, name, off, w, w2, intern[name]['b2b'], quarks) 231 else: 232 intern[name]["spec"][quarks][off][w]["0"] = {} 233 intern[name]["spec"][quarks][off][w]["0"]["pattern"] = _make_pattern(version, name, off, w, 0, intern[name]['b2b'], quarks) 234 235 internal_ret_dict = {} 236 needed_keys = [] 237 for name, corr_type in zip(name_list, corr_type_list): 238 b2b, single = _extract_corr_type(corr_type) 239 if b2b: 240 needed_keys.extend(_lists2key([name], quarks_list, noffset_list, wf_list, wf2_list)) 241 else: 242 needed_keys.extend(_lists2key([name], quarks_list, noffset_list, wf_list, ["0"])) 243 244 for key in needed_keys: 245 internal_ret_dict[key] = [] 246 247 if not appended: 248 for i, item in enumerate(ls): 249 rep_path = path + '/' + item 250 if "files" in kwargs: 251 files = kwargs.get("files") 252 if isinstance(files, list): 253 if all(isinstance(f, list) for f in files): 254 files = files[i] 255 elif all(isinstance(f, str) for f in files): 256 files = files 257 else: 258 raise TypeError("files has to be of type list[list[str]] or list[str]!") 259 else: 260 raise TypeError("files has to be of type list[list[str]] or list[str]!") 261 262 else: 263 files = [] 264 sub_ls = _find_files(rep_path, prefix, compact, files) 265 rep_idl = [] 266 no_cfg = len(sub_ls) 267 for cfg in sub_ls: 268 try: 269 if compact: 270 rep_idl.append(int(cfg.split(cfg_separator)[-1])) 271 else: 272 rep_idl.append(int(cfg[3:])) 273 except Exception: 274 raise Exception("Couldn't parse idl from directory, problem with file " + cfg) 275 rep_idl.sort() 276 # maybe there is a better way to print the idls 277 if not silent: 278 print(item, ':', no_cfg, ' configurations') 279 idl.append(rep_idl) 280 # here we have found all the files we need to look into. 281 if i == 0: 282 if version != "0.0" and compact: 283 file = path + '/' + item + '/' + sub_ls[0] 284 for name_index, name in enumerate(name_list): 285 if version == "0.0" or not compact: 286 file = path + '/' + item + '/' + sub_ls[0] + '/' + name 287 if corr_type_list[name_index] == 'bi': 288 name_keys = _lists2key(quarks_list, noffset_list, wf_list, ["0"]) 289 else: 290 name_keys = _lists2key(quarks_list, noffset_list, wf_list, wf2_list) 291 for key in name_keys: 292 specs = _key2specs(key) 293 quarks = specs[0] 294 off = specs[1] 295 w = specs[2] 296 w2 = specs[3] 297 # here, we want to find the place within the file, 298 # where the correlator we need is stored. 299 # to do so, the pattern needed is put together 300 # from the input values 301 start_read, T = _find_correlator(file, version, intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["pattern"], intern[name]['b2b'], silent=silent) 302 intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["start"] = start_read 303 intern[name]["T"] = T 304 # preparing the datastructure 305 # the correlators get parsed into... 306 deltas = [] 307 for j in range(intern[name]["T"]): 308 deltas.append([]) 309 internal_ret_dict[sep.join([name, key])] = deltas 310 311 if compact: 312 rep_deltas = _read_compact_rep(path, item, sub_ls, intern, needed_keys, im) 313 for key in needed_keys: 314 name = _key2specs(key)[0] 315 for t in range(intern[name]["T"]): 316 internal_ret_dict[key][t].append(rep_deltas[key][t]) 317 else: 318 for key in needed_keys: 319 rep_data = [] 320 name = _key2specs(key)[0] 321 for subitem in sub_ls: 322 cfg_path = path + '/' + item + '/' + subitem 323 file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im) 324 rep_data.append(file_data) 325 for t in range(intern[name]["T"]): 326 internal_ret_dict[key][t].append([]) 327 for cfg in range(no_cfg): 328 internal_ret_dict[key][t][i].append(rep_data[cfg][key][t]) 329 else: 330 for key in needed_keys: 331 specs = _key2specs(key) 332 name = specs[0] 333 quarks = specs[1] 334 off = specs[2] 335 w = specs[3] 336 w2 = specs[4] 337 if "files" in kwargs: 338 if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")): 339 name_ls = kwargs.get("files") 340 else: 341 raise TypeError("In append mode, files has to be of type list[str]!") 342 else: 343 name_ls = ls 344 for exc in name_ls: 345 if not fnmatch.fnmatch(exc, prefix + '*.' + name): 346 name_ls = list(set(name_ls) - set([exc])) 347 name_ls = sort_names(name_ls) 348 pattern = intern[name]['spec'][quarks][off][w][w2]['pattern'] 349 deltas = [] 350 for rep, file in enumerate(name_ls): 351 rep_idl = [] 352 filename = path + '/' + file 353 T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single']) 354 if rep == 0: 355 intern[name]['T'] = T 356 for t in range(intern[name]['T']): 357 deltas.append([]) 358 for t in range(intern[name]['T']): 359 deltas[t].append(rep_data[t]) 360 internal_ret_dict[key] = deltas 361 if name == name_list[0]: 362 idl.append(rep_idl) 363 364 if kwargs.get("check_configs") is True: 365 if not silent: 366 print("Checking for missing configs...") 367 che = kwargs.get("check_configs") 368 if not (len(che) == len(idl)): 369 raise Exception("check_configs has to be the same length as replica!") 370 for r in range(len(idl)): 371 if not silent: 372 print("checking " + new_names[r]) 373 check_idl(idl[r], che[r]) 374 if not silent: 375 print("Done") 376 377 result_dict = {} 378 if keyed_out: 379 for key in needed_keys: 380 name = _key2specs(key)[0] 381 result = [] 382 for t in range(intern[name]["T"]): 383 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) 384 result_dict[key] = result 385 else: 386 for name, corr_type in zip(name_list, corr_type_list): 387 result_dict[name] = {} 388 for quarks in quarks_list: 389 result_dict[name][quarks] = {} 390 for off in noffset_list: 391 result_dict[name][quarks][off] = {} 392 for w in wf_list: 393 result_dict[name][quarks][off][w] = {} 394 if corr_type != 'bi': 395 for w2 in wf2_list: 396 key = _specs2key(name, quarks, off, w, w2) 397 result = [] 398 for t in range(intern[name]["T"]): 399 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) 400 result_dict[name][quarks][str(off)][str(w)][str(w2)] = result 401 else: 402 key = _specs2key(name, quarks, off, w, "0") 403 result = [] 404 for t in range(intern[name]["T"]): 405 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) 406 result_dict[name][quarks][str(off)][str(w)][str(0)] = result 407 return result_dict 408 409 410def _lists2key(*lists): 411 keys = [] 412 for tup in itertools.product(*lists): 413 keys.append(sep.join(tup)) 414 return keys 415 416 417def _key2specs(key): 418 return key.split(sep) 419 420 421def _specs2key(*specs): 422 return sep.join(specs) 423 424 425def _read_o_file(cfg_path, name, needed_keys, intern, version, im): 426 return_vals = {} 427 for key in needed_keys: 428 file = cfg_path + '/' + name 429 specs = _key2specs(key) 430 if specs[0] == name: 431 with open(file) as fp: 432 lines = fp.readlines() 433 quarks = specs[1] 434 off = specs[2] 435 w = specs[3] 436 w2 = specs[4] 437 T = intern[name]["T"] 438 start_read = intern[name]["spec"][quarks][off][w][w2]["start"] 439 deltas = [] 440 for line in lines[start_read:start_read + T]: 441 floats = list(map(float, line.split())) 442 if version == "0.0": 443 deltas.append(floats[im - intern[name]["single"]]) 444 else: 445 deltas.append(floats[1 + im - intern[name]["single"]]) 446 return_vals[key] = deltas 447 return return_vals 448 449 450def _extract_corr_type(corr_type): 451 if corr_type == 'bb': 452 b2b = True 453 single = True 454 elif corr_type == 'bib': 455 b2b = True 456 single = False 457 else: 458 b2b = False 459 single = False 460 return b2b, single 461 462 463def _find_files(rep_path, prefix, compact, files=[]): 464 sub_ls = [] 465 if not files == []: 466 files.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) 467 else: 468 for (dirpath, dirnames, filenames) in os.walk(rep_path): 469 if compact: 470 sub_ls.extend(filenames) 471 else: 472 sub_ls.extend(dirnames) 473 break 474 if compact: 475 for exc in sub_ls: 476 if not fnmatch.fnmatch(exc, prefix + '*'): 477 sub_ls = list(set(sub_ls) - set([exc])) 478 sub_ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) 479 else: 480 for exc in sub_ls: 481 if not fnmatch.fnmatch(exc, 'cfg*'): 482 sub_ls = list(set(sub_ls) - set([exc])) 483 sub_ls.sort(key=lambda x: int(x[3:])) 484 files = sub_ls 485 if len(files) == 0: 486 raise FileNotFoundError("Did not find files in", rep_path, "with prefix", prefix, "and the given structure.") 487 return files 488 489 490def _make_pattern(version, name, noffset, wf, wf2, b2b, quarks): 491 if version == "0.0": 492 pattern = "# " + name + " : offset " + str(noffset) + ", wf " + str(wf) 493 if b2b: 494 pattern += ", wf_2 " + str(wf2) 495 qs = quarks.split(" ") 496 pattern += " : " + qs[0] + " - " + qs[1] 497 else: 498 pattern = 'name ' + name + '\nquarks ' + quarks + '\noffset ' + str(noffset) + '\nwf ' + str(wf) 499 if b2b: 500 pattern += '\nwf_2 ' + str(wf2) 501 return pattern 502 503 504def _find_correlator(file_name, version, pattern, b2b, silent=False): 505 T = 0 506 507 with open(file_name, "r") as my_file: 508 509 content = my_file.read() 510 match = re.search(pattern, content) 511 if match: 512 if version == "0.0": 513 start_read = content.count('\n', 0, match.start()) + 1 514 T = content.count('\n', start_read) 515 else: 516 start_read = content.count('\n', 0, match.start()) + 5 + b2b 517 end_match = re.search(r'\n\s*\n', content[match.start():]) 518 T = content[match.start():].count('\n', 0, end_match.start()) - 4 - b2b 519 if not T > 0: 520 raise ValueError("Correlator with pattern\n" + pattern + "\nis empty!") 521 if not silent: 522 print(T, 'entries, starting to read in line', start_read) 523 524 else: 525 raise ValueError('Correlator with pattern\n' + pattern + '\nnot found.') 526 527 return start_read, T 528 529 530def _read_compact_file(rep_path, cfg_file, intern, needed_keys, im): 531 return_vals = {} 532 with open(rep_path + cfg_file) as fp: 533 lines = fp.readlines() 534 for key in needed_keys: 535 keys = _key2specs(key) 536 name = keys[0] 537 quarks = keys[1] 538 off = keys[2] 539 w = keys[3] 540 w2 = keys[4] 541 542 T = intern[name]["T"] 543 start_read = intern[name]["spec"][quarks][off][w][w2]["start"] 544 # check, if the correlator is in fact 545 # printed completely 546 if (start_read + T + 1 > len(lines)): 547 raise Exception("EOF before end of correlator data! Maybe " + rep_path + cfg_file + " is corrupted?") 548 corr_lines = lines[start_read - 6: start_read + T] 549 t_vals = [] 550 551 if corr_lines[1 - intern[name]["b2b"]].strip() != 'name ' + name: 552 raise Exception('Wrong format in file', cfg_file) 553 554 for k in range(6, T + 6): 555 floats = list(map(float, corr_lines[k].split())) 556 t_vals.append(floats[-2:][im]) 557 return_vals[key] = t_vals 558 return return_vals 559 560 561def _read_compact_rep(path, rep, sub_ls, intern, needed_keys, im): 562 rep_path = path + '/' + rep + '/' 563 no_cfg = len(sub_ls) 564 565 return_vals = {} 566 for key in needed_keys: 567 name = _key2specs(key)[0] 568 deltas = [] 569 for t in range(intern[name]["T"]): 570 deltas.append(np.zeros(no_cfg)) 571 return_vals[key] = deltas 572 573 for cfg in range(no_cfg): 574 cfg_file = sub_ls[cfg] 575 cfg_data = _read_compact_file(rep_path, cfg_file, intern, needed_keys, im) 576 for key in needed_keys: 577 name = _key2specs(key)[0] 578 for t in range(intern[name]["T"]): 579 return_vals[key][t][cfg] = cfg_data[key][t] 580 return return_vals 581 582 583def _read_chunk(chunk, gauge_line, cfg_sep, start_read, T, corr_line, b2b, pattern, im, single): 584 try: 585 idl = int(chunk[gauge_line].split(cfg_sep)[-1]) 586 except Exception: 587 raise Exception("Couldn't parse idl from directory, problem with chunk around line ", gauge_line) 588 589 found_pat = "" 590 data = [] 591 for li in chunk[corr_line + 1:corr_line + 6 + b2b]: 592 found_pat += li 593 if re.search(pattern, found_pat): 594 for t, line in enumerate(chunk[start_read:start_read + T]): 595 floats = list(map(float, line.split())) 596 data.append(floats[im + 1 - single]) 597 return idl, data 598 599 600def _read_append_rep(filename, pattern, b2b, cfg_separator, im, single): 601 with open(filename, 'r') as fp: 602 content = fp.readlines() 603 data_starts = [] 604 for linenumber, line in enumerate(content): 605 if "[run]" in line: 606 data_starts.append(linenumber) 607 if len(set([data_starts[i] - data_starts[i - 1] for i in range(1, len(data_starts))])) > 1: 608 raise Exception("Irregularities in file structure found, not all runs have the same output length") 609 chunk = content[:data_starts[1]] 610 for linenumber, line in enumerate(chunk): 611 if line.startswith("gauge_name"): 612 gauge_line = linenumber 613 elif line.startswith("[correlator]"): 614 corr_line = linenumber 615 found_pat = "" 616 for li in chunk[corr_line + 1: corr_line + 6 + b2b]: 617 found_pat += li 618 if re.search(pattern, found_pat): 619 start_read = corr_line + 7 + b2b 620 break 621 else: 622 raise ValueError("Did not find pattern\n", pattern, "\nin\n", filename) 623 endline = corr_line + 6 + b2b 624 while not chunk[endline] == "\n": 625 endline += 1 626 T = endline - start_read 627 628 # all other chunks should follow the same structure 629 rep_idl = [] 630 rep_data = [] 631 632 for cnfg in range(len(data_starts)): 633 start = data_starts[cnfg] 634 stop = start + data_starts[1] 635 chunk = content[start:stop] 636 idl, data = _read_chunk(chunk, gauge_line, cfg_separator, start_read, T, corr_line, b2b, pattern, im, single) 637 rep_idl.append(idl) 638 rep_data.append(data) 639 640 data = [] 641 642 for t in range(T): 643 data.append([]) 644 for c in range(len(rep_data)): 645 data[t].append(rep_data[c][t]) 646 return T, rep_idl, data 647 648 649def _get_rep_names(ls, ens_name=None): 650 new_names = [] 651 for entry in ls: 652 try: 653 idx = entry.index('r') 654 except Exception: 655 raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") 656 657 if ens_name: 658 new_names.append('ens_name' + '|' + entry[idx:]) 659 else: 660 new_names.append(entry[:idx] + '|' + entry[idx:]) 661 return new_names 662 663 664def _get_appended_rep_names(ls, prefix, name, ens_name=None): 665 new_names = [] 666 for exc in ls: 667 if not fnmatch.fnmatch(exc, prefix + '*.' + name): 668 ls = list(set(ls) - set([exc])) 669 ls.sort(key=lambda x: int(re.findall(r'\d+', x)[-1])) 670 for entry in ls: 671 myentry = entry[:-len(name) - 1] 672 try: 673 idx = myentry.index('r') 674 except Exception: 675 raise Exception("Automatic recognition of replicum failed, please enter the key word 'names'.") 676 677 if ens_name: 678 new_names.append('ens_name' + '|' + entry[idx:]) 679 else: 680 new_names.append(myentry[:idx] + '|' + myentry[idx:]) 681 return new_names
sep =
'/'
def
read_sfcf( path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0, wf2=0, version='1.0c', cfg_separator='n', silent=False, **kwargs):
14def read_sfcf(path, prefix, name, quarks='.*', corr_type="bi", noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", silent=False, **kwargs): 15 """Read sfcf files from given folder structure. 16 17 Parameters 18 ---------- 19 path : str 20 Path to the sfcf files. 21 prefix : str 22 Prefix of the sfcf files. 23 name : str 24 Name of the correlation function to read. 25 quarks : str 26 Label of the quarks used in the sfcf input file. e.g. "quark quark" 27 for version 0.0 this does NOT need to be given with the typical " - " 28 that is present in the output file, 29 this is done automatically for this version 30 corr_type : str 31 Type of correlation function to read. Can be 32 - 'bi' for boundary-inner 33 - 'bb' for boundary-boundary 34 - 'bib' for boundary-inner-boundary 35 noffset : int 36 Offset of the source (only relevant when wavefunctions are used) 37 wf : int 38 ID of wave function 39 wf2 : int 40 ID of the second wavefunction 41 (only relevant for boundary-to-boundary correlation functions) 42 im : bool 43 if True, read imaginary instead of real part 44 of the correlation function. 45 names : list 46 Alternative labeling for replicas/ensembles. 47 Has to have the appropriate length 48 ens_name : str 49 replaces the name of the ensemble 50 version: str 51 version of SFCF, with which the measurement was done. 52 if the compact output option (-c) was specified, 53 append a "c" to the version (e.g. "1.0c") 54 if the append output option (-a) was specified, 55 append an "a" to the version 56 cfg_separator : str 57 String that separates the ensemble identifier from the configuration number (default 'n'). 58 replica: list 59 list of replica to be read, default is all 60 files: list 61 list of files to be read per replica, default is all. 62 for non-compact output format, hand the folders to be read here. 63 check_configs: list[list[int]] 64 list of list of supposed configs, eg. [range(1,1000)] 65 for one replicum with 1000 configs 66 67 Returns 68 ------- 69 result: list[Obs] 70 list of Observables with length T, observable per timeslice. 71 bb-type correlators have length 1. 72 """ 73 ret = read_sfcf_multi(path, prefix, [name], quarks_list=[quarks], corr_type_list=[corr_type], 74 noffset_list=[noffset], wf_list=[wf], wf2_list=[wf2], version=version, 75 cfg_separator=cfg_separator, silent=silent, **kwargs) 76 return ret[name][quarks][str(noffset)][str(wf)][str(wf2)]
Read sfcf files from given folder structure.
Parameters
- path (str): Path to the sfcf files.
- prefix (str): Prefix of the sfcf files.
- name (str): Name of the correlation function to read.
- quarks (str): Label of the quarks used in the sfcf input file. e.g. "quark quark" for version 0.0 this does NOT need to be given with the typical " - " that is present in the output file, this is done automatically for this version
- corr_type (str):
Type of correlation function to read. Can be
- 'bi' for boundary-inner
- 'bb' for boundary-boundary
- 'bib' for boundary-inner-boundary
- noffset (int): Offset of the source (only relevant when wavefunctions are used)
- wf (int): ID of wave function
- wf2 (int): ID of the second wavefunction (only relevant for boundary-to-boundary correlation functions)
- im (bool): if True, read imaginary instead of real part of the correlation function.
- names (list): Alternative labeling for replicas/ensembles. Has to have the appropriate length
- ens_name (str): replaces the name of the ensemble
- version (str): version of SFCF, with which the measurement was done. if the compact output option (-c) was specified, append a "c" to the version (e.g. "1.0c") if the append output option (-a) was specified, append an "a" to the version
- cfg_separator (str): String that separates the ensemble identifier from the configuration number (default 'n').
- replica (list): list of replica to be read, default is all
- files (list): list of files to be read per replica, default is all. for non-compact output format, hand the folders to be read here.
- check_configs (list[list[int]]): list of list of supposed configs, eg. [range(1,1000)] for one replicum with 1000 configs
Returns
- result (list[Obs]): list of Observables with length T, observable per timeslice. bb-type correlators have length 1.
def
read_sfcf_multi( path, prefix, name_list, quarks_list=['.*'], corr_type_list=['bi'], noffset_list=[0], wf_list=[0], wf2_list=[0], version='1.0c', cfg_separator='n', silent=False, keyed_out=False, **kwargs):
79def read_sfcf_multi(path, prefix, name_list, quarks_list=['.*'], corr_type_list=['bi'], noffset_list=[0], wf_list=[0], wf2_list=[0], version="1.0c", cfg_separator="n", silent=False, keyed_out=False, **kwargs): 80 """Read sfcf files from given folder structure. 81 82 Parameters 83 ---------- 84 path : str 85 Path to the sfcf files. 86 prefix : str 87 Prefix of the sfcf files. 88 name : str 89 Name of the correlation function to read. 90 quarks_list : list[str] 91 Label of the quarks used in the sfcf input file. e.g. "quark quark" 92 for version 0.0 this does NOT need to be given with the typical " - " 93 that is present in the output file, 94 this is done automatically for this version 95 corr_type_list : list[str] 96 Type of correlation function to read. Can be 97 - 'bi' for boundary-inner 98 - 'bb' for boundary-boundary 99 - 'bib' for boundary-inner-boundary 100 noffset_list : list[int] 101 Offset of the source (only relevant when wavefunctions are used) 102 wf_list : int 103 ID of wave function 104 wf2_list : list[int] 105 ID of the second wavefunction 106 (only relevant for boundary-to-boundary correlation functions) 107 im : bool 108 if True, read imaginary instead of real part 109 of the correlation function. 110 names : list 111 Alternative labeling for replicas/ensembles. 112 Has to have the appropriate length 113 ens_name : str 114 replaces the name of the ensemble 115 version: str 116 version of SFCF, with which the measurement was done. 117 if the compact output option (-c) was specified, 118 append a "c" to the version (e.g. "1.0c") 119 if the append output option (-a) was specified, 120 append an "a" to the version 121 cfg_separator : str 122 String that separates the ensemble identifier from the configuration number (default 'n'). 123 replica: list 124 list of replica to be read, default is all 125 files: list[list[int]] 126 list of files to be read per replica, default is all. 127 for non-compact output format, hand the folders to be read here. 128 check_configs: list[list[int]] 129 list of list of supposed configs, eg. [range(1,1000)] 130 for one replicum with 1000 configs 131 132 Returns 133 ------- 134 result: dict[list[Obs]] 135 dict with one of the following properties: 136 if keyed_out: 137 dict[key] = list[Obs] 138 where key has the form name/quarks/offset/wf/wf2 139 if not keyed_out: 140 dict[name][quarks][offset][wf][wf2] = list[Obs] 141 """ 142 143 if kwargs.get('im'): 144 im = 1 145 part = 'imaginary' 146 else: 147 im = 0 148 part = 'real' 149 150 known_versions = ["0.0", "1.0", "2.0", "1.0c", "2.0c", "1.0a", "2.0a"] 151 152 if version not in known_versions: 153 raise Exception("This version is not known!") 154 if (version[-1] == "c"): 155 appended = False 156 compact = True 157 version = version[:-1] 158 elif (version[-1] == "a"): 159 appended = True 160 compact = False 161 version = version[:-1] 162 else: 163 compact = False 164 appended = False 165 ls = [] 166 if "replica" in kwargs: 167 ls = kwargs.get("replica") 168 else: 169 for (dirpath, dirnames, filenames) in os.walk(path): 170 if not appended: 171 ls.extend(dirnames) 172 else: 173 ls.extend(filenames) 174 break 175 if not ls: 176 raise Exception('Error, directory not found') 177 # Exclude folders with different names 178 for exc in ls: 179 if not fnmatch.fnmatch(exc, prefix + '*'): 180 ls = list(set(ls) - set([exc])) 181 182 if not appended: 183 ls = sort_names(ls) 184 replica = len(ls) 185 186 else: 187 replica = len([file.split(".")[-1] for file in ls]) // len(set([file.split(".")[-1] for file in ls])) 188 if replica == 0: 189 raise Exception('No replica found in directory') 190 if not silent: 191 print('Read', part, 'part of', name_list, 'from', prefix[:-1], ',', replica, 'replica') 192 193 if 'names' in kwargs: 194 new_names = kwargs.get('names') 195 if len(new_names) != len(set(new_names)): 196 raise Exception("names are not unique!") 197 if len(new_names) != replica: 198 raise Exception('names should have the length', replica) 199 200 else: 201 ens_name = kwargs.get("ens_name") 202 if not appended: 203 new_names = _get_rep_names(ls, ens_name) 204 else: 205 new_names = _get_appended_rep_names(ls, prefix, name_list[0], ens_name) 206 new_names = sort_names(new_names) 207 208 idl = [] 209 210 noffset_list = [str(x) for x in noffset_list] 211 wf_list = [str(x) for x in wf_list] 212 wf2_list = [str(x) for x in wf2_list] 213 214 # setup dict structures 215 intern = {} 216 for name, corr_type in zip(name_list, corr_type_list): 217 intern[name] = {} 218 b2b, single = _extract_corr_type(corr_type) 219 intern[name]["b2b"] = b2b 220 intern[name]["single"] = single 221 intern[name]["spec"] = {} 222 for quarks in quarks_list: 223 intern[name]["spec"][quarks] = {} 224 for off in noffset_list: 225 intern[name]["spec"][quarks][off] = {} 226 for w in wf_list: 227 intern[name]["spec"][quarks][off][w] = {} 228 if b2b: 229 for w2 in wf2_list: 230 intern[name]["spec"][quarks][off][w][w2] = {} 231 intern[name]["spec"][quarks][off][w][w2]["pattern"] = _make_pattern(version, name, off, w, w2, intern[name]['b2b'], quarks) 232 else: 233 intern[name]["spec"][quarks][off][w]["0"] = {} 234 intern[name]["spec"][quarks][off][w]["0"]["pattern"] = _make_pattern(version, name, off, w, 0, intern[name]['b2b'], quarks) 235 236 internal_ret_dict = {} 237 needed_keys = [] 238 for name, corr_type in zip(name_list, corr_type_list): 239 b2b, single = _extract_corr_type(corr_type) 240 if b2b: 241 needed_keys.extend(_lists2key([name], quarks_list, noffset_list, wf_list, wf2_list)) 242 else: 243 needed_keys.extend(_lists2key([name], quarks_list, noffset_list, wf_list, ["0"])) 244 245 for key in needed_keys: 246 internal_ret_dict[key] = [] 247 248 if not appended: 249 for i, item in enumerate(ls): 250 rep_path = path + '/' + item 251 if "files" in kwargs: 252 files = kwargs.get("files") 253 if isinstance(files, list): 254 if all(isinstance(f, list) for f in files): 255 files = files[i] 256 elif all(isinstance(f, str) for f in files): 257 files = files 258 else: 259 raise TypeError("files has to be of type list[list[str]] or list[str]!") 260 else: 261 raise TypeError("files has to be of type list[list[str]] or list[str]!") 262 263 else: 264 files = [] 265 sub_ls = _find_files(rep_path, prefix, compact, files) 266 rep_idl = [] 267 no_cfg = len(sub_ls) 268 for cfg in sub_ls: 269 try: 270 if compact: 271 rep_idl.append(int(cfg.split(cfg_separator)[-1])) 272 else: 273 rep_idl.append(int(cfg[3:])) 274 except Exception: 275 raise Exception("Couldn't parse idl from directory, problem with file " + cfg) 276 rep_idl.sort() 277 # maybe there is a better way to print the idls 278 if not silent: 279 print(item, ':', no_cfg, ' configurations') 280 idl.append(rep_idl) 281 # here we have found all the files we need to look into. 282 if i == 0: 283 if version != "0.0" and compact: 284 file = path + '/' + item + '/' + sub_ls[0] 285 for name_index, name in enumerate(name_list): 286 if version == "0.0" or not compact: 287 file = path + '/' + item + '/' + sub_ls[0] + '/' + name 288 if corr_type_list[name_index] == 'bi': 289 name_keys = _lists2key(quarks_list, noffset_list, wf_list, ["0"]) 290 else: 291 name_keys = _lists2key(quarks_list, noffset_list, wf_list, wf2_list) 292 for key in name_keys: 293 specs = _key2specs(key) 294 quarks = specs[0] 295 off = specs[1] 296 w = specs[2] 297 w2 = specs[3] 298 # here, we want to find the place within the file, 299 # where the correlator we need is stored. 300 # to do so, the pattern needed is put together 301 # from the input values 302 start_read, T = _find_correlator(file, version, intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["pattern"], intern[name]['b2b'], silent=silent) 303 intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["start"] = start_read 304 intern[name]["T"] = T 305 # preparing the datastructure 306 # the correlators get parsed into... 307 deltas = [] 308 for j in range(intern[name]["T"]): 309 deltas.append([]) 310 internal_ret_dict[sep.join([name, key])] = deltas 311 312 if compact: 313 rep_deltas = _read_compact_rep(path, item, sub_ls, intern, needed_keys, im) 314 for key in needed_keys: 315 name = _key2specs(key)[0] 316 for t in range(intern[name]["T"]): 317 internal_ret_dict[key][t].append(rep_deltas[key][t]) 318 else: 319 for key in needed_keys: 320 rep_data = [] 321 name = _key2specs(key)[0] 322 for subitem in sub_ls: 323 cfg_path = path + '/' + item + '/' + subitem 324 file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im) 325 rep_data.append(file_data) 326 for t in range(intern[name]["T"]): 327 internal_ret_dict[key][t].append([]) 328 for cfg in range(no_cfg): 329 internal_ret_dict[key][t][i].append(rep_data[cfg][key][t]) 330 else: 331 for key in needed_keys: 332 specs = _key2specs(key) 333 name = specs[0] 334 quarks = specs[1] 335 off = specs[2] 336 w = specs[3] 337 w2 = specs[4] 338 if "files" in kwargs: 339 if isinstance(kwargs.get("files"), list) and all(isinstance(f, str) for f in kwargs.get("files")): 340 name_ls = kwargs.get("files") 341 else: 342 raise TypeError("In append mode, files has to be of type list[str]!") 343 else: 344 name_ls = ls 345 for exc in name_ls: 346 if not fnmatch.fnmatch(exc, prefix + '*.' + name): 347 name_ls = list(set(name_ls) - set([exc])) 348 name_ls = sort_names(name_ls) 349 pattern = intern[name]['spec'][quarks][off][w][w2]['pattern'] 350 deltas = [] 351 for rep, file in enumerate(name_ls): 352 rep_idl = [] 353 filename = path + '/' + file 354 T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single']) 355 if rep == 0: 356 intern[name]['T'] = T 357 for t in range(intern[name]['T']): 358 deltas.append([]) 359 for t in range(intern[name]['T']): 360 deltas[t].append(rep_data[t]) 361 internal_ret_dict[key] = deltas 362 if name == name_list[0]: 363 idl.append(rep_idl) 364 365 if kwargs.get("check_configs") is True: 366 if not silent: 367 print("Checking for missing configs...") 368 che = kwargs.get("check_configs") 369 if not (len(che) == len(idl)): 370 raise Exception("check_configs has to be the same length as replica!") 371 for r in range(len(idl)): 372 if not silent: 373 print("checking " + new_names[r]) 374 check_idl(idl[r], che[r]) 375 if not silent: 376 print("Done") 377 378 result_dict = {} 379 if keyed_out: 380 for key in needed_keys: 381 name = _key2specs(key)[0] 382 result = [] 383 for t in range(intern[name]["T"]): 384 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) 385 result_dict[key] = result 386 else: 387 for name, corr_type in zip(name_list, corr_type_list): 388 result_dict[name] = {} 389 for quarks in quarks_list: 390 result_dict[name][quarks] = {} 391 for off in noffset_list: 392 result_dict[name][quarks][off] = {} 393 for w in wf_list: 394 result_dict[name][quarks][off][w] = {} 395 if corr_type != 'bi': 396 for w2 in wf2_list: 397 key = _specs2key(name, quarks, off, w, w2) 398 result = [] 399 for t in range(intern[name]["T"]): 400 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) 401 result_dict[name][quarks][str(off)][str(w)][str(w2)] = result 402 else: 403 key = _specs2key(name, quarks, off, w, "0") 404 result = [] 405 for t in range(intern[name]["T"]): 406 result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl)) 407 result_dict[name][quarks][str(off)][str(w)][str(0)] = result 408 return result_dict
Read sfcf files from given folder structure.
Parameters
- path (str): Path to the sfcf files.
- prefix (str): Prefix of the sfcf files.
- name (str): Name of the correlation function to read.
- quarks_list (list[str]): Label of the quarks used in the sfcf input file. e.g. "quark quark" for version 0.0 this does NOT need to be given with the typical " - " that is present in the output file, this is done automatically for this version
- corr_type_list (list[str]):
Type of correlation function to read. Can be
- 'bi' for boundary-inner
- 'bb' for boundary-boundary
- 'bib' for boundary-inner-boundary
- noffset_list (list[int]): Offset of the source (only relevant when wavefunctions are used)
- wf_list (int): ID of wave function
- wf2_list (list[int]): ID of the second wavefunction (only relevant for boundary-to-boundary correlation functions)
- im (bool): if True, read imaginary instead of real part of the correlation function.
- names (list): Alternative labeling for replicas/ensembles. Has to have the appropriate length
- ens_name (str): replaces the name of the ensemble
- version (str): version of SFCF, with which the measurement was done. if the compact output option (-c) was specified, append a "c" to the version (e.g. "1.0c") if the append output option (-a) was specified, append an "a" to the version
- cfg_separator (str): String that separates the ensemble identifier from the configuration number (default 'n').
- replica (list): list of replica to be read, default is all
- files (list[list[int]]): list of files to be read per replica, default is all. for non-compact output format, hand the folders to be read here.
- check_configs (list[list[int]]): list of list of supposed configs, eg. [range(1,1000)] for one replicum with 1000 configs
Returns
- result (dict[list[Obs]]): dict with one of the following properties: if keyed_out: dict[key] = list[Obs] where key has the form name/quarks/offset/wf/wf2 if not keyed_out: dict[name][quarks][offset][wf][wf2] = list[Obs]