pyerrors.input.json
1import rapidjson as json 2import gzip 3import getpass 4import socket 5import datetime 6import platform 7import warnings 8import re 9import numpy as np 10from ..obs import Obs 11from ..covobs import Covobs 12from ..correlators import Corr 13from ..misc import _assert_equal_properties 14from .. import version as pyerrorsversion 15 16 17def create_json_string(ol, description='', indent=1): 18 """Generate the string for the export of a list of Obs or structures containing Obs 19 to a .json(.gz) file 20 21 Parameters 22 ---------- 23 ol : list 24 List of objects that will be exported. At the moment, these objects can be 25 either of: Obs, list, numpy.ndarray, Corr. 26 All Obs inside a structure have to be defined on the same set of configurations. 27 description : str 28 Optional string that describes the contents of the json file. 29 indent : int 30 Specify the indentation level of the json file. None or 0 is permissible and 31 saves disk space. 32 33 Returns 34 ------- 35 json_string : str 36 String for export to .json(.gz) file 37 """ 38 39 def _gen_data_d_from_list(ol): 40 dl = [] 41 No = len(ol) 42 for name in ol[0].mc_names: 43 ed = {} 44 ed['id'] = name 45 ed['replica'] = [] 46 for r_name in ol[0].e_content[name]: 47 rd = {} 48 rd['name'] = r_name 49 rd['deltas'] = [] 50 offsets = [o.r_values[r_name] - o.value for o in ol] 51 deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)]) 52 for i in range(len(ol[0].idl[r_name])): 53 rd['deltas'].append([ol[0].idl[r_name][i]]) 54 rd['deltas'][-1] += deltas[i].tolist() 55 ed['replica'].append(rd) 56 dl.append(ed) 57 return dl 58 59 def _gen_cdata_d_from_list(ol): 60 dl = [] 61 for name in ol[0].cov_names: 62 ed = {} 63 ed['id'] = name 64 ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') 65 ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) 66 ncov = ol[0].covobs[name].cov.shape[0] 67 ed['grad'] = [] 68 for i in range(ncov): 69 ed['grad'].append([]) 70 for o in ol: 71 ed['grad'][-1].append(o.covobs[name].grad[i][0]) 72 dl.append(ed) 73 return dl 74 75 def write_Obs_to_dict(o): 76 d = {} 77 d['type'] = 'Obs' 78 d['layout'] = '1' 79 if o.tag: 80 d['tag'] = [o.tag] 81 if o.reweighted: 82 d['reweighted'] = o.reweighted 83 d['value'] = [o.value] 84 data = _gen_data_d_from_list([o]) 85 if len(data) > 0: 86 d['data'] = data 87 cdata = _gen_cdata_d_from_list([o]) 88 if len(cdata) > 0: 89 d['cdata'] = cdata 90 return d 91 92 def write_List_to_dict(ol): 93 _assert_equal_properties(ol) 94 d = {} 95 d['type'] = 'List' 96 d['layout'] = '%d' % len(ol) 97 taglist = [o.tag for o in ol] 98 if np.any([tag is not None for tag in taglist]): 99 d['tag'] = taglist 100 if ol[0].reweighted: 101 d['reweighted'] = ol[0].reweighted 102 d['value'] = [o.value for o in ol] 103 data = _gen_data_d_from_list(ol) 104 if len(data) > 0: 105 d['data'] = data 106 cdata = _gen_cdata_d_from_list(ol) 107 if len(cdata) > 0: 108 d['cdata'] = cdata 109 return d 110 111 def write_Array_to_dict(oa): 112 ol = np.ravel(oa) 113 _assert_equal_properties(ol) 114 d = {} 115 d['type'] = 'Array' 116 d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') 117 taglist = [o.tag for o in ol] 118 if np.any([tag is not None for tag in taglist]): 119 d['tag'] = taglist 120 if ol[0].reweighted: 121 d['reweighted'] = ol[0].reweighted 122 d['value'] = [o.value for o in ol] 123 data = _gen_data_d_from_list(ol) 124 if len(data) > 0: 125 d['data'] = data 126 cdata = _gen_cdata_d_from_list(ol) 127 if len(cdata) > 0: 128 d['cdata'] = cdata 129 return d 130 131 def _nan_Obs_like(obs): 132 samples = [] 133 names = [] 134 idl = [] 135 for key, value in obs.idl.items(): 136 samples.append([np.nan] * len(value)) 137 names.append(key) 138 idl.append(value) 139 my_obs = Obs(samples, names, idl) 140 my_obs._covobs = obs._covobs 141 for name in obs._covobs: 142 my_obs.names.append(name) 143 my_obs.reweighted = obs.reweighted 144 return my_obs 145 146 def write_Corr_to_dict(my_corr): 147 first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j)) 148 dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object) 149 dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0]) 150 content = [o if o is not None else dummy_array for o in my_corr.content] 151 dat = write_Array_to_dict(np.array(content, dtype=object)) 152 dat['type'] = 'Corr' 153 corr_meta_data = str(my_corr.tag) 154 if 'tag' in dat.keys(): 155 dat['tag'].append(corr_meta_data) 156 else: 157 dat['tag'] = [corr_meta_data] 158 taglist = dat['tag'] 159 dat['tag'] = {} # tag is now a dictionary, that contains the previous taglist in the key "tag" 160 dat['tag']['tag'] = taglist 161 if my_corr.prange is not None: 162 dat['tag']['prange'] = my_corr.prange 163 return dat 164 165 if not isinstance(ol, list): 166 ol = [ol] 167 168 d = {} 169 d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__) 170 d['version'] = '1.1' 171 d['who'] = getpass.getuser() 172 d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z') 173 d['host'] = socket.gethostname() + ', ' + platform.platform() 174 175 if description: 176 d['description'] = description 177 178 d['obsdata'] = [] 179 for io in ol: 180 if isinstance(io, Obs): 181 d['obsdata'].append(write_Obs_to_dict(io)) 182 elif isinstance(io, list): 183 d['obsdata'].append(write_List_to_dict(io)) 184 elif isinstance(io, np.ndarray): 185 d['obsdata'].append(write_Array_to_dict(io)) 186 elif isinstance(io, Corr): 187 d['obsdata'].append(write_Corr_to_dict(io)) 188 else: 189 raise Exception("Unkown datatype.") 190 191 def _jsonifier(o): 192 if isinstance(o, np.int64): 193 return int(o) 194 raise TypeError('%r is not JSON serializable' % o) 195 196 if indent: 197 return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY) 198 else: 199 return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT) 200 201 202def dump_to_json(ol, fname, description='', indent=1, gz=True): 203 """Export a list of Obs or structures containing Obs to a .json(.gz) file 204 205 Parameters 206 ---------- 207 ol : list 208 List of objects that will be exported. At the moment, these objects can be 209 either of: Obs, list, numpy.ndarray, Corr. 210 All Obs inside a structure have to be defined on the same set of configurations. 211 fname : str 212 Filename of the output file. 213 description : str 214 Optional string that describes the contents of the json file. 215 indent : int 216 Specify the indentation level of the json file. None or 0 is permissible and 217 saves disk space. 218 gz : bool 219 If True, the output is a gzipped json. If False, the output is a json file. 220 221 Returns 222 ------- 223 Null 224 """ 225 226 jsonstring = create_json_string(ol, description, indent) 227 228 if not fname.endswith('.json') and not fname.endswith('.gz'): 229 fname += '.json' 230 231 if gz: 232 if not fname.endswith('.gz'): 233 fname += '.gz' 234 235 fp = gzip.open(fname, 'wb') 236 fp.write(jsonstring.encode('utf-8')) 237 else: 238 fp = open(fname, 'w', encoding='utf-8') 239 fp.write(jsonstring) 240 fp.close() 241 242 243def _parse_json_dict(json_dict, verbose=True, full_output=False): 244 """Reconstruct a list of Obs or structures containing Obs from a dict that 245 was built out of a json string. 246 247 The following structures are supported: Obs, list, numpy.ndarray, Corr 248 If the list contains only one element, it is unpacked from the list. 249 250 Parameters 251 ---------- 252 json_string : str 253 json string containing the data. 254 verbose : bool 255 Print additional information that was written to the file. 256 full_output : bool 257 If True, a dict containing auxiliary information and the data is returned. 258 If False, only the data is returned. 259 260 Returns 261 ------- 262 result : list[Obs] 263 reconstructed list of observables from the json string 264 or 265 result : Obs 266 only one observable if the list only has one entry 267 or 268 result : dict 269 if full_output=True 270 """ 271 272 def _gen_obsd_from_datad(d): 273 retd = {} 274 if d: 275 retd['names'] = [] 276 retd['idl'] = [] 277 retd['deltas'] = [] 278 for ens in d: 279 for rep in ens['replica']: 280 rep_name = rep['name'] 281 if len(rep_name) > len(ens["id"]): 282 if rep_name[len(ens["id"])] != "|": 283 tmp_list = list(rep_name) 284 tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):] 285 rep_name = ''.join(tmp_list) 286 retd['names'].append(rep_name) 287 retd['idl'].append([di[0] for di in rep['deltas']]) 288 retd['deltas'].append(np.array([di[1:] for di in rep['deltas']])) 289 return retd 290 291 def _gen_covobsd_from_cdatad(d): 292 retd = {} 293 for ens in d: 294 retl = [] 295 name = ens['id'] 296 layouts = ens.get('layout', '1').strip() 297 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] 298 cov = np.reshape(ens['cov'], layout) 299 grad = ens['grad'] 300 nobs = len(grad[0]) 301 for i in range(nobs): 302 retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]}) 303 retd[name] = retl 304 return retd 305 306 def get_Obs_from_dict(o): 307 layouts = o.get('layout', '1').strip() 308 if layouts != '1': 309 raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning) 310 311 values = o['value'] 312 od = _gen_obsd_from_datad(o.get('data', {})) 313 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 314 315 if od: 316 ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl']) 317 ret._value = values[0] 318 else: 319 ret = Obs([], [], means=[]) 320 ret._value = values[0] 321 for name in cd: 322 co = cd[name][0] 323 ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 324 ret.names.append(co['name']) 325 326 ret.reweighted = o.get('reweighted', False) 327 ret.tag = o.get('tag', [None])[0] 328 return ret 329 330 def get_List_from_dict(o): 331 layouts = o.get('layout', '1').strip() 332 layout = int(layouts) 333 values = o['value'] 334 od = _gen_obsd_from_datad(o.get('data', {})) 335 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 336 337 ret = [] 338 taglist = o.get('tag', layout * [None]) 339 for i in range(layout): 340 if od: 341 ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl'])) 342 ret[-1]._value = values[i] 343 else: 344 ret.append(Obs([], [], means=[])) 345 ret[-1]._value = values[i] 346 print('Created Obs with means= ', values[i]) 347 for name in cd: 348 co = cd[name][i] 349 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 350 ret[-1].names.append(co['name']) 351 352 ret[-1].reweighted = o.get('reweighted', False) 353 ret[-1].tag = taglist[i] 354 return ret 355 356 def get_Array_from_dict(o): 357 layouts = o.get('layout', '1').strip() 358 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] 359 N = np.prod(layout) 360 values = o['value'] 361 od = _gen_obsd_from_datad(o.get('data', {})) 362 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 363 364 ret = [] 365 taglist = o.get('tag', N * [None]) 366 for i in range(N): 367 if od: 368 ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl'])) 369 ret[-1]._value = values[i] 370 else: 371 ret.append(Obs([], [], means=[])) 372 ret[-1]._value = values[i] 373 for name in cd: 374 co = cd[name][i] 375 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 376 ret[-1].names.append(co['name']) 377 ret[-1].reweighted = o.get('reweighted', False) 378 ret[-1].tag = taglist[i] 379 return np.reshape(ret, layout) 380 381 def get_Corr_from_dict(o): 382 if isinstance(o.get('tag'), list): # supports the old way 383 taglist = o.get('tag') # This had to be modified to get the taglist from the dictionary 384 temp_prange = None 385 elif isinstance(o.get('tag'), dict): 386 tagdic = o.get('tag') 387 taglist = tagdic['tag'] 388 if 'prange' in tagdic: 389 temp_prange = tagdic['prange'] 390 else: 391 temp_prange = None 392 else: 393 raise Exception("The tag is not a list or dict") 394 395 corr_tag = taglist[-1] 396 tmp_o = o 397 tmp_o['tag'] = taglist[:-1] 398 if len(tmp_o['tag']) == 0: 399 del tmp_o['tag'] 400 dat = get_Array_from_dict(tmp_o) 401 my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)]) 402 if corr_tag != 'None': 403 my_corr.tag = corr_tag 404 405 my_corr.prange = temp_prange 406 return my_corr 407 408 prog = json_dict.get('program', '') 409 version = json_dict.get('version', '') 410 who = json_dict.get('who', '') 411 date = json_dict.get('date', '') 412 host = json_dict.get('host', '') 413 if prog and verbose: 414 print('Data has been written using %s.' % (prog)) 415 if version and verbose: 416 print('Format version %s' % (version)) 417 if np.any([who, date, host] and verbose): 418 print('Written by %s on %s on host %s' % (who, date, host)) 419 description = json_dict.get('description', '') 420 if description and verbose: 421 print() 422 print('Description: ', description) 423 obsdata = json_dict['obsdata'] 424 ol = [] 425 for io in obsdata: 426 if io['type'] == 'Obs': 427 ol.append(get_Obs_from_dict(io)) 428 elif io['type'] == 'List': 429 ol.append(get_List_from_dict(io)) 430 elif io['type'] == 'Array': 431 ol.append(get_Array_from_dict(io)) 432 elif io['type'] == 'Corr': 433 ol.append(get_Corr_from_dict(io)) 434 else: 435 raise Exception("Unknown datatype.") 436 437 if full_output: 438 retd = {} 439 retd['program'] = prog 440 retd['version'] = version 441 retd['who'] = who 442 retd['date'] = date 443 retd['host'] = host 444 retd['description'] = description 445 retd['obsdata'] = ol 446 447 return retd 448 else: 449 if len(obsdata) == 1: 450 ol = ol[0] 451 452 return ol 453 454 455def import_json_string(json_string, verbose=True, full_output=False): 456 """Reconstruct a list of Obs or structures containing Obs from a json string. 457 458 The following structures are supported: Obs, list, numpy.ndarray, Corr 459 If the list contains only one element, it is unpacked from the list. 460 461 Parameters 462 ---------- 463 json_string : str 464 json string containing the data. 465 verbose : bool 466 Print additional information that was written to the file. 467 full_output : bool 468 If True, a dict containing auxiliary information and the data is returned. 469 If False, only the data is returned. 470 471 Returns 472 ------- 473 result : list[Obs] 474 reconstructed list of observables from the json string 475 or 476 result : Obs 477 only one observable if the list only has one entry 478 or 479 result : dict 480 if full_output=True 481 """ 482 483 return _parse_json_dict(json.loads(json_string), verbose, full_output) 484 485 486def load_json(fname, verbose=True, gz=True, full_output=False): 487 """Import a list of Obs or structures containing Obs from a .json(.gz) file. 488 489 The following structures are supported: Obs, list, numpy.ndarray, Corr 490 If the list contains only one element, it is unpacked from the list. 491 492 Parameters 493 ---------- 494 fname : str 495 Filename of the input file. 496 verbose : bool 497 Print additional information that was written to the file. 498 gz : bool 499 If True, assumes that data is gzipped. If False, assumes JSON file. 500 full_output : bool 501 If True, a dict containing auxiliary information and the data is returned. 502 If False, only the data is returned. 503 504 Returns 505 ------- 506 result : list[Obs] 507 reconstructed list of observables from the json string 508 or 509 result : Obs 510 only one observable if the list only has one entry 511 or 512 result : dict 513 if full_output=True 514 """ 515 if not fname.endswith('.json') and not fname.endswith('.gz'): 516 fname += '.json' 517 if gz: 518 if not fname.endswith('.gz'): 519 fname += '.gz' 520 with gzip.open(fname, 'r') as fin: 521 d = json.load(fin) 522 else: 523 if fname.endswith('.gz'): 524 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) 525 with open(fname, 'r', encoding='utf-8') as fin: 526 d = json.loads(fin.read()) 527 528 return _parse_json_dict(d, verbose, full_output) 529 530 531def _ol_from_dict(ind, reps='DICTOBS'): 532 """Convert a dictionary of Obs objects to a list and a dictionary that contains 533 placeholders instead of the Obs objects. 534 535 Parameters 536 ---------- 537 ind : dict 538 Dict of JSON valid structures and objects that will be exported. 539 At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr. 540 All Obs inside a structure have to be defined on the same set of configurations. 541 reps : str 542 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 543 """ 544 545 obstypes = (Obs, Corr, np.ndarray) 546 547 if not reps.isalnum(): 548 raise Exception('Placeholder string has to be alphanumeric!') 549 ol = [] 550 counter = 0 551 552 def dict_replace_obs(d): 553 nonlocal ol 554 nonlocal counter 555 x = {} 556 for k, v in d.items(): 557 if isinstance(v, dict): 558 v = dict_replace_obs(v) 559 elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]): 560 v = obslist_replace_obs(v) 561 elif isinstance(v, list): 562 v = list_replace_obs(v) 563 elif isinstance(v, obstypes): 564 ol.append(v) 565 v = reps + '%d' % (counter) 566 counter += 1 567 elif isinstance(v, str): 568 if bool(re.match(r'%s[0-9]+' % (reps), v)): 569 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps)) 570 x[k] = v 571 return x 572 573 def list_replace_obs(li): 574 nonlocal ol 575 nonlocal counter 576 x = [] 577 for e in li: 578 if isinstance(e, list): 579 e = list_replace_obs(e) 580 elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]): 581 e = obslist_replace_obs(e) 582 elif isinstance(e, dict): 583 e = dict_replace_obs(e) 584 elif isinstance(e, obstypes): 585 ol.append(e) 586 e = reps + '%d' % (counter) 587 counter += 1 588 elif isinstance(e, str): 589 if bool(re.match(r'%s[0-9]+' % (reps), e)): 590 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps)) 591 x.append(e) 592 return x 593 594 def obslist_replace_obs(li): 595 nonlocal ol 596 nonlocal counter 597 il = [] 598 for e in li: 599 il.append(e) 600 601 ol.append(il) 602 x = reps + '%d' % (counter) 603 counter += 1 604 return x 605 606 nd = dict_replace_obs(ind) 607 608 return ol, nd 609 610 611def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): 612 """Export a dict of Obs or structures containing Obs to a .json(.gz) file 613 614 Parameters 615 ---------- 616 od : dict 617 Dict of JSON valid structures and objects that will be exported. 618 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 619 All Obs inside a structure have to be defined on the same set of configurations. 620 fname : str 621 Filename of the output file. 622 description : str 623 Optional string that describes the contents of the json file. 624 indent : int 625 Specify the indentation level of the json file. None or 0 is permissible and 626 saves disk space. 627 reps : str 628 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 629 gz : bool 630 If True, the output is a gzipped json. If False, the output is a json file. 631 632 Returns 633 ------- 634 None 635 """ 636 637 if not isinstance(od, dict): 638 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') 639 640 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' 641 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' 642 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' 643 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') 644 645 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} 646 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) 647 648 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz) 649 650 651def _od_from_list_and_dict(ol, ind, reps='DICTOBS'): 652 """Parse a list of Obs or structures containing Obs and an accompanying 653 dict, where the structures have been replaced by placeholders to a 654 dict that contains the structures. 655 656 The following structures are supported: Obs, list, numpy.ndarray, Corr 657 658 Parameters 659 ---------- 660 ol : list 661 List of objects - 662 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 663 All Obs inside a structure have to be defined on the same set of configurations. 664 ind : dict 665 Dict that defines the structure of the resulting dict and contains placeholders 666 reps : str 667 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 668 """ 669 if not reps.isalnum(): 670 raise Exception('Placeholder string has to be alphanumeric!') 671 672 counter = 0 673 674 def dict_replace_string(d): 675 nonlocal counter 676 nonlocal ol 677 x = {} 678 for k, v in d.items(): 679 if isinstance(v, dict): 680 v = dict_replace_string(v) 681 elif isinstance(v, list): 682 v = list_replace_string(v) 683 elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)): 684 index = int(v[len(reps):]) 685 v = ol[index] 686 counter += 1 687 x[k] = v 688 return x 689 690 def list_replace_string(li): 691 nonlocal counter 692 nonlocal ol 693 x = [] 694 for e in li: 695 if isinstance(e, list): 696 e = list_replace_string(e) 697 elif isinstance(e, dict): 698 e = dict_replace_string(e) 699 elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)): 700 index = int(e[len(reps):]) 701 e = ol[index] 702 counter += 1 703 x.append(e) 704 return x 705 706 nd = dict_replace_string(ind) 707 708 if counter == 0: 709 raise Exception('No placeholder has been replaced! Check if reps is set correctly.') 710 711 return nd 712 713 714def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): 715 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. 716 717 The following structures are supported: Obs, list, numpy.ndarray, Corr 718 719 Parameters 720 ---------- 721 fname : str 722 Filename of the input file. 723 verbose : bool 724 Print additional information that was written to the file. 725 gz : bool 726 If True, assumes that data is gzipped. If False, assumes JSON file. 727 full_output : bool 728 If True, a dict containing auxiliary information and the data is returned. 729 If False, only the data is returned. 730 reps : str 731 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 732 733 Returns 734 ------- 735 data : Obs / list / Corr 736 Read data 737 or 738 data : dict 739 Read data and meta-data 740 """ 741 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) 742 description = indata['description']['description'] 743 indict = indata['description']['OBSDICT'] 744 ol = indata['obsdata'] 745 od = _od_from_list_and_dict(ol, indict, reps=reps) 746 747 if full_output: 748 indata['description'] = description 749 indata['obsdata'] = od 750 return indata 751 else: 752 return od
18def create_json_string(ol, description='', indent=1): 19 """Generate the string for the export of a list of Obs or structures containing Obs 20 to a .json(.gz) file 21 22 Parameters 23 ---------- 24 ol : list 25 List of objects that will be exported. At the moment, these objects can be 26 either of: Obs, list, numpy.ndarray, Corr. 27 All Obs inside a structure have to be defined on the same set of configurations. 28 description : str 29 Optional string that describes the contents of the json file. 30 indent : int 31 Specify the indentation level of the json file. None or 0 is permissible and 32 saves disk space. 33 34 Returns 35 ------- 36 json_string : str 37 String for export to .json(.gz) file 38 """ 39 40 def _gen_data_d_from_list(ol): 41 dl = [] 42 No = len(ol) 43 for name in ol[0].mc_names: 44 ed = {} 45 ed['id'] = name 46 ed['replica'] = [] 47 for r_name in ol[0].e_content[name]: 48 rd = {} 49 rd['name'] = r_name 50 rd['deltas'] = [] 51 offsets = [o.r_values[r_name] - o.value for o in ol] 52 deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)]) 53 for i in range(len(ol[0].idl[r_name])): 54 rd['deltas'].append([ol[0].idl[r_name][i]]) 55 rd['deltas'][-1] += deltas[i].tolist() 56 ed['replica'].append(rd) 57 dl.append(ed) 58 return dl 59 60 def _gen_cdata_d_from_list(ol): 61 dl = [] 62 for name in ol[0].cov_names: 63 ed = {} 64 ed['id'] = name 65 ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') 66 ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) 67 ncov = ol[0].covobs[name].cov.shape[0] 68 ed['grad'] = [] 69 for i in range(ncov): 70 ed['grad'].append([]) 71 for o in ol: 72 ed['grad'][-1].append(o.covobs[name].grad[i][0]) 73 dl.append(ed) 74 return dl 75 76 def write_Obs_to_dict(o): 77 d = {} 78 d['type'] = 'Obs' 79 d['layout'] = '1' 80 if o.tag: 81 d['tag'] = [o.tag] 82 if o.reweighted: 83 d['reweighted'] = o.reweighted 84 d['value'] = [o.value] 85 data = _gen_data_d_from_list([o]) 86 if len(data) > 0: 87 d['data'] = data 88 cdata = _gen_cdata_d_from_list([o]) 89 if len(cdata) > 0: 90 d['cdata'] = cdata 91 return d 92 93 def write_List_to_dict(ol): 94 _assert_equal_properties(ol) 95 d = {} 96 d['type'] = 'List' 97 d['layout'] = '%d' % len(ol) 98 taglist = [o.tag for o in ol] 99 if np.any([tag is not None for tag in taglist]): 100 d['tag'] = taglist 101 if ol[0].reweighted: 102 d['reweighted'] = ol[0].reweighted 103 d['value'] = [o.value for o in ol] 104 data = _gen_data_d_from_list(ol) 105 if len(data) > 0: 106 d['data'] = data 107 cdata = _gen_cdata_d_from_list(ol) 108 if len(cdata) > 0: 109 d['cdata'] = cdata 110 return d 111 112 def write_Array_to_dict(oa): 113 ol = np.ravel(oa) 114 _assert_equal_properties(ol) 115 d = {} 116 d['type'] = 'Array' 117 d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') 118 taglist = [o.tag for o in ol] 119 if np.any([tag is not None for tag in taglist]): 120 d['tag'] = taglist 121 if ol[0].reweighted: 122 d['reweighted'] = ol[0].reweighted 123 d['value'] = [o.value for o in ol] 124 data = _gen_data_d_from_list(ol) 125 if len(data) > 0: 126 d['data'] = data 127 cdata = _gen_cdata_d_from_list(ol) 128 if len(cdata) > 0: 129 d['cdata'] = cdata 130 return d 131 132 def _nan_Obs_like(obs): 133 samples = [] 134 names = [] 135 idl = [] 136 for key, value in obs.idl.items(): 137 samples.append([np.nan] * len(value)) 138 names.append(key) 139 idl.append(value) 140 my_obs = Obs(samples, names, idl) 141 my_obs._covobs = obs._covobs 142 for name in obs._covobs: 143 my_obs.names.append(name) 144 my_obs.reweighted = obs.reweighted 145 return my_obs 146 147 def write_Corr_to_dict(my_corr): 148 first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j)) 149 dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object) 150 dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0]) 151 content = [o if o is not None else dummy_array for o in my_corr.content] 152 dat = write_Array_to_dict(np.array(content, dtype=object)) 153 dat['type'] = 'Corr' 154 corr_meta_data = str(my_corr.tag) 155 if 'tag' in dat.keys(): 156 dat['tag'].append(corr_meta_data) 157 else: 158 dat['tag'] = [corr_meta_data] 159 taglist = dat['tag'] 160 dat['tag'] = {} # tag is now a dictionary, that contains the previous taglist in the key "tag" 161 dat['tag']['tag'] = taglist 162 if my_corr.prange is not None: 163 dat['tag']['prange'] = my_corr.prange 164 return dat 165 166 if not isinstance(ol, list): 167 ol = [ol] 168 169 d = {} 170 d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__) 171 d['version'] = '1.1' 172 d['who'] = getpass.getuser() 173 d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z') 174 d['host'] = socket.gethostname() + ', ' + platform.platform() 175 176 if description: 177 d['description'] = description 178 179 d['obsdata'] = [] 180 for io in ol: 181 if isinstance(io, Obs): 182 d['obsdata'].append(write_Obs_to_dict(io)) 183 elif isinstance(io, list): 184 d['obsdata'].append(write_List_to_dict(io)) 185 elif isinstance(io, np.ndarray): 186 d['obsdata'].append(write_Array_to_dict(io)) 187 elif isinstance(io, Corr): 188 d['obsdata'].append(write_Corr_to_dict(io)) 189 else: 190 raise Exception("Unkown datatype.") 191 192 def _jsonifier(o): 193 if isinstance(o, np.int64): 194 return int(o) 195 raise TypeError('%r is not JSON serializable' % o) 196 197 if indent: 198 return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY) 199 else: 200 return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)
Generate the string for the export of a list of Obs or structures containing Obs to a .json(.gz) file
Parameters
- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
Returns
- json_string (str): String for export to .json(.gz) file
203def dump_to_json(ol, fname, description='', indent=1, gz=True): 204 """Export a list of Obs or structures containing Obs to a .json(.gz) file 205 206 Parameters 207 ---------- 208 ol : list 209 List of objects that will be exported. At the moment, these objects can be 210 either of: Obs, list, numpy.ndarray, Corr. 211 All Obs inside a structure have to be defined on the same set of configurations. 212 fname : str 213 Filename of the output file. 214 description : str 215 Optional string that describes the contents of the json file. 216 indent : int 217 Specify the indentation level of the json file. None or 0 is permissible and 218 saves disk space. 219 gz : bool 220 If True, the output is a gzipped json. If False, the output is a json file. 221 222 Returns 223 ------- 224 Null 225 """ 226 227 jsonstring = create_json_string(ol, description, indent) 228 229 if not fname.endswith('.json') and not fname.endswith('.gz'): 230 fname += '.json' 231 232 if gz: 233 if not fname.endswith('.gz'): 234 fname += '.gz' 235 236 fp = gzip.open(fname, 'wb') 237 fp.write(jsonstring.encode('utf-8')) 238 else: 239 fp = open(fname, 'w', encoding='utf-8') 240 fp.write(jsonstring) 241 fp.close()
Export a list of Obs or structures containing Obs to a .json(.gz) file
Parameters
- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.
Returns
- Null
456def import_json_string(json_string, verbose=True, full_output=False): 457 """Reconstruct a list of Obs or structures containing Obs from a json string. 458 459 The following structures are supported: Obs, list, numpy.ndarray, Corr 460 If the list contains only one element, it is unpacked from the list. 461 462 Parameters 463 ---------- 464 json_string : str 465 json string containing the data. 466 verbose : bool 467 Print additional information that was written to the file. 468 full_output : bool 469 If True, a dict containing auxiliary information and the data is returned. 470 If False, only the data is returned. 471 472 Returns 473 ------- 474 result : list[Obs] 475 reconstructed list of observables from the json string 476 or 477 result : Obs 478 only one observable if the list only has one entry 479 or 480 result : dict 481 if full_output=True 482 """ 483 484 return _parse_json_dict(json.loads(json_string), verbose, full_output)
Reconstruct a list of Obs or structures containing Obs from a json string.
The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.
Parameters
- json_string (str): json string containing the data.
- verbose (bool): Print additional information that was written to the file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
Returns
- result (list[Obs]): reconstructed list of observables from the json string
- or
- result (Obs): only one observable if the list only has one entry
- or
- result (dict): if full_output=True
487def load_json(fname, verbose=True, gz=True, full_output=False): 488 """Import a list of Obs or structures containing Obs from a .json(.gz) file. 489 490 The following structures are supported: Obs, list, numpy.ndarray, Corr 491 If the list contains only one element, it is unpacked from the list. 492 493 Parameters 494 ---------- 495 fname : str 496 Filename of the input file. 497 verbose : bool 498 Print additional information that was written to the file. 499 gz : bool 500 If True, assumes that data is gzipped. If False, assumes JSON file. 501 full_output : bool 502 If True, a dict containing auxiliary information and the data is returned. 503 If False, only the data is returned. 504 505 Returns 506 ------- 507 result : list[Obs] 508 reconstructed list of observables from the json string 509 or 510 result : Obs 511 only one observable if the list only has one entry 512 or 513 result : dict 514 if full_output=True 515 """ 516 if not fname.endswith('.json') and not fname.endswith('.gz'): 517 fname += '.json' 518 if gz: 519 if not fname.endswith('.gz'): 520 fname += '.gz' 521 with gzip.open(fname, 'r') as fin: 522 d = json.load(fin) 523 else: 524 if fname.endswith('.gz'): 525 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) 526 with open(fname, 'r', encoding='utf-8') as fin: 527 d = json.loads(fin.read()) 528 529 return _parse_json_dict(d, verbose, full_output)
Import a list of Obs or structures containing Obs from a .json(.gz) file.
The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.
Parameters
- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
Returns
- result (list[Obs]): reconstructed list of observables from the json string
- or
- result (Obs): only one observable if the list only has one entry
- or
- result (dict): if full_output=True
612def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): 613 """Export a dict of Obs or structures containing Obs to a .json(.gz) file 614 615 Parameters 616 ---------- 617 od : dict 618 Dict of JSON valid structures and objects that will be exported. 619 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 620 All Obs inside a structure have to be defined on the same set of configurations. 621 fname : str 622 Filename of the output file. 623 description : str 624 Optional string that describes the contents of the json file. 625 indent : int 626 Specify the indentation level of the json file. None or 0 is permissible and 627 saves disk space. 628 reps : str 629 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 630 gz : bool 631 If True, the output is a gzipped json. If False, the output is a json file. 632 633 Returns 634 ------- 635 None 636 """ 637 638 if not isinstance(od, dict): 639 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') 640 641 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' 642 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' 643 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' 644 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') 645 646 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} 647 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) 648 649 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
Export a dict of Obs or structures containing Obs to a .json(.gz) file
Parameters
- od (dict): Dict of JSON valid structures and objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- reps (str): Specify the structure of the placeholder in exported dict to be reps[0-9]+.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.
Returns
- None
715def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): 716 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. 717 718 The following structures are supported: Obs, list, numpy.ndarray, Corr 719 720 Parameters 721 ---------- 722 fname : str 723 Filename of the input file. 724 verbose : bool 725 Print additional information that was written to the file. 726 gz : bool 727 If True, assumes that data is gzipped. If False, assumes JSON file. 728 full_output : bool 729 If True, a dict containing auxiliary information and the data is returned. 730 If False, only the data is returned. 731 reps : str 732 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 733 734 Returns 735 ------- 736 data : Obs / list / Corr 737 Read data 738 or 739 data : dict 740 Read data and meta-data 741 """ 742 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) 743 description = indata['description']['description'] 744 indict = indata['description']['OBSDICT'] 745 ol = indata['obsdata'] 746 od = _od_from_list_and_dict(ol, indict, reps=reps) 747 748 if full_output: 749 indata['description'] = description 750 indata['obsdata'] = od 751 return indata 752 else: 753 return od
Import a dict of Obs or structures containing Obs from a .json(.gz) file.
The following structures are supported: Obs, list, numpy.ndarray, Corr
Parameters
- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
- reps (str): Specify the structure of the placeholder in imported dict to be reps[0-9]+.
Returns
- data (Obs / list / Corr): Read data
- or
- data (dict): Read data and meta-data