pyerrors.input.json
1import rapidjson as json 2import gzip 3import getpass 4import socket 5import datetime 6import platform 7import warnings 8import re 9import numpy as np 10from ..obs import Obs 11from ..covobs import Covobs 12from ..correlators import Corr 13from ..misc import _assert_equal_properties 14from .. import version as pyerrorsversion 15 16 17def create_json_string(ol, description='', indent=1): 18 """Generate the string for the export of a list of Obs or structures containing Obs 19 to a .json(.gz) file 20 21 Parameters 22 ---------- 23 ol : list 24 List of objects that will be exported. At the moment, these objects can be 25 either of: Obs, list, numpy.ndarray, Corr. 26 All Obs inside a structure have to be defined on the same set of configurations. 27 description : str 28 Optional string that describes the contents of the json file. 29 indent : int 30 Specify the indentation level of the json file. None or 0 is permissible and 31 saves disk space. 32 """ 33 34 def _gen_data_d_from_list(ol): 35 dl = [] 36 No = len(ol) 37 for name in ol[0].mc_names: 38 ed = {} 39 ed['id'] = name 40 ed['replica'] = [] 41 for r_name in ol[0].e_content[name]: 42 rd = {} 43 rd['name'] = r_name 44 rd['deltas'] = [] 45 offsets = [o.r_values[r_name] - o.value for o in ol] 46 deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)]) 47 for i in range(len(ol[0].idl[r_name])): 48 rd['deltas'].append([ol[0].idl[r_name][i]]) 49 rd['deltas'][-1] += deltas[i].tolist() 50 ed['replica'].append(rd) 51 dl.append(ed) 52 return dl 53 54 def _gen_cdata_d_from_list(ol): 55 dl = [] 56 for name in ol[0].cov_names: 57 ed = {} 58 ed['id'] = name 59 ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') 60 ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) 61 ncov = ol[0].covobs[name].cov.shape[0] 62 ed['grad'] = [] 63 for i in range(ncov): 64 ed['grad'].append([]) 65 for o in ol: 66 ed['grad'][-1].append(o.covobs[name].grad[i][0]) 67 dl.append(ed) 68 return dl 69 70 def write_Obs_to_dict(o): 71 d = {} 72 d['type'] = 'Obs' 73 d['layout'] = '1' 74 if o.tag: 75 d['tag'] = [o.tag] 76 if o.reweighted: 77 d['reweighted'] = o.reweighted 78 d['value'] = [o.value] 79 data = _gen_data_d_from_list([o]) 80 if len(data) > 0: 81 d['data'] = data 82 cdata = _gen_cdata_d_from_list([o]) 83 if len(cdata) > 0: 84 d['cdata'] = cdata 85 return d 86 87 def write_List_to_dict(ol): 88 _assert_equal_properties(ol) 89 d = {} 90 d['type'] = 'List' 91 d['layout'] = '%d' % len(ol) 92 taglist = [o.tag for o in ol] 93 if np.any([tag is not None for tag in taglist]): 94 d['tag'] = taglist 95 if ol[0].reweighted: 96 d['reweighted'] = ol[0].reweighted 97 d['value'] = [o.value for o in ol] 98 data = _gen_data_d_from_list(ol) 99 if len(data) > 0: 100 d['data'] = data 101 cdata = _gen_cdata_d_from_list(ol) 102 if len(cdata) > 0: 103 d['cdata'] = cdata 104 return d 105 106 def write_Array_to_dict(oa): 107 ol = np.ravel(oa) 108 _assert_equal_properties(ol) 109 d = {} 110 d['type'] = 'Array' 111 d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') 112 taglist = [o.tag for o in ol] 113 if np.any([tag is not None for tag in taglist]): 114 d['tag'] = taglist 115 if ol[0].reweighted: 116 d['reweighted'] = ol[0].reweighted 117 d['value'] = [o.value for o in ol] 118 data = _gen_data_d_from_list(ol) 119 if len(data) > 0: 120 d['data'] = data 121 cdata = _gen_cdata_d_from_list(ol) 122 if len(cdata) > 0: 123 d['cdata'] = cdata 124 return d 125 126 def _nan_Obs_like(obs): 127 samples = [] 128 names = [] 129 idl = [] 130 for key, value in obs.idl.items(): 131 samples.append([np.nan] * len(value)) 132 names.append(key) 133 idl.append(value) 134 my_obs = Obs(samples, names, idl) 135 my_obs._covobs = obs._covobs 136 for name in obs._covobs: 137 my_obs.names.append(name) 138 my_obs.reweighted = obs.reweighted 139 return my_obs 140 141 def write_Corr_to_dict(my_corr): 142 first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j)) 143 dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object) 144 dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0]) 145 content = [o if o is not None else dummy_array for o in my_corr.content] 146 dat = write_Array_to_dict(np.array(content, dtype=object)) 147 dat['type'] = 'Corr' 148 corr_meta_data = str(my_corr.tag) 149 if 'tag' in dat.keys(): 150 dat['tag'].append(corr_meta_data) 151 else: 152 dat['tag'] = [corr_meta_data] 153 taglist = dat['tag'] 154 dat['tag'] = {} # tag is now a dictionary, that contains the previous taglist in the key "tag" 155 dat['tag']['tag'] = taglist 156 if my_corr.prange is not None: 157 dat['tag']['prange'] = my_corr.prange 158 return dat 159 160 if not isinstance(ol, list): 161 ol = [ol] 162 163 d = {} 164 d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__) 165 d['version'] = '1.1' 166 d['who'] = getpass.getuser() 167 d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z') 168 d['host'] = socket.gethostname() + ', ' + platform.platform() 169 170 if description: 171 d['description'] = description 172 173 d['obsdata'] = [] 174 for io in ol: 175 if isinstance(io, Obs): 176 d['obsdata'].append(write_Obs_to_dict(io)) 177 elif isinstance(io, list): 178 d['obsdata'].append(write_List_to_dict(io)) 179 elif isinstance(io, np.ndarray): 180 d['obsdata'].append(write_Array_to_dict(io)) 181 elif isinstance(io, Corr): 182 d['obsdata'].append(write_Corr_to_dict(io)) 183 else: 184 raise Exception("Unkown datatype.") 185 186 def _jsonifier(o): 187 if isinstance(o, np.int64): 188 return int(o) 189 raise TypeError('%r is not JSON serializable' % o) 190 191 if indent: 192 return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY) 193 else: 194 return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT) 195 196 197def dump_to_json(ol, fname, description='', indent=1, gz=True): 198 """Export a list of Obs or structures containing Obs to a .json(.gz) file 199 200 Parameters 201 ---------- 202 ol : list 203 List of objects that will be exported. At the moment, these objects can be 204 either of: Obs, list, numpy.ndarray, Corr. 205 All Obs inside a structure have to be defined on the same set of configurations. 206 fname : str 207 Filename of the output file. 208 description : str 209 Optional string that describes the contents of the json file. 210 indent : int 211 Specify the indentation level of the json file. None or 0 is permissible and 212 saves disk space. 213 gz : bool 214 If True, the output is a gzipped json. If False, the output is a json file. 215 """ 216 217 jsonstring = create_json_string(ol, description, indent) 218 219 if not fname.endswith('.json') and not fname.endswith('.gz'): 220 fname += '.json' 221 222 if gz: 223 if not fname.endswith('.gz'): 224 fname += '.gz' 225 226 fp = gzip.open(fname, 'wb') 227 fp.write(jsonstring.encode('utf-8')) 228 else: 229 fp = open(fname, 'w', encoding='utf-8') 230 fp.write(jsonstring) 231 fp.close() 232 233 234def _parse_json_dict(json_dict, verbose=True, full_output=False): 235 """Reconstruct a list of Obs or structures containing Obs from a dict that 236 was built out of a json string. 237 238 The following structures are supported: Obs, list, numpy.ndarray, Corr 239 If the list contains only one element, it is unpacked from the list. 240 241 Parameters 242 ---------- 243 json_string : str 244 json string containing the data. 245 verbose : bool 246 Print additional information that was written to the file. 247 full_output : bool 248 If True, a dict containing auxiliary information and the data is returned. 249 If False, only the data is returned. 250 """ 251 252 def _gen_obsd_from_datad(d): 253 retd = {} 254 if d: 255 retd['names'] = [] 256 retd['idl'] = [] 257 retd['deltas'] = [] 258 for ens in d: 259 for rep in ens['replica']: 260 rep_name = rep['name'] 261 if len(rep_name) > len(ens["id"]): 262 if rep_name[len(ens["id"])] != "|": 263 tmp_list = list(rep_name) 264 tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):] 265 rep_name = ''.join(tmp_list) 266 retd['names'].append(rep_name) 267 retd['idl'].append([di[0] for di in rep['deltas']]) 268 retd['deltas'].append(np.array([di[1:] for di in rep['deltas']])) 269 return retd 270 271 def _gen_covobsd_from_cdatad(d): 272 retd = {} 273 for ens in d: 274 retl = [] 275 name = ens['id'] 276 layouts = ens.get('layout', '1').strip() 277 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] 278 cov = np.reshape(ens['cov'], layout) 279 grad = ens['grad'] 280 nobs = len(grad[0]) 281 for i in range(nobs): 282 retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]}) 283 retd[name] = retl 284 return retd 285 286 def get_Obs_from_dict(o): 287 layouts = o.get('layout', '1').strip() 288 if layouts != '1': 289 raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning) 290 291 values = o['value'] 292 od = _gen_obsd_from_datad(o.get('data', {})) 293 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 294 295 if od: 296 ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl']) 297 ret._value = values[0] 298 else: 299 ret = Obs([], [], means=[]) 300 ret._value = values[0] 301 for name in cd: 302 co = cd[name][0] 303 ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 304 ret.names.append(co['name']) 305 306 ret.reweighted = o.get('reweighted', False) 307 ret.tag = o.get('tag', [None])[0] 308 return ret 309 310 def get_List_from_dict(o): 311 layouts = o.get('layout', '1').strip() 312 layout = int(layouts) 313 values = o['value'] 314 od = _gen_obsd_from_datad(o.get('data', {})) 315 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 316 317 ret = [] 318 taglist = o.get('tag', layout * [None]) 319 for i in range(layout): 320 if od: 321 ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl'])) 322 ret[-1]._value = values[i] 323 else: 324 ret.append(Obs([], [], means=[])) 325 ret[-1]._value = values[i] 326 print('Created Obs with means= ', values[i]) 327 for name in cd: 328 co = cd[name][i] 329 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 330 ret[-1].names.append(co['name']) 331 332 ret[-1].reweighted = o.get('reweighted', False) 333 ret[-1].tag = taglist[i] 334 return ret 335 336 def get_Array_from_dict(o): 337 layouts = o.get('layout', '1').strip() 338 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] 339 N = np.prod(layout) 340 values = o['value'] 341 od = _gen_obsd_from_datad(o.get('data', {})) 342 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 343 344 ret = [] 345 taglist = o.get('tag', N * [None]) 346 for i in range(N): 347 if od: 348 ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl'])) 349 ret[-1]._value = values[i] 350 else: 351 ret.append(Obs([], [], means=[])) 352 ret[-1]._value = values[i] 353 for name in cd: 354 co = cd[name][i] 355 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 356 ret[-1].names.append(co['name']) 357 ret[-1].reweighted = o.get('reweighted', False) 358 ret[-1].tag = taglist[i] 359 return np.reshape(ret, layout) 360 361 def get_Corr_from_dict(o): 362 if isinstance(o.get('tag'), list): # supports the old way 363 taglist = o.get('tag') # This had to be modified to get the taglist from the dictionary 364 temp_prange = None 365 elif isinstance(o.get('tag'), dict): 366 tagdic = o.get('tag') 367 taglist = tagdic['tag'] 368 if 'prange' in tagdic: 369 temp_prange = tagdic['prange'] 370 else: 371 temp_prange = None 372 else: 373 raise Exception("The tag is not a list or dict") 374 375 corr_tag = taglist[-1] 376 tmp_o = o 377 tmp_o['tag'] = taglist[:-1] 378 if len(tmp_o['tag']) == 0: 379 del tmp_o['tag'] 380 dat = get_Array_from_dict(tmp_o) 381 my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)]) 382 if corr_tag != 'None': 383 my_corr.tag = corr_tag 384 385 my_corr.prange = temp_prange 386 return my_corr 387 388 prog = json_dict.get('program', '') 389 version = json_dict.get('version', '') 390 who = json_dict.get('who', '') 391 date = json_dict.get('date', '') 392 host = json_dict.get('host', '') 393 if prog and verbose: 394 print('Data has been written using %s.' % (prog)) 395 if version and verbose: 396 print('Format version %s' % (version)) 397 if np.any([who, date, host] and verbose): 398 print('Written by %s on %s on host %s' % (who, date, host)) 399 description = json_dict.get('description', '') 400 if description and verbose: 401 print() 402 print('Description: ', description) 403 obsdata = json_dict['obsdata'] 404 ol = [] 405 for io in obsdata: 406 if io['type'] == 'Obs': 407 ol.append(get_Obs_from_dict(io)) 408 elif io['type'] == 'List': 409 ol.append(get_List_from_dict(io)) 410 elif io['type'] == 'Array': 411 ol.append(get_Array_from_dict(io)) 412 elif io['type'] == 'Corr': 413 ol.append(get_Corr_from_dict(io)) 414 else: 415 raise Exception("Unknown datatype.") 416 417 if full_output: 418 retd = {} 419 retd['program'] = prog 420 retd['version'] = version 421 retd['who'] = who 422 retd['date'] = date 423 retd['host'] = host 424 retd['description'] = description 425 retd['obsdata'] = ol 426 427 return retd 428 else: 429 if len(obsdata) == 1: 430 ol = ol[0] 431 432 return ol 433 434 435def import_json_string(json_string, verbose=True, full_output=False): 436 """Reconstruct a list of Obs or structures containing Obs from a json string. 437 438 The following structures are supported: Obs, list, numpy.ndarray, Corr 439 If the list contains only one element, it is unpacked from the list. 440 441 Parameters 442 ---------- 443 json_string : str 444 json string containing the data. 445 verbose : bool 446 Print additional information that was written to the file. 447 full_output : bool 448 If True, a dict containing auxiliary information and the data is returned. 449 If False, only the data is returned. 450 """ 451 452 return _parse_json_dict(json.loads(json_string), verbose, full_output) 453 454 455def load_json(fname, verbose=True, gz=True, full_output=False): 456 """Import a list of Obs or structures containing Obs from a .json(.gz) file. 457 458 The following structures are supported: Obs, list, numpy.ndarray, Corr 459 If the list contains only one element, it is unpacked from the list. 460 461 Parameters 462 ---------- 463 fname : str 464 Filename of the input file. 465 verbose : bool 466 Print additional information that was written to the file. 467 gz : bool 468 If True, assumes that data is gzipped. If False, assumes JSON file. 469 full_output : bool 470 If True, a dict containing auxiliary information and the data is returned. 471 If False, only the data is returned. 472 """ 473 if not fname.endswith('.json') and not fname.endswith('.gz'): 474 fname += '.json' 475 if gz: 476 if not fname.endswith('.gz'): 477 fname += '.gz' 478 with gzip.open(fname, 'r') as fin: 479 d = json.load(fin) 480 else: 481 if fname.endswith('.gz'): 482 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) 483 with open(fname, 'r', encoding='utf-8') as fin: 484 d = json.loads(fin.read()) 485 486 return _parse_json_dict(d, verbose, full_output) 487 488 489def _ol_from_dict(ind, reps='DICTOBS'): 490 """Convert a dictionary of Obs objects to a list and a dictionary that contains 491 placeholders instead of the Obs objects. 492 493 Parameters 494 ---------- 495 ind : dict 496 Dict of JSON valid structures and objects that will be exported. 497 At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr. 498 All Obs inside a structure have to be defined on the same set of configurations. 499 reps : str 500 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 501 """ 502 503 obstypes = (Obs, Corr, np.ndarray) 504 505 if not reps.isalnum(): 506 raise Exception('Placeholder string has to be alphanumeric!') 507 ol = [] 508 counter = 0 509 510 def dict_replace_obs(d): 511 nonlocal ol 512 nonlocal counter 513 x = {} 514 for k, v in d.items(): 515 if isinstance(v, dict): 516 v = dict_replace_obs(v) 517 elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]): 518 v = obslist_replace_obs(v) 519 elif isinstance(v, list): 520 v = list_replace_obs(v) 521 elif isinstance(v, obstypes): 522 ol.append(v) 523 v = reps + '%d' % (counter) 524 counter += 1 525 elif isinstance(v, str): 526 if bool(re.match(r'%s[0-9]+' % (reps), v)): 527 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps)) 528 x[k] = v 529 return x 530 531 def list_replace_obs(li): 532 nonlocal ol 533 nonlocal counter 534 x = [] 535 for e in li: 536 if isinstance(e, list): 537 e = list_replace_obs(e) 538 elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]): 539 e = obslist_replace_obs(e) 540 elif isinstance(e, dict): 541 e = dict_replace_obs(e) 542 elif isinstance(e, obstypes): 543 ol.append(e) 544 e = reps + '%d' % (counter) 545 counter += 1 546 elif isinstance(e, str): 547 if bool(re.match(r'%s[0-9]+' % (reps), e)): 548 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps)) 549 x.append(e) 550 return x 551 552 def obslist_replace_obs(li): 553 nonlocal ol 554 nonlocal counter 555 il = [] 556 for e in li: 557 il.append(e) 558 559 ol.append(il) 560 x = reps + '%d' % (counter) 561 counter += 1 562 return x 563 564 nd = dict_replace_obs(ind) 565 566 return ol, nd 567 568 569def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): 570 """Export a dict of Obs or structures containing Obs to a .json(.gz) file 571 572 Parameters 573 ---------- 574 od : dict 575 Dict of JSON valid structures and objects that will be exported. 576 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 577 All Obs inside a structure have to be defined on the same set of configurations. 578 fname : str 579 Filename of the output file. 580 description : str 581 Optional string that describes the contents of the json file. 582 indent : int 583 Specify the indentation level of the json file. None or 0 is permissible and 584 saves disk space. 585 reps : str 586 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 587 gz : bool 588 If True, the output is a gzipped json. If False, the output is a json file. 589 """ 590 591 if not isinstance(od, dict): 592 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') 593 594 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' 595 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' 596 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' 597 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') 598 599 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} 600 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) 601 602 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz) 603 604 605def _od_from_list_and_dict(ol, ind, reps='DICTOBS'): 606 """Parse a list of Obs or structures containing Obs and an accompanying 607 dict, where the structures have been replaced by placeholders to a 608 dict that contains the structures. 609 610 The following structures are supported: Obs, list, numpy.ndarray, Corr 611 612 Parameters 613 ---------- 614 ol : list 615 List of objects - 616 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 617 All Obs inside a structure have to be defined on the same set of configurations. 618 ind : dict 619 Dict that defines the structure of the resulting dict and contains placeholders 620 reps : str 621 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 622 """ 623 if not reps.isalnum(): 624 raise Exception('Placeholder string has to be alphanumeric!') 625 626 counter = 0 627 628 def dict_replace_string(d): 629 nonlocal counter 630 nonlocal ol 631 x = {} 632 for k, v in d.items(): 633 if isinstance(v, dict): 634 v = dict_replace_string(v) 635 elif isinstance(v, list): 636 v = list_replace_string(v) 637 elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)): 638 index = int(v[len(reps):]) 639 v = ol[index] 640 counter += 1 641 x[k] = v 642 return x 643 644 def list_replace_string(li): 645 nonlocal counter 646 nonlocal ol 647 x = [] 648 for e in li: 649 if isinstance(e, list): 650 e = list_replace_string(e) 651 elif isinstance(e, dict): 652 e = dict_replace_string(e) 653 elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)): 654 index = int(e[len(reps):]) 655 e = ol[index] 656 counter += 1 657 x.append(e) 658 return x 659 660 nd = dict_replace_string(ind) 661 662 if counter == 0: 663 raise Exception('No placeholder has been replaced! Check if reps is set correctly.') 664 665 return nd 666 667 668def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): 669 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. 670 671 The following structures are supported: Obs, list, numpy.ndarray, Corr 672 673 Parameters 674 ---------- 675 fname : str 676 Filename of the input file. 677 verbose : bool 678 Print additional information that was written to the file. 679 gz : bool 680 If True, assumes that data is gzipped. If False, assumes JSON file. 681 full_output : bool 682 If True, a dict containing auxiliary information and the data is returned. 683 If False, only the data is returned. 684 reps : str 685 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 686 """ 687 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) 688 description = indata['description']['description'] 689 indict = indata['description']['OBSDICT'] 690 ol = indata['obsdata'] 691 od = _od_from_list_and_dict(ol, indict, reps=reps) 692 693 if full_output: 694 indata['description'] = description 695 indata['obsdata'] = od 696 return indata 697 else: 698 return od
18def create_json_string(ol, description='', indent=1): 19 """Generate the string for the export of a list of Obs or structures containing Obs 20 to a .json(.gz) file 21 22 Parameters 23 ---------- 24 ol : list 25 List of objects that will be exported. At the moment, these objects can be 26 either of: Obs, list, numpy.ndarray, Corr. 27 All Obs inside a structure have to be defined on the same set of configurations. 28 description : str 29 Optional string that describes the contents of the json file. 30 indent : int 31 Specify the indentation level of the json file. None or 0 is permissible and 32 saves disk space. 33 """ 34 35 def _gen_data_d_from_list(ol): 36 dl = [] 37 No = len(ol) 38 for name in ol[0].mc_names: 39 ed = {} 40 ed['id'] = name 41 ed['replica'] = [] 42 for r_name in ol[0].e_content[name]: 43 rd = {} 44 rd['name'] = r_name 45 rd['deltas'] = [] 46 offsets = [o.r_values[r_name] - o.value for o in ol] 47 deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)]) 48 for i in range(len(ol[0].idl[r_name])): 49 rd['deltas'].append([ol[0].idl[r_name][i]]) 50 rd['deltas'][-1] += deltas[i].tolist() 51 ed['replica'].append(rd) 52 dl.append(ed) 53 return dl 54 55 def _gen_cdata_d_from_list(ol): 56 dl = [] 57 for name in ol[0].cov_names: 58 ed = {} 59 ed['id'] = name 60 ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') 61 ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) 62 ncov = ol[0].covobs[name].cov.shape[0] 63 ed['grad'] = [] 64 for i in range(ncov): 65 ed['grad'].append([]) 66 for o in ol: 67 ed['grad'][-1].append(o.covobs[name].grad[i][0]) 68 dl.append(ed) 69 return dl 70 71 def write_Obs_to_dict(o): 72 d = {} 73 d['type'] = 'Obs' 74 d['layout'] = '1' 75 if o.tag: 76 d['tag'] = [o.tag] 77 if o.reweighted: 78 d['reweighted'] = o.reweighted 79 d['value'] = [o.value] 80 data = _gen_data_d_from_list([o]) 81 if len(data) > 0: 82 d['data'] = data 83 cdata = _gen_cdata_d_from_list([o]) 84 if len(cdata) > 0: 85 d['cdata'] = cdata 86 return d 87 88 def write_List_to_dict(ol): 89 _assert_equal_properties(ol) 90 d = {} 91 d['type'] = 'List' 92 d['layout'] = '%d' % len(ol) 93 taglist = [o.tag for o in ol] 94 if np.any([tag is not None for tag in taglist]): 95 d['tag'] = taglist 96 if ol[0].reweighted: 97 d['reweighted'] = ol[0].reweighted 98 d['value'] = [o.value for o in ol] 99 data = _gen_data_d_from_list(ol) 100 if len(data) > 0: 101 d['data'] = data 102 cdata = _gen_cdata_d_from_list(ol) 103 if len(cdata) > 0: 104 d['cdata'] = cdata 105 return d 106 107 def write_Array_to_dict(oa): 108 ol = np.ravel(oa) 109 _assert_equal_properties(ol) 110 d = {} 111 d['type'] = 'Array' 112 d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') 113 taglist = [o.tag for o in ol] 114 if np.any([tag is not None for tag in taglist]): 115 d['tag'] = taglist 116 if ol[0].reweighted: 117 d['reweighted'] = ol[0].reweighted 118 d['value'] = [o.value for o in ol] 119 data = _gen_data_d_from_list(ol) 120 if len(data) > 0: 121 d['data'] = data 122 cdata = _gen_cdata_d_from_list(ol) 123 if len(cdata) > 0: 124 d['cdata'] = cdata 125 return d 126 127 def _nan_Obs_like(obs): 128 samples = [] 129 names = [] 130 idl = [] 131 for key, value in obs.idl.items(): 132 samples.append([np.nan] * len(value)) 133 names.append(key) 134 idl.append(value) 135 my_obs = Obs(samples, names, idl) 136 my_obs._covobs = obs._covobs 137 for name in obs._covobs: 138 my_obs.names.append(name) 139 my_obs.reweighted = obs.reweighted 140 return my_obs 141 142 def write_Corr_to_dict(my_corr): 143 first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j)) 144 dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object) 145 dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0]) 146 content = [o if o is not None else dummy_array for o in my_corr.content] 147 dat = write_Array_to_dict(np.array(content, dtype=object)) 148 dat['type'] = 'Corr' 149 corr_meta_data = str(my_corr.tag) 150 if 'tag' in dat.keys(): 151 dat['tag'].append(corr_meta_data) 152 else: 153 dat['tag'] = [corr_meta_data] 154 taglist = dat['tag'] 155 dat['tag'] = {} # tag is now a dictionary, that contains the previous taglist in the key "tag" 156 dat['tag']['tag'] = taglist 157 if my_corr.prange is not None: 158 dat['tag']['prange'] = my_corr.prange 159 return dat 160 161 if not isinstance(ol, list): 162 ol = [ol] 163 164 d = {} 165 d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__) 166 d['version'] = '1.1' 167 d['who'] = getpass.getuser() 168 d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z') 169 d['host'] = socket.gethostname() + ', ' + platform.platform() 170 171 if description: 172 d['description'] = description 173 174 d['obsdata'] = [] 175 for io in ol: 176 if isinstance(io, Obs): 177 d['obsdata'].append(write_Obs_to_dict(io)) 178 elif isinstance(io, list): 179 d['obsdata'].append(write_List_to_dict(io)) 180 elif isinstance(io, np.ndarray): 181 d['obsdata'].append(write_Array_to_dict(io)) 182 elif isinstance(io, Corr): 183 d['obsdata'].append(write_Corr_to_dict(io)) 184 else: 185 raise Exception("Unkown datatype.") 186 187 def _jsonifier(o): 188 if isinstance(o, np.int64): 189 return int(o) 190 raise TypeError('%r is not JSON serializable' % o) 191 192 if indent: 193 return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY) 194 else: 195 return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)
Generate the string for the export of a list of Obs or structures containing Obs to a .json(.gz) file
Parameters
- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
198def dump_to_json(ol, fname, description='', indent=1, gz=True): 199 """Export a list of Obs or structures containing Obs to a .json(.gz) file 200 201 Parameters 202 ---------- 203 ol : list 204 List of objects that will be exported. At the moment, these objects can be 205 either of: Obs, list, numpy.ndarray, Corr. 206 All Obs inside a structure have to be defined on the same set of configurations. 207 fname : str 208 Filename of the output file. 209 description : str 210 Optional string that describes the contents of the json file. 211 indent : int 212 Specify the indentation level of the json file. None or 0 is permissible and 213 saves disk space. 214 gz : bool 215 If True, the output is a gzipped json. If False, the output is a json file. 216 """ 217 218 jsonstring = create_json_string(ol, description, indent) 219 220 if not fname.endswith('.json') and not fname.endswith('.gz'): 221 fname += '.json' 222 223 if gz: 224 if not fname.endswith('.gz'): 225 fname += '.gz' 226 227 fp = gzip.open(fname, 'wb') 228 fp.write(jsonstring.encode('utf-8')) 229 else: 230 fp = open(fname, 'w', encoding='utf-8') 231 fp.write(jsonstring) 232 fp.close()
Export a list of Obs or structures containing Obs to a .json(.gz) file
Parameters
- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.
436def import_json_string(json_string, verbose=True, full_output=False): 437 """Reconstruct a list of Obs or structures containing Obs from a json string. 438 439 The following structures are supported: Obs, list, numpy.ndarray, Corr 440 If the list contains only one element, it is unpacked from the list. 441 442 Parameters 443 ---------- 444 json_string : str 445 json string containing the data. 446 verbose : bool 447 Print additional information that was written to the file. 448 full_output : bool 449 If True, a dict containing auxiliary information and the data is returned. 450 If False, only the data is returned. 451 """ 452 453 return _parse_json_dict(json.loads(json_string), verbose, full_output)
Reconstruct a list of Obs or structures containing Obs from a json string.
The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.
Parameters
- json_string (str): json string containing the data.
- verbose (bool): Print additional information that was written to the file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
456def load_json(fname, verbose=True, gz=True, full_output=False): 457 """Import a list of Obs or structures containing Obs from a .json(.gz) file. 458 459 The following structures are supported: Obs, list, numpy.ndarray, Corr 460 If the list contains only one element, it is unpacked from the list. 461 462 Parameters 463 ---------- 464 fname : str 465 Filename of the input file. 466 verbose : bool 467 Print additional information that was written to the file. 468 gz : bool 469 If True, assumes that data is gzipped. If False, assumes JSON file. 470 full_output : bool 471 If True, a dict containing auxiliary information and the data is returned. 472 If False, only the data is returned. 473 """ 474 if not fname.endswith('.json') and not fname.endswith('.gz'): 475 fname += '.json' 476 if gz: 477 if not fname.endswith('.gz'): 478 fname += '.gz' 479 with gzip.open(fname, 'r') as fin: 480 d = json.load(fin) 481 else: 482 if fname.endswith('.gz'): 483 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) 484 with open(fname, 'r', encoding='utf-8') as fin: 485 d = json.loads(fin.read()) 486 487 return _parse_json_dict(d, verbose, full_output)
Import a list of Obs or structures containing Obs from a .json(.gz) file.
The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.
Parameters
- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
570def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): 571 """Export a dict of Obs or structures containing Obs to a .json(.gz) file 572 573 Parameters 574 ---------- 575 od : dict 576 Dict of JSON valid structures and objects that will be exported. 577 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 578 All Obs inside a structure have to be defined on the same set of configurations. 579 fname : str 580 Filename of the output file. 581 description : str 582 Optional string that describes the contents of the json file. 583 indent : int 584 Specify the indentation level of the json file. None or 0 is permissible and 585 saves disk space. 586 reps : str 587 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 588 gz : bool 589 If True, the output is a gzipped json. If False, the output is a json file. 590 """ 591 592 if not isinstance(od, dict): 593 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') 594 595 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' 596 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' 597 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' 598 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') 599 600 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} 601 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) 602 603 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
Export a dict of Obs or structures containing Obs to a .json(.gz) file
Parameters
- od (dict): Dict of JSON valid structures and objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- reps (str): Specify the structure of the placeholder in exported dict to be reps[0-9]+.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.
669def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): 670 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. 671 672 The following structures are supported: Obs, list, numpy.ndarray, Corr 673 674 Parameters 675 ---------- 676 fname : str 677 Filename of the input file. 678 verbose : bool 679 Print additional information that was written to the file. 680 gz : bool 681 If True, assumes that data is gzipped. If False, assumes JSON file. 682 full_output : bool 683 If True, a dict containing auxiliary information and the data is returned. 684 If False, only the data is returned. 685 reps : str 686 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 687 """ 688 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) 689 description = indata['description']['description'] 690 indict = indata['description']['OBSDICT'] 691 ol = indata['obsdata'] 692 od = _od_from_list_and_dict(ol, indict, reps=reps) 693 694 if full_output: 695 indata['description'] = description 696 indata['obsdata'] = od 697 return indata 698 else: 699 return od
Import a dict of Obs or structures containing Obs from a .json(.gz) file.
The following structures are supported: Obs, list, numpy.ndarray, Corr
Parameters
- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
- reps (str): Specify the structure of the placeholder in imported dict to be reps[0-9]+.