pyerrors.input.json
1import rapidjson as json 2import gzip 3import getpass 4import socket 5import datetime 6import platform 7import warnings 8import re 9import numpy as np 10from ..obs import Obs 11from ..covobs import Covobs 12from ..correlators import Corr 13from ..misc import _assert_equal_properties 14from .. import version as pyerrorsversion 15 16 17def create_json_string(ol, description='', indent=1): 18 """Generate the string for the export of a list of Obs or structures containing Obs 19 to a .json(.gz) file 20 21 Parameters 22 ---------- 23 ol : list 24 List of objects that will be exported. At the moment, these objects can be 25 either of: Obs, list, numpy.ndarray, Corr. 26 All Obs inside a structure have to be defined on the same set of configurations. 27 description : str 28 Optional string that describes the contents of the json file. 29 indent : int 30 Specify the indentation level of the json file. None or 0 is permissible and 31 saves disk space. 32 """ 33 34 def _gen_data_d_from_list(ol): 35 dl = [] 36 No = len(ol) 37 for name in ol[0].mc_names: 38 ed = {} 39 ed['id'] = name 40 ed['replica'] = [] 41 for r_name in ol[0].e_content[name]: 42 rd = {} 43 rd['name'] = r_name 44 if ol[0].is_merged.get(r_name, False): 45 rd['is_merged'] = True 46 rd['deltas'] = [] 47 offsets = [o.r_values[r_name] - o.value for o in ol] 48 deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)]) 49 for i in range(len(ol[0].idl[r_name])): 50 rd['deltas'].append([ol[0].idl[r_name][i]]) 51 rd['deltas'][-1] += deltas[i].tolist() 52 ed['replica'].append(rd) 53 dl.append(ed) 54 return dl 55 56 def _gen_cdata_d_from_list(ol): 57 dl = [] 58 for name in ol[0].cov_names: 59 ed = {} 60 ed['id'] = name 61 ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') 62 ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) 63 ncov = ol[0].covobs[name].cov.shape[0] 64 ed['grad'] = [] 65 for i in range(ncov): 66 ed['grad'].append([]) 67 for o in ol: 68 ed['grad'][-1].append(o.covobs[name].grad[i][0]) 69 dl.append(ed) 70 return dl 71 72 def write_Obs_to_dict(o): 73 d = {} 74 d['type'] = 'Obs' 75 d['layout'] = '1' 76 if o.tag: 77 d['tag'] = [o.tag] 78 if o.reweighted: 79 d['reweighted'] = o.reweighted 80 d['value'] = [o.value] 81 data = _gen_data_d_from_list([o]) 82 if len(data) > 0: 83 d['data'] = data 84 cdata = _gen_cdata_d_from_list([o]) 85 if len(cdata) > 0: 86 d['cdata'] = cdata 87 return d 88 89 def write_List_to_dict(ol): 90 _assert_equal_properties(ol) 91 d = {} 92 d['type'] = 'List' 93 d['layout'] = '%d' % len(ol) 94 taglist = [o.tag for o in ol] 95 if np.any([tag is not None for tag in taglist]): 96 d['tag'] = taglist 97 if ol[0].reweighted: 98 d['reweighted'] = ol[0].reweighted 99 d['value'] = [o.value for o in ol] 100 data = _gen_data_d_from_list(ol) 101 if len(data) > 0: 102 d['data'] = data 103 cdata = _gen_cdata_d_from_list(ol) 104 if len(cdata) > 0: 105 d['cdata'] = cdata 106 return d 107 108 def write_Array_to_dict(oa): 109 ol = np.ravel(oa) 110 _assert_equal_properties(ol) 111 d = {} 112 d['type'] = 'Array' 113 d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') 114 taglist = [o.tag for o in ol] 115 if np.any([tag is not None for tag in taglist]): 116 d['tag'] = taglist 117 if ol[0].reweighted: 118 d['reweighted'] = ol[0].reweighted 119 d['value'] = [o.value for o in ol] 120 data = _gen_data_d_from_list(ol) 121 if len(data) > 0: 122 d['data'] = data 123 cdata = _gen_cdata_d_from_list(ol) 124 if len(cdata) > 0: 125 d['cdata'] = cdata 126 return d 127 128 def _nan_Obs_like(obs): 129 samples = [] 130 names = [] 131 idl = [] 132 for key, value in obs.idl.items(): 133 samples.append([np.nan] * len(value)) 134 names.append(key) 135 idl.append(value) 136 my_obs = Obs(samples, names, idl) 137 my_obs._covobs = obs._covobs 138 for name in obs._covobs: 139 my_obs.names.append(name) 140 my_obs.reweighted = obs.reweighted 141 my_obs.is_merged = obs.is_merged 142 return my_obs 143 144 def write_Corr_to_dict(my_corr): 145 first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j)) 146 dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object) 147 dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0]) 148 content = [o if o is not None else dummy_array for o in my_corr.content] 149 dat = write_Array_to_dict(np.array(content, dtype=object)) 150 dat['type'] = 'Corr' 151 corr_meta_data = str(my_corr.tag) 152 if 'tag' in dat.keys(): 153 dat['tag'].append(corr_meta_data) 154 else: 155 dat['tag'] = [corr_meta_data] 156 taglist = dat['tag'] 157 dat['tag'] = {} # tag is now a dictionary, that contains the previous taglist in the key "tag" 158 dat['tag']['tag'] = taglist 159 if my_corr.prange is not None: 160 dat['tag']['prange'] = my_corr.prange 161 return dat 162 163 if not isinstance(ol, list): 164 ol = [ol] 165 166 d = {} 167 d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__) 168 d['version'] = '1.1' 169 d['who'] = getpass.getuser() 170 d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z') 171 d['host'] = socket.gethostname() + ', ' + platform.platform() 172 173 if description: 174 d['description'] = description 175 176 d['obsdata'] = [] 177 for io in ol: 178 if isinstance(io, Obs): 179 d['obsdata'].append(write_Obs_to_dict(io)) 180 elif isinstance(io, list): 181 d['obsdata'].append(write_List_to_dict(io)) 182 elif isinstance(io, np.ndarray): 183 d['obsdata'].append(write_Array_to_dict(io)) 184 elif isinstance(io, Corr): 185 d['obsdata'].append(write_Corr_to_dict(io)) 186 else: 187 raise Exception("Unkown datatype.") 188 189 if indent: 190 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY) 191 else: 192 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT) 193 194 195def dump_to_json(ol, fname, description='', indent=1, gz=True): 196 """Export a list of Obs or structures containing Obs to a .json(.gz) file 197 198 Parameters 199 ---------- 200 ol : list 201 List of objects that will be exported. At the moment, these objects can be 202 either of: Obs, list, numpy.ndarray, Corr. 203 All Obs inside a structure have to be defined on the same set of configurations. 204 fname : str 205 Filename of the output file. 206 description : str 207 Optional string that describes the contents of the json file. 208 indent : int 209 Specify the indentation level of the json file. None or 0 is permissible and 210 saves disk space. 211 gz : bool 212 If True, the output is a gzipped json. If False, the output is a json file. 213 """ 214 215 jsonstring = create_json_string(ol, description, indent) 216 217 if not fname.endswith('.json') and not fname.endswith('.gz'): 218 fname += '.json' 219 220 if gz: 221 if not fname.endswith('.gz'): 222 fname += '.gz' 223 224 fp = gzip.open(fname, 'wb') 225 fp.write(jsonstring.encode('utf-8')) 226 else: 227 fp = open(fname, 'w', encoding='utf-8') 228 fp.write(jsonstring) 229 fp.close() 230 231 232def _parse_json_dict(json_dict, verbose=True, full_output=False): 233 """Reconstruct a list of Obs or structures containing Obs from a dict that 234 was built out of a json string. 235 236 The following structures are supported: Obs, list, numpy.ndarray, Corr 237 If the list contains only one element, it is unpacked from the list. 238 239 Parameters 240 ---------- 241 json_string : str 242 json string containing the data. 243 verbose : bool 244 Print additional information that was written to the file. 245 full_output : bool 246 If True, a dict containing auxiliary information and the data is returned. 247 If False, only the data is returned. 248 """ 249 250 def _gen_obsd_from_datad(d): 251 retd = {} 252 if d: 253 retd['names'] = [] 254 retd['idl'] = [] 255 retd['deltas'] = [] 256 retd['is_merged'] = {} 257 for ens in d: 258 for rep in ens['replica']: 259 rep_name = rep['name'] 260 if len(rep_name) > len(ens["id"]): 261 if rep_name[len(ens["id"])] != "|": 262 tmp_list = list(rep_name) 263 tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):] 264 rep_name = ''.join(tmp_list) 265 retd['names'].append(rep_name) 266 retd['idl'].append([di[0] for di in rep['deltas']]) 267 retd['deltas'].append(np.array([di[1:] for di in rep['deltas']])) 268 retd['is_merged'][rep_name] = rep.get('is_merged', False) 269 return retd 270 271 def _gen_covobsd_from_cdatad(d): 272 retd = {} 273 for ens in d: 274 retl = [] 275 name = ens['id'] 276 layouts = ens.get('layout', '1').strip() 277 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] 278 cov = np.reshape(ens['cov'], layout) 279 grad = ens['grad'] 280 nobs = len(grad[0]) 281 for i in range(nobs): 282 retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]}) 283 retd[name] = retl 284 return retd 285 286 def get_Obs_from_dict(o): 287 layouts = o.get('layout', '1').strip() 288 if layouts != '1': 289 raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning) 290 291 values = o['value'] 292 od = _gen_obsd_from_datad(o.get('data', {})) 293 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 294 295 if od: 296 ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl']) 297 ret._value = values[0] 298 ret.is_merged = od['is_merged'] 299 else: 300 ret = Obs([], [], means=[]) 301 ret._value = values[0] 302 for name in cd: 303 co = cd[name][0] 304 ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 305 ret.names.append(co['name']) 306 307 ret.reweighted = o.get('reweighted', False) 308 ret.tag = o.get('tag', [None])[0] 309 return ret 310 311 def get_List_from_dict(o): 312 layouts = o.get('layout', '1').strip() 313 layout = int(layouts) 314 values = o['value'] 315 od = _gen_obsd_from_datad(o.get('data', {})) 316 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 317 318 ret = [] 319 taglist = o.get('tag', layout * [None]) 320 for i in range(layout): 321 if od: 322 ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl'])) 323 ret[-1]._value = values[i] 324 ret[-1].is_merged = od['is_merged'] 325 else: 326 ret.append(Obs([], [], means=[])) 327 ret[-1]._value = values[i] 328 print('Created Obs with means= ', values[i]) 329 for name in cd: 330 co = cd[name][i] 331 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 332 ret[-1].names.append(co['name']) 333 334 ret[-1].reweighted = o.get('reweighted', False) 335 ret[-1].tag = taglist[i] 336 return ret 337 338 def get_Array_from_dict(o): 339 layouts = o.get('layout', '1').strip() 340 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] 341 N = np.prod(layout) 342 values = o['value'] 343 od = _gen_obsd_from_datad(o.get('data', {})) 344 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 345 346 ret = [] 347 taglist = o.get('tag', N * [None]) 348 for i in range(N): 349 if od: 350 ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl'])) 351 ret[-1]._value = values[i] 352 ret[-1].is_merged = od['is_merged'] 353 else: 354 ret.append(Obs([], [], means=[])) 355 ret[-1]._value = values[i] 356 for name in cd: 357 co = cd[name][i] 358 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 359 ret[-1].names.append(co['name']) 360 ret[-1].reweighted = o.get('reweighted', False) 361 ret[-1].tag = taglist[i] 362 return np.reshape(ret, layout) 363 364 def get_Corr_from_dict(o): 365 if isinstance(o.get('tag'), list): # supports the old way 366 taglist = o.get('tag') # This had to be modified to get the taglist from the dictionary 367 temp_prange = None 368 elif isinstance(o.get('tag'), dict): 369 tagdic = o.get('tag') 370 taglist = tagdic['tag'] 371 if 'prange' in tagdic: 372 temp_prange = tagdic['prange'] 373 else: 374 temp_prange = None 375 else: 376 raise Exception("The tag is not a list or dict") 377 378 corr_tag = taglist[-1] 379 tmp_o = o 380 tmp_o['tag'] = taglist[:-1] 381 if len(tmp_o['tag']) == 0: 382 del tmp_o['tag'] 383 dat = get_Array_from_dict(tmp_o) 384 my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)]) 385 if corr_tag != 'None': 386 my_corr.tag = corr_tag 387 388 my_corr.prange = temp_prange 389 return my_corr 390 391 prog = json_dict.get('program', '') 392 version = json_dict.get('version', '') 393 who = json_dict.get('who', '') 394 date = json_dict.get('date', '') 395 host = json_dict.get('host', '') 396 if prog and verbose: 397 print('Data has been written using %s.' % (prog)) 398 if version and verbose: 399 print('Format version %s' % (version)) 400 if np.any([who, date, host] and verbose): 401 print('Written by %s on %s on host %s' % (who, date, host)) 402 description = json_dict.get('description', '') 403 if description and verbose: 404 print() 405 print('Description: ', description) 406 obsdata = json_dict['obsdata'] 407 ol = [] 408 for io in obsdata: 409 if io['type'] == 'Obs': 410 ol.append(get_Obs_from_dict(io)) 411 elif io['type'] == 'List': 412 ol.append(get_List_from_dict(io)) 413 elif io['type'] == 'Array': 414 ol.append(get_Array_from_dict(io)) 415 elif io['type'] == 'Corr': 416 ol.append(get_Corr_from_dict(io)) 417 else: 418 raise Exception("Unkown datatype.") 419 420 if full_output: 421 retd = {} 422 retd['program'] = prog 423 retd['version'] = version 424 retd['who'] = who 425 retd['date'] = date 426 retd['host'] = host 427 retd['description'] = description 428 retd['obsdata'] = ol 429 430 return retd 431 else: 432 if len(obsdata) == 1: 433 ol = ol[0] 434 435 return ol 436 437 438def import_json_string(json_string, verbose=True, full_output=False): 439 """Reconstruct a list of Obs or structures containing Obs from a json string. 440 441 The following structures are supported: Obs, list, numpy.ndarray, Corr 442 If the list contains only one element, it is unpacked from the list. 443 444 Parameters 445 ---------- 446 json_string : str 447 json string containing the data. 448 verbose : bool 449 Print additional information that was written to the file. 450 full_output : bool 451 If True, a dict containing auxiliary information and the data is returned. 452 If False, only the data is returned. 453 """ 454 455 return _parse_json_dict(json.loads(json_string), verbose, full_output) 456 457 458def load_json(fname, verbose=True, gz=True, full_output=False): 459 """Import a list of Obs or structures containing Obs from a .json(.gz) file. 460 461 The following structures are supported: Obs, list, numpy.ndarray, Corr 462 If the list contains only one element, it is unpacked from the list. 463 464 Parameters 465 ---------- 466 fname : str 467 Filename of the input file. 468 verbose : bool 469 Print additional information that was written to the file. 470 gz : bool 471 If True, assumes that data is gzipped. If False, assumes JSON file. 472 full_output : bool 473 If True, a dict containing auxiliary information and the data is returned. 474 If False, only the data is returned. 475 """ 476 if not fname.endswith('.json') and not fname.endswith('.gz'): 477 fname += '.json' 478 if gz: 479 if not fname.endswith('.gz'): 480 fname += '.gz' 481 with gzip.open(fname, 'r') as fin: 482 d = json.load(fin) 483 else: 484 if fname.endswith('.gz'): 485 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) 486 with open(fname, 'r', encoding='utf-8') as fin: 487 d = json.loads(fin.read()) 488 489 return _parse_json_dict(d, verbose, full_output) 490 491 492def _ol_from_dict(ind, reps='DICTOBS'): 493 """Convert a dictionary of Obs objects to a list and a dictionary that contains 494 placeholders instead of the Obs objects. 495 496 Parameters 497 ---------- 498 ind : dict 499 Dict of JSON valid structures and objects that will be exported. 500 At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr. 501 All Obs inside a structure have to be defined on the same set of configurations. 502 reps : str 503 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 504 """ 505 506 obstypes = (Obs, Corr, np.ndarray) 507 508 if not reps.isalnum(): 509 raise Exception('Placeholder string has to be alphanumeric!') 510 ol = [] 511 counter = 0 512 513 def dict_replace_obs(d): 514 nonlocal ol 515 nonlocal counter 516 x = {} 517 for k, v in d.items(): 518 if isinstance(v, dict): 519 v = dict_replace_obs(v) 520 elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]): 521 v = obslist_replace_obs(v) 522 elif isinstance(v, list): 523 v = list_replace_obs(v) 524 elif isinstance(v, obstypes): 525 ol.append(v) 526 v = reps + '%d' % (counter) 527 counter += 1 528 elif isinstance(v, str): 529 if bool(re.match(r'%s[0-9]+' % (reps), v)): 530 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps)) 531 x[k] = v 532 return x 533 534 def list_replace_obs(li): 535 nonlocal ol 536 nonlocal counter 537 x = [] 538 for e in li: 539 if isinstance(e, list): 540 e = list_replace_obs(e) 541 elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]): 542 e = obslist_replace_obs(e) 543 elif isinstance(e, dict): 544 e = dict_replace_obs(e) 545 elif isinstance(e, obstypes): 546 ol.append(e) 547 e = reps + '%d' % (counter) 548 counter += 1 549 elif isinstance(e, str): 550 if bool(re.match(r'%s[0-9]+' % (reps), e)): 551 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps)) 552 x.append(e) 553 return x 554 555 def obslist_replace_obs(li): 556 nonlocal ol 557 nonlocal counter 558 il = [] 559 for e in li: 560 il.append(e) 561 562 ol.append(il) 563 x = reps + '%d' % (counter) 564 counter += 1 565 return x 566 567 nd = dict_replace_obs(ind) 568 569 return ol, nd 570 571 572def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): 573 """Export a dict of Obs or structures containing Obs to a .json(.gz) file 574 575 Parameters 576 ---------- 577 od : dict 578 Dict of JSON valid structures and objects that will be exported. 579 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 580 All Obs inside a structure have to be defined on the same set of configurations. 581 fname : str 582 Filename of the output file. 583 description : str 584 Optional string that describes the contents of the json file. 585 indent : int 586 Specify the indentation level of the json file. None or 0 is permissible and 587 saves disk space. 588 reps : str 589 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 590 gz : bool 591 If True, the output is a gzipped json. If False, the output is a json file. 592 """ 593 594 if not isinstance(od, dict): 595 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') 596 597 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' 598 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' 599 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' 600 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') 601 602 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} 603 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) 604 605 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz) 606 607 608def _od_from_list_and_dict(ol, ind, reps='DICTOBS'): 609 """Parse a list of Obs or structures containing Obs and an accompanying 610 dict, where the structures have been replaced by placeholders to a 611 dict that contains the structures. 612 613 The following structures are supported: Obs, list, numpy.ndarray, Corr 614 615 Parameters 616 ---------- 617 ol : list 618 List of objects - 619 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 620 All Obs inside a structure have to be defined on the same set of configurations. 621 ind : dict 622 Dict that defines the structure of the resulting dict and contains placeholders 623 reps : str 624 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 625 """ 626 if not reps.isalnum(): 627 raise Exception('Placeholder string has to be alphanumeric!') 628 629 counter = 0 630 631 def dict_replace_string(d): 632 nonlocal counter 633 nonlocal ol 634 x = {} 635 for k, v in d.items(): 636 if isinstance(v, dict): 637 v = dict_replace_string(v) 638 elif isinstance(v, list): 639 v = list_replace_string(v) 640 elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)): 641 index = int(v[len(reps):]) 642 v = ol[index] 643 counter += 1 644 x[k] = v 645 return x 646 647 def list_replace_string(li): 648 nonlocal counter 649 nonlocal ol 650 x = [] 651 for e in li: 652 if isinstance(e, list): 653 e = list_replace_string(e) 654 elif isinstance(e, dict): 655 e = dict_replace_string(e) 656 elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)): 657 index = int(e[len(reps):]) 658 e = ol[index] 659 counter += 1 660 x.append(e) 661 return x 662 663 nd = dict_replace_string(ind) 664 665 if counter == 0: 666 raise Exception('No placeholder has been replaced! Check if reps is set correctly.') 667 668 return nd 669 670 671def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): 672 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. 673 674 The following structures are supported: Obs, list, numpy.ndarray, Corr 675 676 Parameters 677 ---------- 678 fname : str 679 Filename of the input file. 680 verbose : bool 681 Print additional information that was written to the file. 682 gz : bool 683 If True, assumes that data is gzipped. If False, assumes JSON file. 684 full_output : bool 685 If True, a dict containing auxiliary information and the data is returned. 686 If False, only the data is returned. 687 reps : str 688 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 689 """ 690 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) 691 description = indata['description']['description'] 692 indict = indata['description']['OBSDICT'] 693 ol = indata['obsdata'] 694 od = _od_from_list_and_dict(ol, indict, reps=reps) 695 696 if full_output: 697 indata['description'] = description 698 indata['obsdata'] = od 699 return indata 700 else: 701 return od
18def create_json_string(ol, description='', indent=1): 19 """Generate the string for the export of a list of Obs or structures containing Obs 20 to a .json(.gz) file 21 22 Parameters 23 ---------- 24 ol : list 25 List of objects that will be exported. At the moment, these objects can be 26 either of: Obs, list, numpy.ndarray, Corr. 27 All Obs inside a structure have to be defined on the same set of configurations. 28 description : str 29 Optional string that describes the contents of the json file. 30 indent : int 31 Specify the indentation level of the json file. None or 0 is permissible and 32 saves disk space. 33 """ 34 35 def _gen_data_d_from_list(ol): 36 dl = [] 37 No = len(ol) 38 for name in ol[0].mc_names: 39 ed = {} 40 ed['id'] = name 41 ed['replica'] = [] 42 for r_name in ol[0].e_content[name]: 43 rd = {} 44 rd['name'] = r_name 45 if ol[0].is_merged.get(r_name, False): 46 rd['is_merged'] = True 47 rd['deltas'] = [] 48 offsets = [o.r_values[r_name] - o.value for o in ol] 49 deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)]) 50 for i in range(len(ol[0].idl[r_name])): 51 rd['deltas'].append([ol[0].idl[r_name][i]]) 52 rd['deltas'][-1] += deltas[i].tolist() 53 ed['replica'].append(rd) 54 dl.append(ed) 55 return dl 56 57 def _gen_cdata_d_from_list(ol): 58 dl = [] 59 for name in ol[0].cov_names: 60 ed = {} 61 ed['id'] = name 62 ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') 63 ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) 64 ncov = ol[0].covobs[name].cov.shape[0] 65 ed['grad'] = [] 66 for i in range(ncov): 67 ed['grad'].append([]) 68 for o in ol: 69 ed['grad'][-1].append(o.covobs[name].grad[i][0]) 70 dl.append(ed) 71 return dl 72 73 def write_Obs_to_dict(o): 74 d = {} 75 d['type'] = 'Obs' 76 d['layout'] = '1' 77 if o.tag: 78 d['tag'] = [o.tag] 79 if o.reweighted: 80 d['reweighted'] = o.reweighted 81 d['value'] = [o.value] 82 data = _gen_data_d_from_list([o]) 83 if len(data) > 0: 84 d['data'] = data 85 cdata = _gen_cdata_d_from_list([o]) 86 if len(cdata) > 0: 87 d['cdata'] = cdata 88 return d 89 90 def write_List_to_dict(ol): 91 _assert_equal_properties(ol) 92 d = {} 93 d['type'] = 'List' 94 d['layout'] = '%d' % len(ol) 95 taglist = [o.tag for o in ol] 96 if np.any([tag is not None for tag in taglist]): 97 d['tag'] = taglist 98 if ol[0].reweighted: 99 d['reweighted'] = ol[0].reweighted 100 d['value'] = [o.value for o in ol] 101 data = _gen_data_d_from_list(ol) 102 if len(data) > 0: 103 d['data'] = data 104 cdata = _gen_cdata_d_from_list(ol) 105 if len(cdata) > 0: 106 d['cdata'] = cdata 107 return d 108 109 def write_Array_to_dict(oa): 110 ol = np.ravel(oa) 111 _assert_equal_properties(ol) 112 d = {} 113 d['type'] = 'Array' 114 d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') 115 taglist = [o.tag for o in ol] 116 if np.any([tag is not None for tag in taglist]): 117 d['tag'] = taglist 118 if ol[0].reweighted: 119 d['reweighted'] = ol[0].reweighted 120 d['value'] = [o.value for o in ol] 121 data = _gen_data_d_from_list(ol) 122 if len(data) > 0: 123 d['data'] = data 124 cdata = _gen_cdata_d_from_list(ol) 125 if len(cdata) > 0: 126 d['cdata'] = cdata 127 return d 128 129 def _nan_Obs_like(obs): 130 samples = [] 131 names = [] 132 idl = [] 133 for key, value in obs.idl.items(): 134 samples.append([np.nan] * len(value)) 135 names.append(key) 136 idl.append(value) 137 my_obs = Obs(samples, names, idl) 138 my_obs._covobs = obs._covobs 139 for name in obs._covobs: 140 my_obs.names.append(name) 141 my_obs.reweighted = obs.reweighted 142 my_obs.is_merged = obs.is_merged 143 return my_obs 144 145 def write_Corr_to_dict(my_corr): 146 first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j)) 147 dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object) 148 dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0]) 149 content = [o if o is not None else dummy_array for o in my_corr.content] 150 dat = write_Array_to_dict(np.array(content, dtype=object)) 151 dat['type'] = 'Corr' 152 corr_meta_data = str(my_corr.tag) 153 if 'tag' in dat.keys(): 154 dat['tag'].append(corr_meta_data) 155 else: 156 dat['tag'] = [corr_meta_data] 157 taglist = dat['tag'] 158 dat['tag'] = {} # tag is now a dictionary, that contains the previous taglist in the key "tag" 159 dat['tag']['tag'] = taglist 160 if my_corr.prange is not None: 161 dat['tag']['prange'] = my_corr.prange 162 return dat 163 164 if not isinstance(ol, list): 165 ol = [ol] 166 167 d = {} 168 d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__) 169 d['version'] = '1.1' 170 d['who'] = getpass.getuser() 171 d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z') 172 d['host'] = socket.gethostname() + ', ' + platform.platform() 173 174 if description: 175 d['description'] = description 176 177 d['obsdata'] = [] 178 for io in ol: 179 if isinstance(io, Obs): 180 d['obsdata'].append(write_Obs_to_dict(io)) 181 elif isinstance(io, list): 182 d['obsdata'].append(write_List_to_dict(io)) 183 elif isinstance(io, np.ndarray): 184 d['obsdata'].append(write_Array_to_dict(io)) 185 elif isinstance(io, Corr): 186 d['obsdata'].append(write_Corr_to_dict(io)) 187 else: 188 raise Exception("Unkown datatype.") 189 190 if indent: 191 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY) 192 else: 193 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT)
Generate the string for the export of a list of Obs or structures containing Obs to a .json(.gz) file
Parameters
- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
196def dump_to_json(ol, fname, description='', indent=1, gz=True): 197 """Export a list of Obs or structures containing Obs to a .json(.gz) file 198 199 Parameters 200 ---------- 201 ol : list 202 List of objects that will be exported. At the moment, these objects can be 203 either of: Obs, list, numpy.ndarray, Corr. 204 All Obs inside a structure have to be defined on the same set of configurations. 205 fname : str 206 Filename of the output file. 207 description : str 208 Optional string that describes the contents of the json file. 209 indent : int 210 Specify the indentation level of the json file. None or 0 is permissible and 211 saves disk space. 212 gz : bool 213 If True, the output is a gzipped json. If False, the output is a json file. 214 """ 215 216 jsonstring = create_json_string(ol, description, indent) 217 218 if not fname.endswith('.json') and not fname.endswith('.gz'): 219 fname += '.json' 220 221 if gz: 222 if not fname.endswith('.gz'): 223 fname += '.gz' 224 225 fp = gzip.open(fname, 'wb') 226 fp.write(jsonstring.encode('utf-8')) 227 else: 228 fp = open(fname, 'w', encoding='utf-8') 229 fp.write(jsonstring) 230 fp.close()
Export a list of Obs or structures containing Obs to a .json(.gz) file
Parameters
- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.
439def import_json_string(json_string, verbose=True, full_output=False): 440 """Reconstruct a list of Obs or structures containing Obs from a json string. 441 442 The following structures are supported: Obs, list, numpy.ndarray, Corr 443 If the list contains only one element, it is unpacked from the list. 444 445 Parameters 446 ---------- 447 json_string : str 448 json string containing the data. 449 verbose : bool 450 Print additional information that was written to the file. 451 full_output : bool 452 If True, a dict containing auxiliary information and the data is returned. 453 If False, only the data is returned. 454 """ 455 456 return _parse_json_dict(json.loads(json_string), verbose, full_output)
Reconstruct a list of Obs or structures containing Obs from a json string.
The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.
Parameters
- json_string (str): json string containing the data.
- verbose (bool): Print additional information that was written to the file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
459def load_json(fname, verbose=True, gz=True, full_output=False): 460 """Import a list of Obs or structures containing Obs from a .json(.gz) file. 461 462 The following structures are supported: Obs, list, numpy.ndarray, Corr 463 If the list contains only one element, it is unpacked from the list. 464 465 Parameters 466 ---------- 467 fname : str 468 Filename of the input file. 469 verbose : bool 470 Print additional information that was written to the file. 471 gz : bool 472 If True, assumes that data is gzipped. If False, assumes JSON file. 473 full_output : bool 474 If True, a dict containing auxiliary information and the data is returned. 475 If False, only the data is returned. 476 """ 477 if not fname.endswith('.json') and not fname.endswith('.gz'): 478 fname += '.json' 479 if gz: 480 if not fname.endswith('.gz'): 481 fname += '.gz' 482 with gzip.open(fname, 'r') as fin: 483 d = json.load(fin) 484 else: 485 if fname.endswith('.gz'): 486 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) 487 with open(fname, 'r', encoding='utf-8') as fin: 488 d = json.loads(fin.read()) 489 490 return _parse_json_dict(d, verbose, full_output)
Import a list of Obs or structures containing Obs from a .json(.gz) file.
The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.
Parameters
- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
573def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): 574 """Export a dict of Obs or structures containing Obs to a .json(.gz) file 575 576 Parameters 577 ---------- 578 od : dict 579 Dict of JSON valid structures and objects that will be exported. 580 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 581 All Obs inside a structure have to be defined on the same set of configurations. 582 fname : str 583 Filename of the output file. 584 description : str 585 Optional string that describes the contents of the json file. 586 indent : int 587 Specify the indentation level of the json file. None or 0 is permissible and 588 saves disk space. 589 reps : str 590 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 591 gz : bool 592 If True, the output is a gzipped json. If False, the output is a json file. 593 """ 594 595 if not isinstance(od, dict): 596 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') 597 598 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' 599 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' 600 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' 601 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') 602 603 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} 604 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) 605 606 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
Export a dict of Obs or structures containing Obs to a .json(.gz) file
Parameters
- od (dict): Dict of JSON valid structures and objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- reps (str): Specify the structure of the placeholder in exported dict to be reps[0-9]+.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.
672def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): 673 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. 674 675 The following structures are supported: Obs, list, numpy.ndarray, Corr 676 677 Parameters 678 ---------- 679 fname : str 680 Filename of the input file. 681 verbose : bool 682 Print additional information that was written to the file. 683 gz : bool 684 If True, assumes that data is gzipped. If False, assumes JSON file. 685 full_output : bool 686 If True, a dict containing auxiliary information and the data is returned. 687 If False, only the data is returned. 688 reps : str 689 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 690 """ 691 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) 692 description = indata['description']['description'] 693 indict = indata['description']['OBSDICT'] 694 ol = indata['obsdata'] 695 od = _od_from_list_and_dict(ol, indict, reps=reps) 696 697 if full_output: 698 indata['description'] = description 699 indata['obsdata'] = od 700 return indata 701 else: 702 return od
Import a dict of Obs or structures containing Obs from a .json(.gz) file.
The following structures are supported: Obs, list, numpy.ndarray, Corr
Parameters
- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
- reps (str): Specify the structure of the placeholder in imported dict to be reps[0-9]+.