pyerrors.input.json
1import rapidjson as json 2import gzip 3import getpass 4import socket 5import datetime 6import platform 7import warnings 8import re 9import numpy as np 10from ..obs import Obs 11from ..covobs import Covobs 12from ..correlators import Corr 13from ..misc import _assert_equal_properties 14from .. import version as pyerrorsversion 15 16 17def create_json_string(ol, description='', indent=1): 18 """Generate the string for the export of a list of Obs or structures containing Obs 19 to a .json(.gz) file 20 21 Parameters 22 ---------- 23 ol : list 24 List of objects that will be exported. At the moment, these objects can be 25 either of: Obs, list, numpy.ndarray, Corr. 26 All Obs inside a structure have to be defined on the same set of configurations. 27 description : str 28 Optional string that describes the contents of the json file. 29 indent : int 30 Specify the indentation level of the json file. None or 0 is permissible and 31 saves disk space. 32 """ 33 34 def _gen_data_d_from_list(ol): 35 dl = [] 36 No = len(ol) 37 for name in ol[0].mc_names: 38 ed = {} 39 ed['id'] = name 40 ed['replica'] = [] 41 for r_name in ol[0].e_content[name]: 42 rd = {} 43 rd['name'] = r_name 44 if ol[0].is_merged.get(r_name, False): 45 rd['is_merged'] = True 46 rd['deltas'] = [] 47 offsets = [o.r_values[r_name] - o.value for o in ol] 48 deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)]) 49 for i in range(len(ol[0].idl[r_name])): 50 rd['deltas'].append([ol[0].idl[r_name][i]]) 51 rd['deltas'][-1] += deltas[i].tolist() 52 ed['replica'].append(rd) 53 dl.append(ed) 54 return dl 55 56 def _gen_cdata_d_from_list(ol): 57 dl = [] 58 for name in ol[0].cov_names: 59 ed = {} 60 ed['id'] = name 61 ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') 62 ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) 63 ncov = ol[0].covobs[name].cov.shape[0] 64 ed['grad'] = [] 65 for i in range(ncov): 66 ed['grad'].append([]) 67 for o in ol: 68 ed['grad'][-1].append(o.covobs[name].grad[i][0]) 69 dl.append(ed) 70 return dl 71 72 def write_Obs_to_dict(o): 73 d = {} 74 d['type'] = 'Obs' 75 d['layout'] = '1' 76 if o.tag: 77 d['tag'] = [o.tag] 78 if o.reweighted: 79 d['reweighted'] = o.reweighted 80 d['value'] = [o.value] 81 data = _gen_data_d_from_list([o]) 82 if len(data) > 0: 83 d['data'] = data 84 cdata = _gen_cdata_d_from_list([o]) 85 if len(cdata) > 0: 86 d['cdata'] = cdata 87 return d 88 89 def write_List_to_dict(ol): 90 _assert_equal_properties(ol) 91 d = {} 92 d['type'] = 'List' 93 d['layout'] = '%d' % len(ol) 94 taglist = [o.tag for o in ol] 95 if np.any([tag is not None for tag in taglist]): 96 d['tag'] = taglist 97 if ol[0].reweighted: 98 d['reweighted'] = ol[0].reweighted 99 d['value'] = [o.value for o in ol] 100 data = _gen_data_d_from_list(ol) 101 if len(data) > 0: 102 d['data'] = data 103 cdata = _gen_cdata_d_from_list(ol) 104 if len(cdata) > 0: 105 d['cdata'] = cdata 106 return d 107 108 def write_Array_to_dict(oa): 109 ol = np.ravel(oa) 110 _assert_equal_properties(ol) 111 d = {} 112 d['type'] = 'Array' 113 d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') 114 taglist = [o.tag for o in ol] 115 if np.any([tag is not None for tag in taglist]): 116 d['tag'] = taglist 117 if ol[0].reweighted: 118 d['reweighted'] = ol[0].reweighted 119 d['value'] = [o.value for o in ol] 120 data = _gen_data_d_from_list(ol) 121 if len(data) > 0: 122 d['data'] = data 123 cdata = _gen_cdata_d_from_list(ol) 124 if len(cdata) > 0: 125 d['cdata'] = cdata 126 return d 127 128 def _nan_Obs_like(obs): 129 samples = [] 130 names = [] 131 idl = [] 132 for key, value in obs.idl.items(): 133 samples.append([np.nan] * len(value)) 134 names.append(key) 135 idl.append(value) 136 my_obs = Obs(samples, names, idl) 137 my_obs._covobs = obs._covobs 138 for name in obs._covobs: 139 my_obs.names.append(name) 140 my_obs.reweighted = obs.reweighted 141 my_obs.is_merged = obs.is_merged 142 return my_obs 143 144 def write_Corr_to_dict(my_corr): 145 first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j)) 146 dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object) 147 dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0]) 148 content = [o if o is not None else dummy_array for o in my_corr.content] 149 dat = write_Array_to_dict(np.array(content, dtype=object)) 150 dat['type'] = 'Corr' 151 corr_meta_data = str(my_corr.tag) 152 if 'tag' in dat.keys(): 153 dat['tag'].append(corr_meta_data) 154 else: 155 dat['tag'] = [corr_meta_data] 156 taglist = dat['tag'] 157 dat['tag'] = {} # tag is now a dictionary, that contains the previous taglist in the key "tag" 158 dat['tag']['tag'] = taglist 159 if my_corr.prange is not None: 160 dat['tag']['prange'] = my_corr.prange 161 return dat 162 163 if not isinstance(ol, list): 164 ol = [ol] 165 166 d = {} 167 d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__) 168 d['version'] = '1.1' 169 d['who'] = getpass.getuser() 170 d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z') 171 d['host'] = socket.gethostname() + ', ' + platform.platform() 172 173 if description: 174 d['description'] = description 175 176 d['obsdata'] = [] 177 for io in ol: 178 if isinstance(io, Obs): 179 d['obsdata'].append(write_Obs_to_dict(io)) 180 elif isinstance(io, list): 181 d['obsdata'].append(write_List_to_dict(io)) 182 elif isinstance(io, np.ndarray): 183 d['obsdata'].append(write_Array_to_dict(io)) 184 elif isinstance(io, Corr): 185 d['obsdata'].append(write_Corr_to_dict(io)) 186 else: 187 raise Exception("Unkown datatype.") 188 189 if indent: 190 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY) 191 else: 192 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT) 193 194 195def dump_to_json(ol, fname, description='', indent=1, gz=True): 196 """Export a list of Obs or structures containing Obs to a .json(.gz) file 197 198 Parameters 199 ---------- 200 ol : list 201 List of objects that will be exported. At the moment, these objects can be 202 either of: Obs, list, numpy.ndarray, Corr. 203 All Obs inside a structure have to be defined on the same set of configurations. 204 fname : str 205 Filename of the output file. 206 description : str 207 Optional string that describes the contents of the json file. 208 indent : int 209 Specify the indentation level of the json file. None or 0 is permissible and 210 saves disk space. 211 gz : bool 212 If True, the output is a gzipped json. If False, the output is a json file. 213 """ 214 215 jsonstring = create_json_string(ol, description, indent) 216 217 if not fname.endswith('.json') and not fname.endswith('.gz'): 218 fname += '.json' 219 220 if gz: 221 if not fname.endswith('.gz'): 222 fname += '.gz' 223 224 fp = gzip.open(fname, 'wb') 225 fp.write(jsonstring.encode('utf-8')) 226 else: 227 fp = open(fname, 'w', encoding='utf-8') 228 fp.write(jsonstring) 229 fp.close() 230 231 232def _parse_json_dict(json_dict, verbose=True, full_output=False): 233 """Reconstruct a list of Obs or structures containing Obs from a dict that 234 was built out of a json string. 235 236 The following structures are supported: Obs, list, numpy.ndarray, Corr 237 If the list contains only one element, it is unpacked from the list. 238 239 Parameters 240 ---------- 241 json_string : str 242 json string containing the data. 243 verbose : bool 244 Print additional information that was written to the file. 245 full_output : bool 246 If True, a dict containing auxiliary information and the data is returned. 247 If False, only the data is returned. 248 """ 249 250 def _gen_obsd_from_datad(d): 251 retd = {} 252 if d: 253 retd['names'] = [] 254 retd['idl'] = [] 255 retd['deltas'] = [] 256 retd['is_merged'] = {} 257 for ens in d: 258 for rep in ens['replica']: 259 rep_name = rep['name'] 260 if len(rep_name) > len(ens["id"]): 261 if rep_name[len(ens["id"])] != "|": 262 tmp_list = list(rep_name) 263 tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):] 264 rep_name = ''.join(tmp_list) 265 retd['names'].append(rep_name) 266 retd['idl'].append([di[0] for di in rep['deltas']]) 267 retd['deltas'].append(np.array([di[1:] for di in rep['deltas']])) 268 retd['is_merged'][rep_name] = rep.get('is_merged', False) 269 return retd 270 271 def _gen_covobsd_from_cdatad(d): 272 retd = {} 273 for ens in d: 274 retl = [] 275 name = ens['id'] 276 layouts = ens.get('layout', '1').strip() 277 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] 278 cov = np.reshape(ens['cov'], layout) 279 grad = ens['grad'] 280 nobs = len(grad[0]) 281 for i in range(nobs): 282 retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]}) 283 retd[name] = retl 284 return retd 285 286 def get_Obs_from_dict(o): 287 layouts = o.get('layout', '1').strip() 288 if layouts != '1': 289 raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning) 290 291 values = o['value'] 292 od = _gen_obsd_from_datad(o.get('data', {})) 293 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 294 295 if od: 296 ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl']) 297 ret.is_merged = od['is_merged'] 298 else: 299 ret = Obs([], [], means=[]) 300 ret._value = values[0] 301 for name in cd: 302 co = cd[name][0] 303 ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 304 ret.names.append(co['name']) 305 306 ret.reweighted = o.get('reweighted', False) 307 ret.tag = o.get('tag', [None])[0] 308 return ret 309 310 def get_List_from_dict(o): 311 layouts = o.get('layout', '1').strip() 312 layout = int(layouts) 313 values = o['value'] 314 od = _gen_obsd_from_datad(o.get('data', {})) 315 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 316 317 ret = [] 318 taglist = o.get('tag', layout * [None]) 319 for i in range(layout): 320 if od: 321 ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl'])) 322 ret[-1].is_merged = od['is_merged'] 323 else: 324 ret.append(Obs([], [], means=[])) 325 ret[-1]._value = values[i] 326 print('Created Obs with means= ', values[i]) 327 for name in cd: 328 co = cd[name][i] 329 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 330 ret[-1].names.append(co['name']) 331 332 ret[-1].reweighted = o.get('reweighted', False) 333 ret[-1].tag = taglist[i] 334 return ret 335 336 def get_Array_from_dict(o): 337 layouts = o.get('layout', '1').strip() 338 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] 339 N = np.prod(layout) 340 values = o['value'] 341 od = _gen_obsd_from_datad(o.get('data', {})) 342 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) 343 344 ret = [] 345 taglist = o.get('tag', N * [None]) 346 for i in range(N): 347 if od: 348 ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl'])) 349 ret[-1].is_merged = od['is_merged'] 350 else: 351 ret.append(Obs([], [], means=[])) 352 ret[-1]._value = values[i] 353 for name in cd: 354 co = cd[name][i] 355 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) 356 ret[-1].names.append(co['name']) 357 ret[-1].reweighted = o.get('reweighted', False) 358 ret[-1].tag = taglist[i] 359 return np.reshape(ret, layout) 360 361 def get_Corr_from_dict(o): 362 if isinstance(o.get('tag'), list): # supports the old way 363 taglist = o.get('tag') # This had to be modified to get the taglist from the dictionary 364 temp_prange = None 365 elif isinstance(o.get('tag'), dict): 366 tagdic = o.get('tag') 367 taglist = tagdic['tag'] 368 if 'prange' in tagdic: 369 temp_prange = tagdic['prange'] 370 else: 371 temp_prange = None 372 else: 373 raise Exception("The tag is not a list or dict") 374 375 corr_tag = taglist[-1] 376 tmp_o = o 377 tmp_o['tag'] = taglist[:-1] 378 if len(tmp_o['tag']) == 0: 379 del tmp_o['tag'] 380 dat = get_Array_from_dict(tmp_o) 381 my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)]) 382 if corr_tag != 'None': 383 my_corr.tag = corr_tag 384 385 my_corr.prange = temp_prange 386 return my_corr 387 388 prog = json_dict.get('program', '') 389 version = json_dict.get('version', '') 390 who = json_dict.get('who', '') 391 date = json_dict.get('date', '') 392 host = json_dict.get('host', '') 393 if prog and verbose: 394 print('Data has been written using %s.' % (prog)) 395 if version and verbose: 396 print('Format version %s' % (version)) 397 if np.any([who, date, host] and verbose): 398 print('Written by %s on %s on host %s' % (who, date, host)) 399 description = json_dict.get('description', '') 400 if description and verbose: 401 print() 402 print('Description: ', description) 403 obsdata = json_dict['obsdata'] 404 ol = [] 405 for io in obsdata: 406 if io['type'] == 'Obs': 407 ol.append(get_Obs_from_dict(io)) 408 elif io['type'] == 'List': 409 ol.append(get_List_from_dict(io)) 410 elif io['type'] == 'Array': 411 ol.append(get_Array_from_dict(io)) 412 elif io['type'] == 'Corr': 413 ol.append(get_Corr_from_dict(io)) 414 else: 415 raise Exception("Unkown datatype.") 416 417 if full_output: 418 retd = {} 419 retd['program'] = prog 420 retd['version'] = version 421 retd['who'] = who 422 retd['date'] = date 423 retd['host'] = host 424 retd['description'] = description 425 retd['obsdata'] = ol 426 427 return retd 428 else: 429 if len(obsdata) == 1: 430 ol = ol[0] 431 432 return ol 433 434 435def import_json_string(json_string, verbose=True, full_output=False): 436 """Reconstruct a list of Obs or structures containing Obs from a json string. 437 438 The following structures are supported: Obs, list, numpy.ndarray, Corr 439 If the list contains only one element, it is unpacked from the list. 440 441 Parameters 442 ---------- 443 json_string : str 444 json string containing the data. 445 verbose : bool 446 Print additional information that was written to the file. 447 full_output : bool 448 If True, a dict containing auxiliary information and the data is returned. 449 If False, only the data is returned. 450 """ 451 452 return _parse_json_dict(json.loads(json_string), verbose, full_output) 453 454 455def load_json(fname, verbose=True, gz=True, full_output=False): 456 """Import a list of Obs or structures containing Obs from a .json(.gz) file. 457 458 The following structures are supported: Obs, list, numpy.ndarray, Corr 459 If the list contains only one element, it is unpacked from the list. 460 461 Parameters 462 ---------- 463 fname : str 464 Filename of the input file. 465 verbose : bool 466 Print additional information that was written to the file. 467 gz : bool 468 If True, assumes that data is gzipped. If False, assumes JSON file. 469 full_output : bool 470 If True, a dict containing auxiliary information and the data is returned. 471 If False, only the data is returned. 472 """ 473 if not fname.endswith('.json') and not fname.endswith('.gz'): 474 fname += '.json' 475 if gz: 476 if not fname.endswith('.gz'): 477 fname += '.gz' 478 with gzip.open(fname, 'r') as fin: 479 d = json.load(fin) 480 else: 481 if fname.endswith('.gz'): 482 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) 483 with open(fname, 'r', encoding='utf-8') as fin: 484 d = json.loads(fin.read()) 485 486 return _parse_json_dict(d, verbose, full_output) 487 488 489def _ol_from_dict(ind, reps='DICTOBS'): 490 """Convert a dictionary of Obs objects to a list and a dictionary that contains 491 placeholders instead of the Obs objects. 492 493 Parameters 494 ---------- 495 ind : dict 496 Dict of JSON valid structures and objects that will be exported. 497 At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr. 498 All Obs inside a structure have to be defined on the same set of configurations. 499 reps : str 500 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 501 """ 502 503 obstypes = (Obs, Corr, np.ndarray) 504 505 if not reps.isalnum(): 506 raise Exception('Placeholder string has to be alphanumeric!') 507 ol = [] 508 counter = 0 509 510 def dict_replace_obs(d): 511 nonlocal ol 512 nonlocal counter 513 x = {} 514 for k, v in d.items(): 515 if isinstance(v, dict): 516 v = dict_replace_obs(v) 517 elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]): 518 v = obslist_replace_obs(v) 519 elif isinstance(v, list): 520 v = list_replace_obs(v) 521 elif isinstance(v, obstypes): 522 ol.append(v) 523 v = reps + '%d' % (counter) 524 counter += 1 525 elif isinstance(v, str): 526 if bool(re.match(r'%s[0-9]+' % (reps), v)): 527 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps)) 528 x[k] = v 529 return x 530 531 def list_replace_obs(li): 532 nonlocal ol 533 nonlocal counter 534 x = [] 535 for e in li: 536 if isinstance(e, list): 537 e = list_replace_obs(e) 538 elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]): 539 e = obslist_replace_obs(e) 540 elif isinstance(e, dict): 541 e = dict_replace_obs(e) 542 elif isinstance(e, obstypes): 543 ol.append(e) 544 e = reps + '%d' % (counter) 545 counter += 1 546 elif isinstance(e, str): 547 if bool(re.match(r'%s[0-9]+' % (reps), e)): 548 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps)) 549 x.append(e) 550 return x 551 552 def obslist_replace_obs(li): 553 nonlocal ol 554 nonlocal counter 555 il = [] 556 for e in li: 557 il.append(e) 558 559 ol.append(il) 560 x = reps + '%d' % (counter) 561 counter += 1 562 return x 563 564 nd = dict_replace_obs(ind) 565 566 return ol, nd 567 568 569def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): 570 """Export a dict of Obs or structures containing Obs to a .json(.gz) file 571 572 Parameters 573 ---------- 574 od : dict 575 Dict of JSON valid structures and objects that will be exported. 576 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 577 All Obs inside a structure have to be defined on the same set of configurations. 578 fname : str 579 Filename of the output file. 580 description : str 581 Optional string that describes the contents of the json file. 582 indent : int 583 Specify the indentation level of the json file. None or 0 is permissible and 584 saves disk space. 585 reps : str 586 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 587 gz : bool 588 If True, the output is a gzipped json. If False, the output is a json file. 589 """ 590 591 if not isinstance(od, dict): 592 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') 593 594 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' 595 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' 596 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' 597 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') 598 599 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} 600 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) 601 602 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz) 603 604 605def _od_from_list_and_dict(ol, ind, reps='DICTOBS'): 606 """Parse a list of Obs or structures containing Obs and an accompanying 607 dict, where the structures have been replaced by placeholders to a 608 dict that contains the structures. 609 610 The following structures are supported: Obs, list, numpy.ndarray, Corr 611 612 Parameters 613 ---------- 614 ol : list 615 List of objects - 616 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 617 All Obs inside a structure have to be defined on the same set of configurations. 618 ind : dict 619 Dict that defines the structure of the resulting dict and contains placeholders 620 reps : str 621 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 622 """ 623 if not reps.isalnum(): 624 raise Exception('Placeholder string has to be alphanumeric!') 625 626 counter = 0 627 628 def dict_replace_string(d): 629 nonlocal counter 630 nonlocal ol 631 x = {} 632 for k, v in d.items(): 633 if isinstance(v, dict): 634 v = dict_replace_string(v) 635 elif isinstance(v, list): 636 v = list_replace_string(v) 637 elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)): 638 index = int(v[len(reps):]) 639 v = ol[index] 640 counter += 1 641 x[k] = v 642 return x 643 644 def list_replace_string(li): 645 nonlocal counter 646 nonlocal ol 647 x = [] 648 for e in li: 649 if isinstance(e, list): 650 e = list_replace_string(e) 651 elif isinstance(e, dict): 652 e = dict_replace_string(e) 653 elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)): 654 index = int(e[len(reps):]) 655 e = ol[index] 656 counter += 1 657 x.append(e) 658 return x 659 660 nd = dict_replace_string(ind) 661 662 if counter == 0: 663 raise Exception('No placeholder has been replaced! Check if reps is set correctly.') 664 665 return nd 666 667 668def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): 669 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. 670 671 The following structures are supported: Obs, list, numpy.ndarray, Corr 672 673 Parameters 674 ---------- 675 fname : str 676 Filename of the input file. 677 verbose : bool 678 Print additional information that was written to the file. 679 gz : bool 680 If True, assumes that data is gzipped. If False, assumes JSON file. 681 full_output : bool 682 If True, a dict containing auxiliary information and the data is returned. 683 If False, only the data is returned. 684 reps : str 685 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 686 """ 687 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) 688 description = indata['description']['description'] 689 indict = indata['description']['OBSDICT'] 690 ol = indata['obsdata'] 691 od = _od_from_list_and_dict(ol, indict, reps=reps) 692 693 if full_output: 694 indata['description'] = description 695 indata['obsdata'] = od 696 return indata 697 else: 698 return od
18def create_json_string(ol, description='', indent=1): 19 """Generate the string for the export of a list of Obs or structures containing Obs 20 to a .json(.gz) file 21 22 Parameters 23 ---------- 24 ol : list 25 List of objects that will be exported. At the moment, these objects can be 26 either of: Obs, list, numpy.ndarray, Corr. 27 All Obs inside a structure have to be defined on the same set of configurations. 28 description : str 29 Optional string that describes the contents of the json file. 30 indent : int 31 Specify the indentation level of the json file. None or 0 is permissible and 32 saves disk space. 33 """ 34 35 def _gen_data_d_from_list(ol): 36 dl = [] 37 No = len(ol) 38 for name in ol[0].mc_names: 39 ed = {} 40 ed['id'] = name 41 ed['replica'] = [] 42 for r_name in ol[0].e_content[name]: 43 rd = {} 44 rd['name'] = r_name 45 if ol[0].is_merged.get(r_name, False): 46 rd['is_merged'] = True 47 rd['deltas'] = [] 48 offsets = [o.r_values[r_name] - o.value for o in ol] 49 deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)]) 50 for i in range(len(ol[0].idl[r_name])): 51 rd['deltas'].append([ol[0].idl[r_name][i]]) 52 rd['deltas'][-1] += deltas[i].tolist() 53 ed['replica'].append(rd) 54 dl.append(ed) 55 return dl 56 57 def _gen_cdata_d_from_list(ol): 58 dl = [] 59 for name in ol[0].cov_names: 60 ed = {} 61 ed['id'] = name 62 ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') 63 ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) 64 ncov = ol[0].covobs[name].cov.shape[0] 65 ed['grad'] = [] 66 for i in range(ncov): 67 ed['grad'].append([]) 68 for o in ol: 69 ed['grad'][-1].append(o.covobs[name].grad[i][0]) 70 dl.append(ed) 71 return dl 72 73 def write_Obs_to_dict(o): 74 d = {} 75 d['type'] = 'Obs' 76 d['layout'] = '1' 77 if o.tag: 78 d['tag'] = [o.tag] 79 if o.reweighted: 80 d['reweighted'] = o.reweighted 81 d['value'] = [o.value] 82 data = _gen_data_d_from_list([o]) 83 if len(data) > 0: 84 d['data'] = data 85 cdata = _gen_cdata_d_from_list([o]) 86 if len(cdata) > 0: 87 d['cdata'] = cdata 88 return d 89 90 def write_List_to_dict(ol): 91 _assert_equal_properties(ol) 92 d = {} 93 d['type'] = 'List' 94 d['layout'] = '%d' % len(ol) 95 taglist = [o.tag for o in ol] 96 if np.any([tag is not None for tag in taglist]): 97 d['tag'] = taglist 98 if ol[0].reweighted: 99 d['reweighted'] = ol[0].reweighted 100 d['value'] = [o.value for o in ol] 101 data = _gen_data_d_from_list(ol) 102 if len(data) > 0: 103 d['data'] = data 104 cdata = _gen_cdata_d_from_list(ol) 105 if len(cdata) > 0: 106 d['cdata'] = cdata 107 return d 108 109 def write_Array_to_dict(oa): 110 ol = np.ravel(oa) 111 _assert_equal_properties(ol) 112 d = {} 113 d['type'] = 'Array' 114 d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') 115 taglist = [o.tag for o in ol] 116 if np.any([tag is not None for tag in taglist]): 117 d['tag'] = taglist 118 if ol[0].reweighted: 119 d['reweighted'] = ol[0].reweighted 120 d['value'] = [o.value for o in ol] 121 data = _gen_data_d_from_list(ol) 122 if len(data) > 0: 123 d['data'] = data 124 cdata = _gen_cdata_d_from_list(ol) 125 if len(cdata) > 0: 126 d['cdata'] = cdata 127 return d 128 129 def _nan_Obs_like(obs): 130 samples = [] 131 names = [] 132 idl = [] 133 for key, value in obs.idl.items(): 134 samples.append([np.nan] * len(value)) 135 names.append(key) 136 idl.append(value) 137 my_obs = Obs(samples, names, idl) 138 my_obs._covobs = obs._covobs 139 for name in obs._covobs: 140 my_obs.names.append(name) 141 my_obs.reweighted = obs.reweighted 142 my_obs.is_merged = obs.is_merged 143 return my_obs 144 145 def write_Corr_to_dict(my_corr): 146 first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j)) 147 dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object) 148 dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0]) 149 content = [o if o is not None else dummy_array for o in my_corr.content] 150 dat = write_Array_to_dict(np.array(content, dtype=object)) 151 dat['type'] = 'Corr' 152 corr_meta_data = str(my_corr.tag) 153 if 'tag' in dat.keys(): 154 dat['tag'].append(corr_meta_data) 155 else: 156 dat['tag'] = [corr_meta_data] 157 taglist = dat['tag'] 158 dat['tag'] = {} # tag is now a dictionary, that contains the previous taglist in the key "tag" 159 dat['tag']['tag'] = taglist 160 if my_corr.prange is not None: 161 dat['tag']['prange'] = my_corr.prange 162 return dat 163 164 if not isinstance(ol, list): 165 ol = [ol] 166 167 d = {} 168 d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__) 169 d['version'] = '1.1' 170 d['who'] = getpass.getuser() 171 d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z') 172 d['host'] = socket.gethostname() + ', ' + platform.platform() 173 174 if description: 175 d['description'] = description 176 177 d['obsdata'] = [] 178 for io in ol: 179 if isinstance(io, Obs): 180 d['obsdata'].append(write_Obs_to_dict(io)) 181 elif isinstance(io, list): 182 d['obsdata'].append(write_List_to_dict(io)) 183 elif isinstance(io, np.ndarray): 184 d['obsdata'].append(write_Array_to_dict(io)) 185 elif isinstance(io, Corr): 186 d['obsdata'].append(write_Corr_to_dict(io)) 187 else: 188 raise Exception("Unkown datatype.") 189 190 if indent: 191 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY) 192 else: 193 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT)
Generate the string for the export of a list of Obs or structures containing Obs to a .json(.gz) file
Parameters
- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
196def dump_to_json(ol, fname, description='', indent=1, gz=True): 197 """Export a list of Obs or structures containing Obs to a .json(.gz) file 198 199 Parameters 200 ---------- 201 ol : list 202 List of objects that will be exported. At the moment, these objects can be 203 either of: Obs, list, numpy.ndarray, Corr. 204 All Obs inside a structure have to be defined on the same set of configurations. 205 fname : str 206 Filename of the output file. 207 description : str 208 Optional string that describes the contents of the json file. 209 indent : int 210 Specify the indentation level of the json file. None or 0 is permissible and 211 saves disk space. 212 gz : bool 213 If True, the output is a gzipped json. If False, the output is a json file. 214 """ 215 216 jsonstring = create_json_string(ol, description, indent) 217 218 if not fname.endswith('.json') and not fname.endswith('.gz'): 219 fname += '.json' 220 221 if gz: 222 if not fname.endswith('.gz'): 223 fname += '.gz' 224 225 fp = gzip.open(fname, 'wb') 226 fp.write(jsonstring.encode('utf-8')) 227 else: 228 fp = open(fname, 'w', encoding='utf-8') 229 fp.write(jsonstring) 230 fp.close()
Export a list of Obs or structures containing Obs to a .json(.gz) file
Parameters
- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.
436def import_json_string(json_string, verbose=True, full_output=False): 437 """Reconstruct a list of Obs or structures containing Obs from a json string. 438 439 The following structures are supported: Obs, list, numpy.ndarray, Corr 440 If the list contains only one element, it is unpacked from the list. 441 442 Parameters 443 ---------- 444 json_string : str 445 json string containing the data. 446 verbose : bool 447 Print additional information that was written to the file. 448 full_output : bool 449 If True, a dict containing auxiliary information and the data is returned. 450 If False, only the data is returned. 451 """ 452 453 return _parse_json_dict(json.loads(json_string), verbose, full_output)
Reconstruct a list of Obs or structures containing Obs from a json string.
The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.
Parameters
- json_string (str): json string containing the data.
- verbose (bool): Print additional information that was written to the file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
456def load_json(fname, verbose=True, gz=True, full_output=False): 457 """Import a list of Obs or structures containing Obs from a .json(.gz) file. 458 459 The following structures are supported: Obs, list, numpy.ndarray, Corr 460 If the list contains only one element, it is unpacked from the list. 461 462 Parameters 463 ---------- 464 fname : str 465 Filename of the input file. 466 verbose : bool 467 Print additional information that was written to the file. 468 gz : bool 469 If True, assumes that data is gzipped. If False, assumes JSON file. 470 full_output : bool 471 If True, a dict containing auxiliary information and the data is returned. 472 If False, only the data is returned. 473 """ 474 if not fname.endswith('.json') and not fname.endswith('.gz'): 475 fname += '.json' 476 if gz: 477 if not fname.endswith('.gz'): 478 fname += '.gz' 479 with gzip.open(fname, 'r') as fin: 480 d = json.load(fin) 481 else: 482 if fname.endswith('.gz'): 483 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) 484 with open(fname, 'r', encoding='utf-8') as fin: 485 d = json.loads(fin.read()) 486 487 return _parse_json_dict(d, verbose, full_output)
Import a list of Obs or structures containing Obs from a .json(.gz) file.
The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.
Parameters
- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
570def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): 571 """Export a dict of Obs or structures containing Obs to a .json(.gz) file 572 573 Parameters 574 ---------- 575 od : dict 576 Dict of JSON valid structures and objects that will be exported. 577 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. 578 All Obs inside a structure have to be defined on the same set of configurations. 579 fname : str 580 Filename of the output file. 581 description : str 582 Optional string that describes the contents of the json file. 583 indent : int 584 Specify the indentation level of the json file. None or 0 is permissible and 585 saves disk space. 586 reps : str 587 Specify the structure of the placeholder in exported dict to be reps[0-9]+. 588 gz : bool 589 If True, the output is a gzipped json. If False, the output is a json file. 590 """ 591 592 if not isinstance(od, dict): 593 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') 594 595 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' 596 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' 597 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' 598 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') 599 600 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} 601 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) 602 603 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
Export a dict of Obs or structures containing Obs to a .json(.gz) file
Parameters
- od (dict): Dict of JSON valid structures and objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- reps (str): Specify the structure of the placeholder in exported dict to be reps[0-9]+.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.
669def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): 670 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. 671 672 The following structures are supported: Obs, list, numpy.ndarray, Corr 673 674 Parameters 675 ---------- 676 fname : str 677 Filename of the input file. 678 verbose : bool 679 Print additional information that was written to the file. 680 gz : bool 681 If True, assumes that data is gzipped. If False, assumes JSON file. 682 full_output : bool 683 If True, a dict containing auxiliary information and the data is returned. 684 If False, only the data is returned. 685 reps : str 686 Specify the structure of the placeholder in imported dict to be reps[0-9]+. 687 """ 688 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) 689 description = indata['description']['description'] 690 indict = indata['description']['OBSDICT'] 691 ol = indata['obsdata'] 692 od = _od_from_list_and_dict(ol, indict, reps=reps) 693 694 if full_output: 695 indata['description'] = description 696 indata['obsdata'] = od 697 return indata 698 else: 699 return od
Import a dict of Obs or structures containing Obs from a .json(.gz) file.
The following structures are supported: Obs, list, numpy.ndarray, Corr
Parameters
- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
- reps (str): Specify the structure of the placeholder in imported dict to be reps[0-9]+.