diff --git a/docs/pyerrors/input/json.html b/docs/pyerrors/input/json.html index 00d92682..e17d5257 100644 --- a/docs/pyerrors/input/json.html +++ b/docs/pyerrors/input/json.html @@ -382,408 +382,411 @@ 294 295 if od: 296 ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl']) -297 ret.is_merged = od['is_merged'] -298 else: -299 ret = Obs([], [], means=[]) -300 ret._value = values[0] -301 for name in cd: -302 co = cd[name][0] -303 ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) -304 ret.names.append(co['name']) -305 -306 ret.reweighted = o.get('reweighted', False) -307 ret.tag = o.get('tag', [None])[0] -308 return ret -309 -310 def get_List_from_dict(o): -311 layouts = o.get('layout', '1').strip() -312 layout = int(layouts) -313 values = o['value'] -314 od = _gen_obsd_from_datad(o.get('data', {})) -315 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) -316 -317 ret = [] -318 taglist = o.get('tag', layout * [None]) -319 for i in range(layout): -320 if od: -321 ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl'])) -322 ret[-1].is_merged = od['is_merged'] -323 else: -324 ret.append(Obs([], [], means=[])) -325 ret[-1]._value = values[i] -326 print('Created Obs with means= ', values[i]) -327 for name in cd: -328 co = cd[name][i] -329 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) -330 ret[-1].names.append(co['name']) -331 -332 ret[-1].reweighted = o.get('reweighted', False) -333 ret[-1].tag = taglist[i] -334 return ret -335 -336 def get_Array_from_dict(o): -337 layouts = o.get('layout', '1').strip() -338 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] -339 N = np.prod(layout) -340 values = o['value'] -341 od = _gen_obsd_from_datad(o.get('data', {})) -342 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) -343 -344 ret = [] -345 taglist = o.get('tag', N * [None]) -346 for i in range(N): -347 if od: -348 ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl'])) -349 ret[-1].is_merged = od['is_merged'] -350 else: -351 ret.append(Obs([], [], means=[])) -352 ret[-1]._value = values[i] -353 for name in cd: -354 co = cd[name][i] -355 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) -356 ret[-1].names.append(co['name']) -357 ret[-1].reweighted = o.get('reweighted', False) -358 ret[-1].tag = taglist[i] -359 return np.reshape(ret, layout) -360 -361 def get_Corr_from_dict(o): -362 if isinstance(o.get('tag'), list): # supports the old way -363 taglist = o.get('tag') # This had to be modified to get the taglist from the dictionary -364 temp_prange = None -365 elif isinstance(o.get('tag'), dict): -366 tagdic = o.get('tag') -367 taglist = tagdic['tag'] -368 if 'prange' in tagdic: -369 temp_prange = tagdic['prange'] -370 else: -371 temp_prange = None -372 else: -373 raise Exception("The tag is not a list or dict") -374 -375 corr_tag = taglist[-1] -376 tmp_o = o -377 tmp_o['tag'] = taglist[:-1] -378 if len(tmp_o['tag']) == 0: -379 del tmp_o['tag'] -380 dat = get_Array_from_dict(tmp_o) -381 my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)]) -382 if corr_tag != 'None': -383 my_corr.tag = corr_tag -384 -385 my_corr.prange = temp_prange -386 return my_corr +297 ret._value = values[0] +298 ret.is_merged = od['is_merged'] +299 else: +300 ret = Obs([], [], means=[]) +301 ret._value = values[0] +302 for name in cd: +303 co = cd[name][0] +304 ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) +305 ret.names.append(co['name']) +306 +307 ret.reweighted = o.get('reweighted', False) +308 ret.tag = o.get('tag', [None])[0] +309 return ret +310 +311 def get_List_from_dict(o): +312 layouts = o.get('layout', '1').strip() +313 layout = int(layouts) +314 values = o['value'] +315 od = _gen_obsd_from_datad(o.get('data', {})) +316 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) +317 +318 ret = [] +319 taglist = o.get('tag', layout * [None]) +320 for i in range(layout): +321 if od: +322 ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl'])) +323 ret[-1]._value = values[i] +324 ret[-1].is_merged = od['is_merged'] +325 else: +326 ret.append(Obs([], [], means=[])) +327 ret[-1]._value = values[i] +328 print('Created Obs with means= ', values[i]) +329 for name in cd: +330 co = cd[name][i] +331 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) +332 ret[-1].names.append(co['name']) +333 +334 ret[-1].reweighted = o.get('reweighted', False) +335 ret[-1].tag = taglist[i] +336 return ret +337 +338 def get_Array_from_dict(o): +339 layouts = o.get('layout', '1').strip() +340 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] +341 N = np.prod(layout) +342 values = o['value'] +343 od = _gen_obsd_from_datad(o.get('data', {})) +344 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) +345 +346 ret = [] +347 taglist = o.get('tag', N * [None]) +348 for i in range(N): +349 if od: +350 ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl'])) +351 ret[-1]._value = values[i] +352 ret[-1].is_merged = od['is_merged'] +353 else: +354 ret.append(Obs([], [], means=[])) +355 ret[-1]._value = values[i] +356 for name in cd: +357 co = cd[name][i] +358 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) +359 ret[-1].names.append(co['name']) +360 ret[-1].reweighted = o.get('reweighted', False) +361 ret[-1].tag = taglist[i] +362 return np.reshape(ret, layout) +363 +364 def get_Corr_from_dict(o): +365 if isinstance(o.get('tag'), list): # supports the old way +366 taglist = o.get('tag') # This had to be modified to get the taglist from the dictionary +367 temp_prange = None +368 elif isinstance(o.get('tag'), dict): +369 tagdic = o.get('tag') +370 taglist = tagdic['tag'] +371 if 'prange' in tagdic: +372 temp_prange = tagdic['prange'] +373 else: +374 temp_prange = None +375 else: +376 raise Exception("The tag is not a list or dict") +377 +378 corr_tag = taglist[-1] +379 tmp_o = o +380 tmp_o['tag'] = taglist[:-1] +381 if len(tmp_o['tag']) == 0: +382 del tmp_o['tag'] +383 dat = get_Array_from_dict(tmp_o) +384 my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)]) +385 if corr_tag != 'None': +386 my_corr.tag = corr_tag 387 -388 prog = json_dict.get('program', '') -389 version = json_dict.get('version', '') -390 who = json_dict.get('who', '') -391 date = json_dict.get('date', '') -392 host = json_dict.get('host', '') -393 if prog and verbose: -394 print('Data has been written using %s.' % (prog)) -395 if version and verbose: -396 print('Format version %s' % (version)) -397 if np.any([who, date, host] and verbose): -398 print('Written by %s on %s on host %s' % (who, date, host)) -399 description = json_dict.get('description', '') -400 if description and verbose: -401 print() -402 print('Description: ', description) -403 obsdata = json_dict['obsdata'] -404 ol = [] -405 for io in obsdata: -406 if io['type'] == 'Obs': -407 ol.append(get_Obs_from_dict(io)) -408 elif io['type'] == 'List': -409 ol.append(get_List_from_dict(io)) -410 elif io['type'] == 'Array': -411 ol.append(get_Array_from_dict(io)) -412 elif io['type'] == 'Corr': -413 ol.append(get_Corr_from_dict(io)) -414 else: -415 raise Exception("Unkown datatype.") -416 -417 if full_output: -418 retd = {} -419 retd['program'] = prog -420 retd['version'] = version -421 retd['who'] = who -422 retd['date'] = date -423 retd['host'] = host -424 retd['description'] = description -425 retd['obsdata'] = ol -426 -427 return retd -428 else: -429 if len(obsdata) == 1: -430 ol = ol[0] -431 -432 return ol -433 +388 my_corr.prange = temp_prange +389 return my_corr +390 +391 prog = json_dict.get('program', '') +392 version = json_dict.get('version', '') +393 who = json_dict.get('who', '') +394 date = json_dict.get('date', '') +395 host = json_dict.get('host', '') +396 if prog and verbose: +397 print('Data has been written using %s.' % (prog)) +398 if version and verbose: +399 print('Format version %s' % (version)) +400 if np.any([who, date, host] and verbose): +401 print('Written by %s on %s on host %s' % (who, date, host)) +402 description = json_dict.get('description', '') +403 if description and verbose: +404 print() +405 print('Description: ', description) +406 obsdata = json_dict['obsdata'] +407 ol = [] +408 for io in obsdata: +409 if io['type'] == 'Obs': +410 ol.append(get_Obs_from_dict(io)) +411 elif io['type'] == 'List': +412 ol.append(get_List_from_dict(io)) +413 elif io['type'] == 'Array': +414 ol.append(get_Array_from_dict(io)) +415 elif io['type'] == 'Corr': +416 ol.append(get_Corr_from_dict(io)) +417 else: +418 raise Exception("Unkown datatype.") +419 +420 if full_output: +421 retd = {} +422 retd['program'] = prog +423 retd['version'] = version +424 retd['who'] = who +425 retd['date'] = date +426 retd['host'] = host +427 retd['description'] = description +428 retd['obsdata'] = ol +429 +430 return retd +431 else: +432 if len(obsdata) == 1: +433 ol = ol[0] 434 -435def import_json_string(json_string, verbose=True, full_output=False): -436 """Reconstruct a list of Obs or structures containing Obs from a json string. +435 return ol +436 437 -438 The following structures are supported: Obs, list, numpy.ndarray, Corr -439 If the list contains only one element, it is unpacked from the list. +438def import_json_string(json_string, verbose=True, full_output=False): +439 """Reconstruct a list of Obs or structures containing Obs from a json string. 440 -441 Parameters -442 ---------- -443 json_string : str -444 json string containing the data. -445 verbose : bool -446 Print additional information that was written to the file. -447 full_output : bool -448 If True, a dict containing auxiliary information and the data is returned. -449 If False, only the data is returned. -450 """ -451 -452 return _parse_json_dict(json.loads(json_string), verbose, full_output) -453 +441 The following structures are supported: Obs, list, numpy.ndarray, Corr +442 If the list contains only one element, it is unpacked from the list. +443 +444 Parameters +445 ---------- +446 json_string : str +447 json string containing the data. +448 verbose : bool +449 Print additional information that was written to the file. +450 full_output : bool +451 If True, a dict containing auxiliary information and the data is returned. +452 If False, only the data is returned. +453 """ 454 -455def load_json(fname, verbose=True, gz=True, full_output=False): -456 """Import a list of Obs or structures containing Obs from a .json(.gz) file. +455 return _parse_json_dict(json.loads(json_string), verbose, full_output) +456 457 -458 The following structures are supported: Obs, list, numpy.ndarray, Corr -459 If the list contains only one element, it is unpacked from the list. +458def load_json(fname, verbose=True, gz=True, full_output=False): +459 """Import a list of Obs or structures containing Obs from a .json(.gz) file. 460 -461 Parameters -462 ---------- -463 fname : str -464 Filename of the input file. -465 verbose : bool -466 Print additional information that was written to the file. -467 gz : bool -468 If True, assumes that data is gzipped. If False, assumes JSON file. -469 full_output : bool -470 If True, a dict containing auxiliary information and the data is returned. -471 If False, only the data is returned. -472 """ -473 if not fname.endswith('.json') and not fname.endswith('.gz'): -474 fname += '.json' -475 if gz: -476 if not fname.endswith('.gz'): -477 fname += '.gz' -478 with gzip.open(fname, 'r') as fin: -479 d = json.load(fin) -480 else: -481 if fname.endswith('.gz'): -482 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) -483 with open(fname, 'r', encoding='utf-8') as fin: -484 d = json.loads(fin.read()) -485 -486 return _parse_json_dict(d, verbose, full_output) -487 +461 The following structures are supported: Obs, list, numpy.ndarray, Corr +462 If the list contains only one element, it is unpacked from the list. +463 +464 Parameters +465 ---------- +466 fname : str +467 Filename of the input file. +468 verbose : bool +469 Print additional information that was written to the file. +470 gz : bool +471 If True, assumes that data is gzipped. If False, assumes JSON file. +472 full_output : bool +473 If True, a dict containing auxiliary information and the data is returned. +474 If False, only the data is returned. +475 """ +476 if not fname.endswith('.json') and not fname.endswith('.gz'): +477 fname += '.json' +478 if gz: +479 if not fname.endswith('.gz'): +480 fname += '.gz' +481 with gzip.open(fname, 'r') as fin: +482 d = json.load(fin) +483 else: +484 if fname.endswith('.gz'): +485 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) +486 with open(fname, 'r', encoding='utf-8') as fin: +487 d = json.loads(fin.read()) 488 -489def _ol_from_dict(ind, reps='DICTOBS'): -490 """Convert a dictionary of Obs objects to a list and a dictionary that contains -491 placeholders instead of the Obs objects. -492 -493 Parameters -494 ---------- -495 ind : dict -496 Dict of JSON valid structures and objects that will be exported. -497 At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr. -498 All Obs inside a structure have to be defined on the same set of configurations. -499 reps : str -500 Specify the structure of the placeholder in exported dict to be reps[0-9]+. -501 """ -502 -503 obstypes = (Obs, Corr, np.ndarray) -504 -505 if not reps.isalnum(): -506 raise Exception('Placeholder string has to be alphanumeric!') -507 ol = [] -508 counter = 0 -509 -510 def dict_replace_obs(d): -511 nonlocal ol -512 nonlocal counter -513 x = {} -514 for k, v in d.items(): -515 if isinstance(v, dict): -516 v = dict_replace_obs(v) -517 elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]): -518 v = obslist_replace_obs(v) -519 elif isinstance(v, list): -520 v = list_replace_obs(v) -521 elif isinstance(v, obstypes): -522 ol.append(v) -523 v = reps + '%d' % (counter) -524 counter += 1 -525 elif isinstance(v, str): -526 if bool(re.match(r'%s[0-9]+' % (reps), v)): -527 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps)) -528 x[k] = v -529 return x -530 -531 def list_replace_obs(li): -532 nonlocal ol -533 nonlocal counter -534 x = [] -535 for e in li: -536 if isinstance(e, list): -537 e = list_replace_obs(e) -538 elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]): -539 e = obslist_replace_obs(e) -540 elif isinstance(e, dict): -541 e = dict_replace_obs(e) -542 elif isinstance(e, obstypes): -543 ol.append(e) -544 e = reps + '%d' % (counter) -545 counter += 1 -546 elif isinstance(e, str): -547 if bool(re.match(r'%s[0-9]+' % (reps), e)): -548 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps)) -549 x.append(e) -550 return x -551 -552 def obslist_replace_obs(li): -553 nonlocal ol -554 nonlocal counter -555 il = [] -556 for e in li: -557 il.append(e) -558 -559 ol.append(il) -560 x = reps + '%d' % (counter) -561 counter += 1 -562 return x -563 -564 nd = dict_replace_obs(ind) -565 -566 return ol, nd -567 +489 return _parse_json_dict(d, verbose, full_output) +490 +491 +492def _ol_from_dict(ind, reps='DICTOBS'): +493 """Convert a dictionary of Obs objects to a list and a dictionary that contains +494 placeholders instead of the Obs objects. +495 +496 Parameters +497 ---------- +498 ind : dict +499 Dict of JSON valid structures and objects that will be exported. +500 At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr. +501 All Obs inside a structure have to be defined on the same set of configurations. +502 reps : str +503 Specify the structure of the placeholder in exported dict to be reps[0-9]+. +504 """ +505 +506 obstypes = (Obs, Corr, np.ndarray) +507 +508 if not reps.isalnum(): +509 raise Exception('Placeholder string has to be alphanumeric!') +510 ol = [] +511 counter = 0 +512 +513 def dict_replace_obs(d): +514 nonlocal ol +515 nonlocal counter +516 x = {} +517 for k, v in d.items(): +518 if isinstance(v, dict): +519 v = dict_replace_obs(v) +520 elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]): +521 v = obslist_replace_obs(v) +522 elif isinstance(v, list): +523 v = list_replace_obs(v) +524 elif isinstance(v, obstypes): +525 ol.append(v) +526 v = reps + '%d' % (counter) +527 counter += 1 +528 elif isinstance(v, str): +529 if bool(re.match(r'%s[0-9]+' % (reps), v)): +530 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps)) +531 x[k] = v +532 return x +533 +534 def list_replace_obs(li): +535 nonlocal ol +536 nonlocal counter +537 x = [] +538 for e in li: +539 if isinstance(e, list): +540 e = list_replace_obs(e) +541 elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]): +542 e = obslist_replace_obs(e) +543 elif isinstance(e, dict): +544 e = dict_replace_obs(e) +545 elif isinstance(e, obstypes): +546 ol.append(e) +547 e = reps + '%d' % (counter) +548 counter += 1 +549 elif isinstance(e, str): +550 if bool(re.match(r'%s[0-9]+' % (reps), e)): +551 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps)) +552 x.append(e) +553 return x +554 +555 def obslist_replace_obs(li): +556 nonlocal ol +557 nonlocal counter +558 il = [] +559 for e in li: +560 il.append(e) +561 +562 ol.append(il) +563 x = reps + '%d' % (counter) +564 counter += 1 +565 return x +566 +567 nd = dict_replace_obs(ind) 568 -569def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): -570 """Export a dict of Obs or structures containing Obs to a .json(.gz) file +569 return ol, nd +570 571 -572 Parameters -573 ---------- -574 od : dict -575 Dict of JSON valid structures and objects that will be exported. -576 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. -577 All Obs inside a structure have to be defined on the same set of configurations. -578 fname : str -579 Filename of the output file. -580 description : str -581 Optional string that describes the contents of the json file. -582 indent : int -583 Specify the indentation level of the json file. None or 0 is permissible and -584 saves disk space. -585 reps : str -586 Specify the structure of the placeholder in exported dict to be reps[0-9]+. -587 gz : bool -588 If True, the output is a gzipped json. If False, the output is a json file. -589 """ -590 -591 if not isinstance(od, dict): -592 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') +572def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): +573 """Export a dict of Obs or structures containing Obs to a .json(.gz) file +574 +575 Parameters +576 ---------- +577 od : dict +578 Dict of JSON valid structures and objects that will be exported. +579 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. +580 All Obs inside a structure have to be defined on the same set of configurations. +581 fname : str +582 Filename of the output file. +583 description : str +584 Optional string that describes the contents of the json file. +585 indent : int +586 Specify the indentation level of the json file. None or 0 is permissible and +587 saves disk space. +588 reps : str +589 Specify the structure of the placeholder in exported dict to be reps[0-9]+. +590 gz : bool +591 If True, the output is a gzipped json. If False, the output is a json file. +592 """ 593 -594 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' -595 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' -596 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' -597 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') -598 -599 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} -600 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) +594 if not isinstance(od, dict): +595 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') +596 +597 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' +598 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' +599 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' +600 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') 601 -602 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz) -603 +602 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} +603 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) 604 -605def _od_from_list_and_dict(ol, ind, reps='DICTOBS'): -606 """Parse a list of Obs or structures containing Obs and an accompanying -607 dict, where the structures have been replaced by placeholders to a -608 dict that contains the structures. -609 -610 The following structures are supported: Obs, list, numpy.ndarray, Corr -611 -612 Parameters -613 ---------- -614 ol : list -615 List of objects - -616 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. -617 All Obs inside a structure have to be defined on the same set of configurations. -618 ind : dict -619 Dict that defines the structure of the resulting dict and contains placeholders -620 reps : str -621 Specify the structure of the placeholder in imported dict to be reps[0-9]+. -622 """ -623 if not reps.isalnum(): -624 raise Exception('Placeholder string has to be alphanumeric!') -625 -626 counter = 0 -627 -628 def dict_replace_string(d): -629 nonlocal counter -630 nonlocal ol -631 x = {} -632 for k, v in d.items(): -633 if isinstance(v, dict): -634 v = dict_replace_string(v) -635 elif isinstance(v, list): -636 v = list_replace_string(v) -637 elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)): -638 index = int(v[len(reps):]) -639 v = ol[index] -640 counter += 1 -641 x[k] = v -642 return x -643 -644 def list_replace_string(li): -645 nonlocal counter -646 nonlocal ol -647 x = [] -648 for e in li: -649 if isinstance(e, list): -650 e = list_replace_string(e) -651 elif isinstance(e, dict): -652 e = dict_replace_string(e) -653 elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)): -654 index = int(e[len(reps):]) -655 e = ol[index] -656 counter += 1 -657 x.append(e) -658 return x -659 -660 nd = dict_replace_string(ind) -661 -662 if counter == 0: -663 raise Exception('No placeholder has been replaced! Check if reps is set correctly.') +605 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz) +606 +607 +608def _od_from_list_and_dict(ol, ind, reps='DICTOBS'): +609 """Parse a list of Obs or structures containing Obs and an accompanying +610 dict, where the structures have been replaced by placeholders to a +611 dict that contains the structures. +612 +613 The following structures are supported: Obs, list, numpy.ndarray, Corr +614 +615 Parameters +616 ---------- +617 ol : list +618 List of objects - +619 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. +620 All Obs inside a structure have to be defined on the same set of configurations. +621 ind : dict +622 Dict that defines the structure of the resulting dict and contains placeholders +623 reps : str +624 Specify the structure of the placeholder in imported dict to be reps[0-9]+. +625 """ +626 if not reps.isalnum(): +627 raise Exception('Placeholder string has to be alphanumeric!') +628 +629 counter = 0 +630 +631 def dict_replace_string(d): +632 nonlocal counter +633 nonlocal ol +634 x = {} +635 for k, v in d.items(): +636 if isinstance(v, dict): +637 v = dict_replace_string(v) +638 elif isinstance(v, list): +639 v = list_replace_string(v) +640 elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)): +641 index = int(v[len(reps):]) +642 v = ol[index] +643 counter += 1 +644 x[k] = v +645 return x +646 +647 def list_replace_string(li): +648 nonlocal counter +649 nonlocal ol +650 x = [] +651 for e in li: +652 if isinstance(e, list): +653 e = list_replace_string(e) +654 elif isinstance(e, dict): +655 e = dict_replace_string(e) +656 elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)): +657 index = int(e[len(reps):]) +658 e = ol[index] +659 counter += 1 +660 x.append(e) +661 return x +662 +663 nd = dict_replace_string(ind) 664 -665 return nd -666 +665 if counter == 0: +666 raise Exception('No placeholder has been replaced! Check if reps is set correctly.') 667 -668def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): -669 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. +668 return nd +669 670 -671 The following structures are supported: Obs, list, numpy.ndarray, Corr -672 -673 Parameters -674 ---------- -675 fname : str -676 Filename of the input file. -677 verbose : bool -678 Print additional information that was written to the file. -679 gz : bool -680 If True, assumes that data is gzipped. If False, assumes JSON file. -681 full_output : bool -682 If True, a dict containing auxiliary information and the data is returned. -683 If False, only the data is returned. -684 reps : str -685 Specify the structure of the placeholder in imported dict to be reps[0-9]+. -686 """ -687 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) -688 description = indata['description']['description'] -689 indict = indata['description']['OBSDICT'] -690 ol = indata['obsdata'] -691 od = _od_from_list_and_dict(ol, indict, reps=reps) -692 -693 if full_output: -694 indata['description'] = description -695 indata['obsdata'] = od -696 return indata -697 else: -698 return od +671def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): +672 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. +673 +674 The following structures are supported: Obs, list, numpy.ndarray, Corr +675 +676 Parameters +677 ---------- +678 fname : str +679 Filename of the input file. +680 verbose : bool +681 Print additional information that was written to the file. +682 gz : bool +683 If True, assumes that data is gzipped. If False, assumes JSON file. +684 full_output : bool +685 If True, a dict containing auxiliary information and the data is returned. +686 If False, only the data is returned. +687 reps : str +688 Specify the structure of the placeholder in imported dict to be reps[0-9]+. +689 """ +690 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) +691 description = indata['description']['description'] +692 indict = indata['description']['OBSDICT'] +693 ol = indata['obsdata'] +694 od = _od_from_list_and_dict(ol, indict, reps=reps) +695 +696 if full_output: +697 indata['description'] = description +698 indata['obsdata'] = od +699 return indata +700 else: +701 return od @@ -1081,24 +1084,24 @@ If True, the output is a gzipped json. If False, the output is a json file. -
436def import_json_string(json_string, verbose=True, full_output=False): -437 """Reconstruct a list of Obs or structures containing Obs from a json string. -438 -439 The following structures are supported: Obs, list, numpy.ndarray, Corr -440 If the list contains only one element, it is unpacked from the list. +@@ -1133,38 +1136,38 @@ If False, only the data is returned.439def import_json_string(json_string, verbose=True, full_output=False): +440 """Reconstruct a list of Obs or structures containing Obs from a json string. 441 -442 Parameters -443 ---------- -444 json_string : str -445 json string containing the data. -446 verbose : bool -447 Print additional information that was written to the file. -448 full_output : bool -449 If True, a dict containing auxiliary information and the data is returned. -450 If False, only the data is returned. -451 """ -452 -453 return _parse_json_dict(json.loads(json_string), verbose, full_output) +442 The following structures are supported: Obs, list, numpy.ndarray, Corr +443 If the list contains only one element, it is unpacked from the list. +444 +445 Parameters +446 ---------- +447 json_string : str +448 json string containing the data. +449 verbose : bool +450 Print additional information that was written to the file. +451 full_output : bool +452 If True, a dict containing auxiliary information and the data is returned. +453 If False, only the data is returned. +454 """ +455 +456 return _parse_json_dict(json.loads(json_string), verbose, full_output)
456def load_json(fname, verbose=True, gz=True, full_output=False): -457 """Import a list of Obs or structures containing Obs from a .json(.gz) file. -458 -459 The following structures are supported: Obs, list, numpy.ndarray, Corr -460 If the list contains only one element, it is unpacked from the list. +@@ -1201,40 +1204,40 @@ If False, only the data is returned.459def load_json(fname, verbose=True, gz=True, full_output=False): +460 """Import a list of Obs or structures containing Obs from a .json(.gz) file. 461 -462 Parameters -463 ---------- -464 fname : str -465 Filename of the input file. -466 verbose : bool -467 Print additional information that was written to the file. -468 gz : bool -469 If True, assumes that data is gzipped. If False, assumes JSON file. -470 full_output : bool -471 If True, a dict containing auxiliary information and the data is returned. -472 If False, only the data is returned. -473 """ -474 if not fname.endswith('.json') and not fname.endswith('.gz'): -475 fname += '.json' -476 if gz: -477 if not fname.endswith('.gz'): -478 fname += '.gz' -479 with gzip.open(fname, 'r') as fin: -480 d = json.load(fin) -481 else: -482 if fname.endswith('.gz'): -483 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) -484 with open(fname, 'r', encoding='utf-8') as fin: -485 d = json.loads(fin.read()) -486 -487 return _parse_json_dict(d, verbose, full_output) +462 The following structures are supported: Obs, list, numpy.ndarray, Corr +463 If the list contains only one element, it is unpacked from the list. +464 +465 Parameters +466 ---------- +467 fname : str +468 Filename of the input file. +469 verbose : bool +470 Print additional information that was written to the file. +471 gz : bool +472 If True, assumes that data is gzipped. If False, assumes JSON file. +473 full_output : bool +474 If True, a dict containing auxiliary information and the data is returned. +475 If False, only the data is returned. +476 """ +477 if not fname.endswith('.json') and not fname.endswith('.gz'): +478 fname += '.json' +479 if gz: +480 if not fname.endswith('.gz'): +481 fname += '.gz' +482 with gzip.open(fname, 'r') as fin: +483 d = json.load(fin) +484 else: +485 if fname.endswith('.gz'): +486 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) +487 with open(fname, 'r', encoding='utf-8') as fin: +488 d = json.loads(fin.read()) +489 +490 return _parse_json_dict(d, verbose, full_output)
570def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): -571 """Export a dict of Obs or structures containing Obs to a .json(.gz) file -572 -573 Parameters -574 ---------- -575 od : dict -576 Dict of JSON valid structures and objects that will be exported. -577 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. -578 All Obs inside a structure have to be defined on the same set of configurations. -579 fname : str -580 Filename of the output file. -581 description : str -582 Optional string that describes the contents of the json file. -583 indent : int -584 Specify the indentation level of the json file. None or 0 is permissible and -585 saves disk space. -586 reps : str -587 Specify the structure of the placeholder in exported dict to be reps[0-9]+. -588 gz : bool -589 If True, the output is a gzipped json. If False, the output is a json file. -590 """ -591 -592 if not isinstance(od, dict): -593 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') +@@ -1274,37 +1277,37 @@ If True, the output is a gzipped json. If False, the output is a json file.573def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): +574 """Export a dict of Obs or structures containing Obs to a .json(.gz) file +575 +576 Parameters +577 ---------- +578 od : dict +579 Dict of JSON valid structures and objects that will be exported. +580 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. +581 All Obs inside a structure have to be defined on the same set of configurations. +582 fname : str +583 Filename of the output file. +584 description : str +585 Optional string that describes the contents of the json file. +586 indent : int +587 Specify the indentation level of the json file. None or 0 is permissible and +588 saves disk space. +589 reps : str +590 Specify the structure of the placeholder in exported dict to be reps[0-9]+. +591 gz : bool +592 If True, the output is a gzipped json. If False, the output is a json file. +593 """ 594 -595 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' -596 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' -597 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' -598 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') -599 -600 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} -601 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) +595 if not isinstance(od, dict): +596 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') +597 +598 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' +599 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' +600 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' +601 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') 602 -603 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz) +603 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} +604 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) +605 +606 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
669def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): -670 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. -671 -672 The following structures are supported: Obs, list, numpy.ndarray, Corr -673 -674 Parameters -675 ---------- -676 fname : str -677 Filename of the input file. -678 verbose : bool -679 Print additional information that was written to the file. -680 gz : bool -681 If True, assumes that data is gzipped. If False, assumes JSON file. -682 full_output : bool -683 If True, a dict containing auxiliary information and the data is returned. -684 If False, only the data is returned. -685 reps : str -686 Specify the structure of the placeholder in imported dict to be reps[0-9]+. -687 """ -688 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) -689 description = indata['description']['description'] -690 indict = indata['description']['OBSDICT'] -691 ol = indata['obsdata'] -692 od = _od_from_list_and_dict(ol, indict, reps=reps) -693 -694 if full_output: -695 indata['description'] = description -696 indata['obsdata'] = od -697 return indata -698 else: -699 return od +672def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): +673 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. +674 +675 The following structures are supported: Obs, list, numpy.ndarray, Corr +676 +677 Parameters +678 ---------- +679 fname : str +680 Filename of the input file. +681 verbose : bool +682 Print additional information that was written to the file. +683 gz : bool +684 If True, assumes that data is gzipped. If False, assumes JSON file. +685 full_output : bool +686 If True, a dict containing auxiliary information and the data is returned. +687 If False, only the data is returned. +688 reps : str +689 Specify the structure of the placeholder in imported dict to be reps[0-9]+. +690 """ +691 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) +692 description = indata['description']['description'] +693 indict = indata['description']['OBSDICT'] +694 ol = indata['obsdata'] +695 od = _od_from_list_and_dict(ol, indict, reps=reps) +696 +697 if full_output: +698 indata['description'] = description +699 indata['obsdata'] = od +700 return indata +701 else: +702 return od