diff --git a/docs/pyerrors/input/json.html b/docs/pyerrors/input/json.html index 93a60603..b6260c13 100644 --- a/docs/pyerrors/input/json.html +++ b/docs/pyerrors/input/json.html @@ -572,277 +572,276 @@ 479 result : dict 480 if full_output=True 481 """ -482 -483 return _parse_json_dict(json.loads(json_string), verbose, full_output) +482 return _parse_json_dict(json.loads(json_string), verbose, full_output) +483 484 -485 -486def load_json(fname, verbose=True, gz=True, full_output=False): -487 """Import a list of Obs or structures containing Obs from a .json(.gz) file. -488 -489 The following structures are supported: Obs, list, numpy.ndarray, Corr -490 If the list contains only one element, it is unpacked from the list. -491 -492 Parameters -493 ---------- -494 fname : str -495 Filename of the input file. -496 verbose : bool -497 Print additional information that was written to the file. -498 gz : bool -499 If True, assumes that data is gzipped. If False, assumes JSON file. -500 full_output : bool -501 If True, a dict containing auxiliary information and the data is returned. -502 If False, only the data is returned. -503 -504 Returns -505 ------- -506 result : list[Obs] -507 reconstructed list of observables from the json string -508 or -509 result : Obs -510 only one observable if the list only has one entry -511 or -512 result : dict -513 if full_output=True -514 """ -515 if not fname.endswith('.json') and not fname.endswith('.gz'): -516 fname += '.json' -517 if gz: -518 if not fname.endswith('.gz'): -519 fname += '.gz' -520 with gzip.open(fname, 'r') as fin: -521 d = json.load(fin) -522 else: -523 if fname.endswith('.gz'): -524 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) -525 with open(fname, 'r', encoding='utf-8') as fin: -526 d = json.loads(fin.read()) -527 -528 return _parse_json_dict(d, verbose, full_output) +485def load_json(fname, verbose=True, gz=True, full_output=False): +486 """Import a list of Obs or structures containing Obs from a .json(.gz) file. +487 +488 The following structures are supported: Obs, list, numpy.ndarray, Corr +489 If the list contains only one element, it is unpacked from the list. +490 +491 Parameters +492 ---------- +493 fname : str +494 Filename of the input file. +495 verbose : bool +496 Print additional information that was written to the file. +497 gz : bool +498 If True, assumes that data is gzipped. If False, assumes JSON file. +499 full_output : bool +500 If True, a dict containing auxiliary information and the data is returned. +501 If False, only the data is returned. +502 +503 Returns +504 ------- +505 result : list[Obs] +506 reconstructed list of observables from the json string +507 or +508 result : Obs +509 only one observable if the list only has one entry +510 or +511 result : dict +512 if full_output=True +513 """ +514 if not fname.endswith('.json') and not fname.endswith('.gz'): +515 fname += '.json' +516 if gz: +517 if not fname.endswith('.gz'): +518 fname += '.gz' +519 with gzip.open(fname, 'r') as fin: +520 d = json.load(fin) +521 else: +522 if fname.endswith('.gz'): +523 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) +524 with open(fname, 'r', encoding='utf-8') as fin: +525 d = json.loads(fin.read()) +526 +527 return _parse_json_dict(d, verbose, full_output) +528 529 -530 -531def _ol_from_dict(ind, reps='DICTOBS'): -532 """Convert a dictionary of Obs objects to a list and a dictionary that contains -533 placeholders instead of the Obs objects. -534 -535 Parameters -536 ---------- -537 ind : dict -538 Dict of JSON valid structures and objects that will be exported. -539 At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr. -540 All Obs inside a structure have to be defined on the same set of configurations. -541 reps : str -542 Specify the structure of the placeholder in exported dict to be reps[0-9]+. -543 """ -544 -545 obstypes = (Obs, Corr, np.ndarray) -546 -547 if not reps.isalnum(): -548 raise Exception('Placeholder string has to be alphanumeric!') -549 ol = [] -550 counter = 0 -551 -552 def dict_replace_obs(d): -553 nonlocal ol -554 nonlocal counter -555 x = {} -556 for k, v in d.items(): -557 if isinstance(v, dict): -558 v = dict_replace_obs(v) -559 elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]): -560 v = obslist_replace_obs(v) -561 elif isinstance(v, list): -562 v = list_replace_obs(v) -563 elif isinstance(v, obstypes): -564 ol.append(v) -565 v = reps + '%d' % (counter) -566 counter += 1 -567 elif isinstance(v, str): -568 if bool(re.match(r'%s[0-9]+' % (reps), v)): -569 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps)) -570 x[k] = v -571 return x -572 -573 def list_replace_obs(li): -574 nonlocal ol -575 nonlocal counter -576 x = [] -577 for e in li: -578 if isinstance(e, list): -579 e = list_replace_obs(e) -580 elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]): -581 e = obslist_replace_obs(e) -582 elif isinstance(e, dict): -583 e = dict_replace_obs(e) -584 elif isinstance(e, obstypes): -585 ol.append(e) -586 e = reps + '%d' % (counter) -587 counter += 1 -588 elif isinstance(e, str): -589 if bool(re.match(r'%s[0-9]+' % (reps), e)): -590 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps)) -591 x.append(e) -592 return x -593 -594 def obslist_replace_obs(li): -595 nonlocal ol -596 nonlocal counter -597 il = [] -598 for e in li: -599 il.append(e) -600 -601 ol.append(il) -602 x = reps + '%d' % (counter) -603 counter += 1 -604 return x -605 -606 nd = dict_replace_obs(ind) -607 -608 return ol, nd +530def _ol_from_dict(ind, reps='DICTOBS'): +531 """Convert a dictionary of Obs objects to a list and a dictionary that contains +532 placeholders instead of the Obs objects. +533 +534 Parameters +535 ---------- +536 ind : dict +537 Dict of JSON valid structures and objects that will be exported. +538 At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr. +539 All Obs inside a structure have to be defined on the same set of configurations. +540 reps : str +541 Specify the structure of the placeholder in exported dict to be reps[0-9]+. +542 """ +543 +544 obstypes = (Obs, Corr, np.ndarray) +545 +546 if not reps.isalnum(): +547 raise Exception('Placeholder string has to be alphanumeric!') +548 ol = [] +549 counter = 0 +550 +551 def dict_replace_obs(d): +552 nonlocal ol +553 nonlocal counter +554 x = {} +555 for k, v in d.items(): +556 if isinstance(v, dict): +557 v = dict_replace_obs(v) +558 elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]): +559 v = obslist_replace_obs(v) +560 elif isinstance(v, list): +561 v = list_replace_obs(v) +562 elif isinstance(v, obstypes): +563 ol.append(v) +564 v = reps + '%d' % (counter) +565 counter += 1 +566 elif isinstance(v, str): +567 if bool(re.match(r'%s[0-9]+' % (reps), v)): +568 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps)) +569 x[k] = v +570 return x +571 +572 def list_replace_obs(li): +573 nonlocal ol +574 nonlocal counter +575 x = [] +576 for e in li: +577 if isinstance(e, list): +578 e = list_replace_obs(e) +579 elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]): +580 e = obslist_replace_obs(e) +581 elif isinstance(e, dict): +582 e = dict_replace_obs(e) +583 elif isinstance(e, obstypes): +584 ol.append(e) +585 e = reps + '%d' % (counter) +586 counter += 1 +587 elif isinstance(e, str): +588 if bool(re.match(r'%s[0-9]+' % (reps), e)): +589 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps)) +590 x.append(e) +591 return x +592 +593 def obslist_replace_obs(li): +594 nonlocal ol +595 nonlocal counter +596 il = [] +597 for e in li: +598 il.append(e) +599 +600 ol.append(il) +601 x = reps + '%d' % (counter) +602 counter += 1 +603 return x +604 +605 nd = dict_replace_obs(ind) +606 +607 return ol, nd +608 609 -610 -611def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): -612 """Export a dict of Obs or structures containing Obs to a .json(.gz) file -613 -614 Parameters -615 ---------- -616 od : dict -617 Dict of JSON valid structures and objects that will be exported. -618 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. -619 All Obs inside a structure have to be defined on the same set of configurations. -620 fname : str -621 Filename of the output file. -622 description : str -623 Optional string that describes the contents of the json file. -624 indent : int -625 Specify the indentation level of the json file. None or 0 is permissible and -626 saves disk space. -627 reps : str -628 Specify the structure of the placeholder in exported dict to be reps[0-9]+. -629 gz : bool -630 If True, the output is a gzipped json. If False, the output is a json file. -631 -632 Returns -633 ------- -634 None -635 """ -636 -637 if not isinstance(od, dict): -638 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') -639 -640 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' -641 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' -642 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' -643 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') -644 -645 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} -646 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) -647 -648 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz) +610def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): +611 """Export a dict of Obs or structures containing Obs to a .json(.gz) file +612 +613 Parameters +614 ---------- +615 od : dict +616 Dict of JSON valid structures and objects that will be exported. +617 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. +618 All Obs inside a structure have to be defined on the same set of configurations. +619 fname : str +620 Filename of the output file. +621 description : str +622 Optional string that describes the contents of the json file. +623 indent : int +624 Specify the indentation level of the json file. None or 0 is permissible and +625 saves disk space. +626 reps : str +627 Specify the structure of the placeholder in exported dict to be reps[0-9]+. +628 gz : bool +629 If True, the output is a gzipped json. If False, the output is a json file. +630 +631 Returns +632 ------- +633 None +634 """ +635 +636 if not isinstance(od, dict): +637 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') +638 +639 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' +640 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' +641 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' +642 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') +643 +644 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} +645 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) +646 +647 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz) +648 649 -650 -651def _od_from_list_and_dict(ol, ind, reps='DICTOBS'): -652 """Parse a list of Obs or structures containing Obs and an accompanying -653 dict, where the structures have been replaced by placeholders to a -654 dict that contains the structures. -655 -656 The following structures are supported: Obs, list, numpy.ndarray, Corr -657 -658 Parameters -659 ---------- -660 ol : list -661 List of objects - -662 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. -663 All Obs inside a structure have to be defined on the same set of configurations. -664 ind : dict -665 Dict that defines the structure of the resulting dict and contains placeholders -666 reps : str -667 Specify the structure of the placeholder in imported dict to be reps[0-9]+. -668 """ -669 if not reps.isalnum(): -670 raise Exception('Placeholder string has to be alphanumeric!') -671 -672 counter = 0 -673 -674 def dict_replace_string(d): -675 nonlocal counter -676 nonlocal ol -677 x = {} -678 for k, v in d.items(): -679 if isinstance(v, dict): -680 v = dict_replace_string(v) -681 elif isinstance(v, list): -682 v = list_replace_string(v) -683 elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)): -684 index = int(v[len(reps):]) -685 v = ol[index] -686 counter += 1 -687 x[k] = v -688 return x -689 -690 def list_replace_string(li): -691 nonlocal counter -692 nonlocal ol -693 x = [] -694 for e in li: -695 if isinstance(e, list): -696 e = list_replace_string(e) -697 elif isinstance(e, dict): -698 e = dict_replace_string(e) -699 elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)): -700 index = int(e[len(reps):]) -701 e = ol[index] -702 counter += 1 -703 x.append(e) -704 return x -705 -706 nd = dict_replace_string(ind) -707 -708 if counter == 0: -709 raise Exception('No placeholder has been replaced! Check if reps is set correctly.') -710 -711 return nd +650def _od_from_list_and_dict(ol, ind, reps='DICTOBS'): +651 """Parse a list of Obs or structures containing Obs and an accompanying +652 dict, where the structures have been replaced by placeholders to a +653 dict that contains the structures. +654 +655 The following structures are supported: Obs, list, numpy.ndarray, Corr +656 +657 Parameters +658 ---------- +659 ol : list +660 List of objects - +661 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. +662 All Obs inside a structure have to be defined on the same set of configurations. +663 ind : dict +664 Dict that defines the structure of the resulting dict and contains placeholders +665 reps : str +666 Specify the structure of the placeholder in imported dict to be reps[0-9]+. +667 """ +668 if not reps.isalnum(): +669 raise Exception('Placeholder string has to be alphanumeric!') +670 +671 counter = 0 +672 +673 def dict_replace_string(d): +674 nonlocal counter +675 nonlocal ol +676 x = {} +677 for k, v in d.items(): +678 if isinstance(v, dict): +679 v = dict_replace_string(v) +680 elif isinstance(v, list): +681 v = list_replace_string(v) +682 elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)): +683 index = int(v[len(reps):]) +684 v = ol[index] +685 counter += 1 +686 x[k] = v +687 return x +688 +689 def list_replace_string(li): +690 nonlocal counter +691 nonlocal ol +692 x = [] +693 for e in li: +694 if isinstance(e, list): +695 e = list_replace_string(e) +696 elif isinstance(e, dict): +697 e = dict_replace_string(e) +698 elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)): +699 index = int(e[len(reps):]) +700 e = ol[index] +701 counter += 1 +702 x.append(e) +703 return x +704 +705 nd = dict_replace_string(ind) +706 +707 if counter == 0: +708 raise Exception('No placeholder has been replaced! Check if reps is set correctly.') +709 +710 return nd +711 712 -713 -714def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): -715 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. -716 -717 The following structures are supported: Obs, list, numpy.ndarray, Corr -718 -719 Parameters -720 ---------- -721 fname : str -722 Filename of the input file. -723 verbose : bool -724 Print additional information that was written to the file. -725 gz : bool -726 If True, assumes that data is gzipped. If False, assumes JSON file. -727 full_output : bool -728 If True, a dict containing auxiliary information and the data is returned. -729 If False, only the data is returned. -730 reps : str -731 Specify the structure of the placeholder in imported dict to be reps[0-9]+. -732 -733 Returns -734 ------- -735 data : Obs / list / Corr -736 Read data -737 or -738 data : dict -739 Read data and meta-data -740 """ -741 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) -742 description = indata['description']['description'] -743 indict = indata['description']['OBSDICT'] -744 ol = indata['obsdata'] -745 od = _od_from_list_and_dict(ol, indict, reps=reps) -746 -747 if full_output: -748 indata['description'] = description -749 indata['obsdata'] = od -750 return indata -751 else: -752 return od +713def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): +714 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. +715 +716 The following structures are supported: Obs, list, numpy.ndarray, Corr +717 +718 Parameters +719 ---------- +720 fname : str +721 Filename of the input file. +722 verbose : bool +723 Print additional information that was written to the file. +724 gz : bool +725 If True, assumes that data is gzipped. If False, assumes JSON file. +726 full_output : bool +727 If True, a dict containing auxiliary information and the data is returned. +728 If False, only the data is returned. +729 reps : str +730 Specify the structure of the placeholder in imported dict to be reps[0-9]+. +731 +732 Returns +733 ------- +734 data : Obs / list / Corr +735 Read data +736 or +737 data : dict +738 Read data and meta-data +739 """ +740 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) +741 description = indata['description']['description'] +742 indict = indata['description']['OBSDICT'] +743 ol = indata['obsdata'] +744 od = _od_from_list_and_dict(ol, indict, reps=reps) +745 +746 if full_output: +747 indata['description'] = description +748 indata['obsdata'] = od +749 return indata +750 else: +751 return od @@ -1191,8 +1190,7 @@ If True, the output is a gzipped json. If False, the output is a json file. 480 result : dict 481 if full_output=True 482 """ -483 -484 return _parse_json_dict(json.loads(json_string), verbose, full_output) +483 return _parse_json_dict(json.loads(json_string), verbose, full_output) @@ -1240,49 +1238,49 @@ if full_output=True -
487def load_json(fname, verbose=True, gz=True, full_output=False): -488 """Import a list of Obs or structures containing Obs from a .json(.gz) file. -489 -490 The following structures are supported: Obs, list, numpy.ndarray, Corr -491 If the list contains only one element, it is unpacked from the list. -492 -493 Parameters -494 ---------- -495 fname : str -496 Filename of the input file. -497 verbose : bool -498 Print additional information that was written to the file. -499 gz : bool -500 If True, assumes that data is gzipped. If False, assumes JSON file. -501 full_output : bool -502 If True, a dict containing auxiliary information and the data is returned. -503 If False, only the data is returned. -504 -505 Returns -506 ------- -507 result : list[Obs] -508 reconstructed list of observables from the json string -509 or -510 result : Obs -511 only one observable if the list only has one entry -512 or -513 result : dict -514 if full_output=True -515 """ -516 if not fname.endswith('.json') and not fname.endswith('.gz'): -517 fname += '.json' -518 if gz: -519 if not fname.endswith('.gz'): -520 fname += '.gz' -521 with gzip.open(fname, 'r') as fin: -522 d = json.load(fin) -523 else: -524 if fname.endswith('.gz'): -525 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) -526 with open(fname, 'r', encoding='utf-8') as fin: -527 d = json.loads(fin.read()) -528 -529 return _parse_json_dict(d, verbose, full_output) +@@ -1332,44 +1330,44 @@ if full_output=True486def load_json(fname, verbose=True, gz=True, full_output=False): +487 """Import a list of Obs or structures containing Obs from a .json(.gz) file. +488 +489 The following structures are supported: Obs, list, numpy.ndarray, Corr +490 If the list contains only one element, it is unpacked from the list. +491 +492 Parameters +493 ---------- +494 fname : str +495 Filename of the input file. +496 verbose : bool +497 Print additional information that was written to the file. +498 gz : bool +499 If True, assumes that data is gzipped. If False, assumes JSON file. +500 full_output : bool +501 If True, a dict containing auxiliary information and the data is returned. +502 If False, only the data is returned. +503 +504 Returns +505 ------- +506 result : list[Obs] +507 reconstructed list of observables from the json string +508 or +509 result : Obs +510 only one observable if the list only has one entry +511 or +512 result : dict +513 if full_output=True +514 """ +515 if not fname.endswith('.json') and not fname.endswith('.gz'): +516 fname += '.json' +517 if gz: +518 if not fname.endswith('.gz'): +519 fname += '.gz' +520 with gzip.open(fname, 'r') as fin: +521 d = json.load(fin) +522 else: +523 if fname.endswith('.gz'): +524 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) +525 with open(fname, 'r', encoding='utf-8') as fin: +526 d = json.loads(fin.read()) +527 +528 return _parse_json_dict(d, verbose, full_output)
612def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): -613 """Export a dict of Obs or structures containing Obs to a .json(.gz) file -614 -615 Parameters -616 ---------- -617 od : dict -618 Dict of JSON valid structures and objects that will be exported. -619 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. -620 All Obs inside a structure have to be defined on the same set of configurations. -621 fname : str -622 Filename of the output file. -623 description : str -624 Optional string that describes the contents of the json file. -625 indent : int -626 Specify the indentation level of the json file. None or 0 is permissible and -627 saves disk space. -628 reps : str -629 Specify the structure of the placeholder in exported dict to be reps[0-9]+. -630 gz : bool -631 If True, the output is a gzipped json. If False, the output is a json file. -632 -633 Returns -634 ------- -635 None -636 """ -637 -638 if not isinstance(od, dict): -639 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') -640 -641 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' -642 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' -643 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' -644 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') -645 -646 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} -647 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) -648 -649 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz) +@@ -1415,45 +1413,45 @@ If True, the output is a gzipped json. If False, the output is a json file.611def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): +612 """Export a dict of Obs or structures containing Obs to a .json(.gz) file +613 +614 Parameters +615 ---------- +616 od : dict +617 Dict of JSON valid structures and objects that will be exported. +618 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. +619 All Obs inside a structure have to be defined on the same set of configurations. +620 fname : str +621 Filename of the output file. +622 description : str +623 Optional string that describes the contents of the json file. +624 indent : int +625 Specify the indentation level of the json file. None or 0 is permissible and +626 saves disk space. +627 reps : str +628 Specify the structure of the placeholder in exported dict to be reps[0-9]+. +629 gz : bool +630 If True, the output is a gzipped json. If False, the output is a json file. +631 +632 Returns +633 ------- +634 None +635 """ +636 +637 if not isinstance(od, dict): +638 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') +639 +640 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' +641 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' +642 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' +643 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') +644 +645 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} +646 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) +647 +648 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
715def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): -716 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. -717 -718 The following structures are supported: Obs, list, numpy.ndarray, Corr -719 -720 Parameters -721 ---------- -722 fname : str -723 Filename of the input file. -724 verbose : bool -725 Print additional information that was written to the file. -726 gz : bool -727 If True, assumes that data is gzipped. If False, assumes JSON file. -728 full_output : bool -729 If True, a dict containing auxiliary information and the data is returned. -730 If False, only the data is returned. -731 reps : str -732 Specify the structure of the placeholder in imported dict to be reps[0-9]+. -733 -734 Returns -735 ------- -736 data : Obs / list / Corr -737 Read data -738 or -739 data : dict -740 Read data and meta-data -741 """ -742 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) -743 description = indata['description']['description'] -744 indict = indata['description']['OBSDICT'] -745 ol = indata['obsdata'] -746 od = _od_from_list_and_dict(ol, indict, reps=reps) -747 -748 if full_output: -749 indata['description'] = description -750 indata['obsdata'] = od -751 return indata -752 else: -753 return od +diff --git a/docs/pyerrors/input/pandas.html b/docs/pyerrors/input/pandas.html index 345f182c..af0434d0 100644 --- a/docs/pyerrors/input/pandas.html +++ b/docs/pyerrors/input/pandas.html @@ -92,178 +92,198 @@ 5from ..obs import Obs 6from ..correlators import Corr 7from .json import create_json_string, import_json_string - 8 + 8import numpy as np 9 - 10def to_sql(df, table_name, db, if_exists='fail', gz=True, **kwargs): - 11 """Write DataFrame including Obs or Corr valued columns to sqlite database. - 12 - 13 Parameters - 14 ---------- - 15 df : pandas.DataFrame - 16 Dataframe to be written to the database. - 17 table_name : str - 18 Name of the table in the database. - 19 db : str - 20 Path to the sqlite database. - 21 if exists : str - 22 How to behave if table already exists. Options 'fail', 'replace', 'append'. - 23 gz : bool - 24 If True the json strings are gzipped. - 25 - 26 Returns - 27 ------- - 28 None - 29 """ - 30 se_df = _serialize_df(df, gz=gz) - 31 con = sqlite3.connect(db) - 32 se_df.to_sql(table_name, con, if_exists=if_exists, index=False, **kwargs) - 33 con.close() - 34 + 10 + 11def to_sql(df, table_name, db, if_exists='fail', gz=True, **kwargs): + 12 """Write DataFrame including Obs or Corr valued columns to sqlite database. + 13 + 14 Parameters + 15 ---------- + 16 df : pandas.DataFrame + 17 Dataframe to be written to the database. + 18 table_name : str + 19 Name of the table in the database. + 20 db : str + 21 Path to the sqlite database. + 22 if exists : str + 23 How to behave if table already exists. Options 'fail', 'replace', 'append'. + 24 gz : bool + 25 If True the json strings are gzipped. + 26 + 27 Returns + 28 ------- + 29 None + 30 """ + 31 se_df = _serialize_df(df, gz=gz) + 32 con = sqlite3.connect(db) + 33 se_df.to_sql(table_name, con, if_exists=if_exists, index=False, **kwargs) + 34 con.close() 35 - 36def read_sql(sql, db, auto_gamma=False, **kwargs): - 37 """Execute SQL query on sqlite database and obtain DataFrame including Obs or Corr valued columns. - 38 - 39 Parameters - 40 ---------- - 41 sql : str - 42 SQL query to be executed. - 43 db : str - 44 Path to the sqlite database. - 45 auto_gamma : bool - 46 If True applies the gamma_method to all imported Obs objects with the default parameters for - 47 the error analysis. Default False. - 48 - 49 Returns - 50 ------- - 51 data : pandas.DataFrame - 52 Dataframe with the content of the sqlite database. - 53 """ - 54 con = sqlite3.connect(db) - 55 extract_df = pd.read_sql(sql, con, **kwargs) - 56 con.close() - 57 return _deserialize_df(extract_df, auto_gamma=auto_gamma) - 58 + 36 + 37def read_sql(sql, db, auto_gamma=False, **kwargs): + 38 """Execute SQL query on sqlite database and obtain DataFrame including Obs or Corr valued columns. + 39 + 40 Parameters + 41 ---------- + 42 sql : str + 43 SQL query to be executed. + 44 db : str + 45 Path to the sqlite database. + 46 auto_gamma : bool + 47 If True applies the gamma_method to all imported Obs objects with the default parameters for + 48 the error analysis. Default False. + 49 + 50 Returns + 51 ------- + 52 data : pandas.DataFrame + 53 Dataframe with the content of the sqlite database. + 54 """ + 55 con = sqlite3.connect(db) + 56 extract_df = pd.read_sql(sql, con, **kwargs) + 57 con.close() + 58 return _deserialize_df(extract_df, auto_gamma=auto_gamma) 59 - 60def dump_df(df, fname, gz=True): - 61 """Exports a pandas DataFrame containing Obs valued columns to a (gzipped) csv file. - 62 - 63 Before making use of pandas to_csv functionality Obs objects are serialized via the standardized - 64 json format of pyerrors. - 65 - 66 Parameters - 67 ---------- - 68 df : pandas.DataFrame - 69 Dataframe to be dumped to a file. - 70 fname : str - 71 Filename of the output file. - 72 gz : bool - 73 If True, the output is a gzipped csv file. If False, the output is a csv file. - 74 - 75 Returns - 76 ------- - 77 None - 78 """ - 79 out = _serialize_df(df, gz=False) - 80 - 81 if not fname.endswith('.csv'): - 82 fname += '.csv' - 83 - 84 if gz is True: - 85 if not fname.endswith('.gz'): - 86 fname += '.gz' - 87 out.to_csv(fname, index=False, compression='gzip') - 88 else: - 89 out.to_csv(fname, index=False) - 90 + 60 + 61def dump_df(df, fname, gz=True): + 62 """Exports a pandas DataFrame containing Obs valued columns to a (gzipped) csv file. + 63 + 64 Before making use of pandas to_csv functionality Obs objects are serialized via the standardized + 65 json format of pyerrors. + 66 + 67 Parameters + 68 ---------- + 69 df : pandas.DataFrame + 70 Dataframe to be dumped to a file. + 71 fname : str + 72 Filename of the output file. + 73 gz : bool + 74 If True, the output is a gzipped csv file. If False, the output is a csv file. + 75 + 76 Returns + 77 ------- + 78 None + 79 """ + 80 for column in df: + 81 serialize = _need_to_serialize(df[column]) + 82 if not serialize: + 83 if all(isinstance(entry, (int, np.integer, float, np.floating)) for entry in df[column]): + 84 if any([np.isnan(entry) for entry in df[column]]): + 85 warnings.warn("nan value in column " + column + " will be replaced by None", UserWarning) + 86 + 87 out = _serialize_df(df, gz=False) + 88 + 89 if not fname.endswith('.csv'): + 90 fname += '.csv' 91 - 92def load_df(fname, auto_gamma=False, gz=True): - 93 """Imports a pandas DataFrame from a csv.(gz) file in which Obs objects are serialized as json strings. - 94 - 95 Parameters - 96 ---------- - 97 fname : str - 98 Filename of the input file. - 99 auto_gamma : bool -100 If True applies the gamma_method to all imported Obs objects with the default parameters for -101 the error analysis. Default False. -102 gz : bool -103 If True, assumes that data is gzipped. If False, assumes JSON file. -104 -105 Returns -106 ------- -107 data : pandas.DataFrame -108 Dataframe with the content of the sqlite database. -109 """ -110 if not fname.endswith('.csv') and not fname.endswith('.gz'): -111 fname += '.csv' + 92 if gz is True: + 93 if not fname.endswith('.gz'): + 94 fname += '.gz' + 95 out.to_csv(fname, index=False, compression='gzip') + 96 else: + 97 out.to_csv(fname, index=False) + 98 + 99 +100def load_df(fname, auto_gamma=False, gz=True): +101 """Imports a pandas DataFrame from a csv.(gz) file in which Obs objects are serialized as json strings. +102 +103 Parameters +104 ---------- +105 fname : str +106 Filename of the input file. +107 auto_gamma : bool +108 If True applies the gamma_method to all imported Obs objects with the default parameters for +109 the error analysis. Default False. +110 gz : bool +111 If True, assumes that data is gzipped. If False, assumes JSON file. 112 -113 if gz is True: -114 if not fname.endswith('.gz'): -115 fname += '.gz' -116 with gzip.open(fname) as f: -117 re_import = pd.read_csv(f) -118 else: -119 if fname.endswith('.gz'): -120 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) -121 re_import = pd.read_csv(fname) -122 -123 return _deserialize_df(re_import, auto_gamma=auto_gamma) -124 -125 -126def _serialize_df(df, gz=False): -127 """Serializes all Obs or Corr valued columns into json strings according to the pyerrors json specification. -128 -129 Parameters -130 ---------- -131 df : pandas.DataFrame -132 DataFrame to be serilized. -133 gz: bool -134 gzip the json string representation. Default False. -135 """ -136 out = df.copy() -137 for column in out: -138 serialize = False -139 if isinstance(out[column][0], (Obs, Corr)): -140 serialize = True -141 elif isinstance(out[column][0], list): -142 if all(isinstance(o, Obs) for o in out[column][0]): -143 serialize = True -144 -145 if serialize is True: -146 out[column] = out[column].transform(lambda x: create_json_string(x, indent=0)) -147 if gz is True: -148 out[column] = out[column].transform(lambda x: gzip.compress(x.encode('utf-8'))) -149 return out -150 -151 -152def _deserialize_df(df, auto_gamma=False): -153 """Deserializes all pyerrors json strings into Obs or Corr objects according to the pyerrors json specification. +113 Returns +114 ------- +115 data : pandas.DataFrame +116 Dataframe with the content of the sqlite database. +117 """ +118 if not fname.endswith('.csv') and not fname.endswith('.gz'): +119 fname += '.csv' +120 +121 if gz is True: +122 if not fname.endswith('.gz'): +123 fname += '.gz' +124 with gzip.open(fname) as f: +125 re_import = pd.read_csv(f, keep_default_na=False) +126 else: +127 if fname.endswith('.gz'): +128 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) +129 re_import = pd.read_csv(fname, keep_default_na=False) +130 +131 return _deserialize_df(re_import, auto_gamma=auto_gamma) +132 +133 +134def _serialize_df(df, gz=False): +135 """Serializes all Obs or Corr valued columns into json strings according to the pyerrors json specification. +136 +137 Parameters +138 ---------- +139 df : pandas.DataFrame +140 DataFrame to be serilized. +141 gz: bool +142 gzip the json string representation. Default False. +143 """ +144 out = df.copy() +145 for column in out: +146 serialize = _need_to_serialize(out[column]) +147 +148 if serialize is True: +149 out[column] = out[column].transform(lambda x: create_json_string(x, indent=0) if x is not None else None) +150 if gz is True: +151 out[column] = out[column].transform(lambda x: gzip.compress((x if x is not None else '').encode('utf-8'))) +152 return out +153 154 -155 Parameters -156 ---------- -157 df : pandas.DataFrame -158 DataFrame to be deserilized. -159 auto_gamma : bool -160 If True applies the gamma_method to all imported Obs objects with the default parameters for -161 the error analysis. Default False. -162 -163 Notes: -164 ------ -165 In case any column of the DataFrame is gzipped it is gunzipped in the process. -166 """ -167 for column in df.select_dtypes(include="object"): -168 if isinstance(df[column][0], bytes): -169 if df[column][0].startswith(b"\x1f\x8b\x08\x00"): -170 df[column] = df[column].transform(lambda x: gzip.decompress(x).decode('utf-8')) -171 if isinstance(df[column][0], str): -172 if '"program":' in df[column][0][:20]: -173 df[column] = df[column].transform(lambda x: import_json_string(x, verbose=False)) -174 if auto_gamma is True: -175 if isinstance(df[column][0], list): -176 df[column].apply(lambda x: [o.gm() for o in x]) -177 else: -178 df[column].apply(lambda x: x.gamma_method()) -179 return df +155def _deserialize_df(df, auto_gamma=False): +156 """Deserializes all pyerrors json strings into Obs or Corr objects according to the pyerrors json specification. +157 +158 Parameters +159 ---------- +160 df : pandas.DataFrame +161 DataFrame to be deserilized. +162 auto_gamma : bool +163 If True applies the gamma_method to all imported Obs objects with the default parameters for +164 the error analysis. Default False. +165 +166 Notes: +167 ------ +168 In case any column of the DataFrame is gzipped it is gunzipped in the process. +169 """ +170 for column in df.select_dtypes(include="object"): +171 if isinstance(df[column][0], bytes): +172 if df[column][0].startswith(b"\x1f\x8b\x08\x00"): +173 df[column] = df[column].transform(lambda x: gzip.decompress(x).decode('utf-8')) +174 df = df.replace({r'^$': None}, regex=True) +175 i = 0 +176 while df[column][i] is None: +177 i += 1 +178 if isinstance(df[column][i], str): +179 if '"program":' in df[column][i][:20]: +180 df[column] = df[column].transform(lambda x: import_json_string(x, verbose=False) if x is not None else None) +181 if auto_gamma is True: +182 if isinstance(df[column][i], list): +183 df[column].apply(lambda x: [o.gm() if o is not None else x for o in x]) +184 else: +185 df[column].apply(lambda x: x.gm() if x is not None else x) +186 return df +187 +188 +189def _need_to_serialize(col): +190 serialize = False +191 i = 0 +192 while col[i] is None: +193 i += 1 +194 if isinstance(col[i], (Obs, Corr)): +195 serialize = True +196 elif isinstance(col[i], list): +197 if all(isinstance(o, Obs) for o in col[i]): +198 serialize = True +199 return serialize714def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): +715 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. +716 +717 The following structures are supported: Obs, list, numpy.ndarray, Corr +718 +719 Parameters +720 ---------- +721 fname : str +722 Filename of the input file. +723 verbose : bool +724 Print additional information that was written to the file. +725 gz : bool +726 If True, assumes that data is gzipped. If False, assumes JSON file. +727 full_output : bool +728 If True, a dict containing auxiliary information and the data is returned. +729 If False, only the data is returned. +730 reps : str +731 Specify the structure of the placeholder in imported dict to be reps[0-9]+. +732 +733 Returns +734 ------- +735 data : Obs / list / Corr +736 Read data +737 or +738 data : dict +739 Read data and meta-data +740 """ +741 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) +742 description = indata['description']['description'] +743 indict = indata['description']['OBSDICT'] +744 ol = indata['obsdata'] +745 od = _od_from_list_and_dict(ol, indict, reps=reps) +746 +747 if full_output: +748 indata['description'] = description +749 indata['obsdata'] = od +750 return indata +751 else: +752 return od
11def to_sql(df, table_name, db, if_exists='fail', gz=True, **kwargs): -12 """Write DataFrame including Obs or Corr valued columns to sqlite database. -13 -14 Parameters -15 ---------- -16 df : pandas.DataFrame -17 Dataframe to be written to the database. -18 table_name : str -19 Name of the table in the database. -20 db : str -21 Path to the sqlite database. -22 if exists : str -23 How to behave if table already exists. Options 'fail', 'replace', 'append'. -24 gz : bool -25 If True the json strings are gzipped. -26 -27 Returns -28 ------- -29 None -30 """ -31 se_df = _serialize_df(df, gz=gz) -32 con = sqlite3.connect(db) -33 se_df.to_sql(table_name, con, if_exists=if_exists, index=False, **kwargs) -34 con.close() +@@ -343,28 +363,28 @@ If True the json strings are gzipped.12def to_sql(df, table_name, db, if_exists='fail', gz=True, **kwargs): +13 """Write DataFrame including Obs or Corr valued columns to sqlite database. +14 +15 Parameters +16 ---------- +17 df : pandas.DataFrame +18 Dataframe to be written to the database. +19 table_name : str +20 Name of the table in the database. +21 db : str +22 Path to the sqlite database. +23 if exists : str +24 How to behave if table already exists. Options 'fail', 'replace', 'append'. +25 gz : bool +26 If True the json strings are gzipped. +27 +28 Returns +29 ------- +30 None +31 """ +32 se_df = _serialize_df(df, gz=gz) +33 con = sqlite3.connect(db) +34 se_df.to_sql(table_name, con, if_exists=if_exists, index=False, **kwargs) +35 con.close()
37def read_sql(sql, db, auto_gamma=False, **kwargs): -38 """Execute SQL query on sqlite database and obtain DataFrame including Obs or Corr valued columns. -39 -40 Parameters -41 ---------- -42 sql : str -43 SQL query to be executed. -44 db : str -45 Path to the sqlite database. -46 auto_gamma : bool -47 If True applies the gamma_method to all imported Obs objects with the default parameters for -48 the error analysis. Default False. -49 -50 Returns -51 ------- -52 data : pandas.DataFrame -53 Dataframe with the content of the sqlite database. -54 """ -55 con = sqlite3.connect(db) -56 extract_df = pd.read_sql(sql, con, **kwargs) -57 con.close() -58 return _deserialize_df(extract_df, auto_gamma=auto_gamma) +@@ -403,36 +423,43 @@ Dataframe with the content of the sqlite database.38def read_sql(sql, db, auto_gamma=False, **kwargs): +39 """Execute SQL query on sqlite database and obtain DataFrame including Obs or Corr valued columns. +40 +41 Parameters +42 ---------- +43 sql : str +44 SQL query to be executed. +45 db : str +46 Path to the sqlite database. +47 auto_gamma : bool +48 If True applies the gamma_method to all imported Obs objects with the default parameters for +49 the error analysis. Default False. +50 +51 Returns +52 ------- +53 data : pandas.DataFrame +54 Dataframe with the content of the sqlite database. +55 """ +56 con = sqlite3.connect(db) +57 extract_df = pd.read_sql(sql, con, **kwargs) +58 con.close() +59 return _deserialize_df(extract_df, auto_gamma=auto_gamma)
61def dump_df(df, fname, gz=True): -62 """Exports a pandas DataFrame containing Obs valued columns to a (gzipped) csv file. -63 -64 Before making use of pandas to_csv functionality Obs objects are serialized via the standardized -65 json format of pyerrors. -66 -67 Parameters -68 ---------- -69 df : pandas.DataFrame -70 Dataframe to be dumped to a file. -71 fname : str -72 Filename of the output file. -73 gz : bool -74 If True, the output is a gzipped csv file. If False, the output is a csv file. -75 -76 Returns -77 ------- -78 None -79 """ -80 out = _serialize_df(df, gz=False) -81 -82 if not fname.endswith('.csv'): -83 fname += '.csv' -84 -85 if gz is True: -86 if not fname.endswith('.gz'): -87 fname += '.gz' -88 out.to_csv(fname, index=False, compression='gzip') -89 else: -90 out.to_csv(fname, index=False) +@@ -472,38 +499,38 @@ If True, the output is a gzipped csv file. If False, the output is a csv file.62def dump_df(df, fname, gz=True): +63 """Exports a pandas DataFrame containing Obs valued columns to a (gzipped) csv file. +64 +65 Before making use of pandas to_csv functionality Obs objects are serialized via the standardized +66 json format of pyerrors. +67 +68 Parameters +69 ---------- +70 df : pandas.DataFrame +71 Dataframe to be dumped to a file. +72 fname : str +73 Filename of the output file. +74 gz : bool +75 If True, the output is a gzipped csv file. If False, the output is a csv file. +76 +77 Returns +78 ------- +79 None +80 """ +81 for column in df: +82 serialize = _need_to_serialize(df[column]) +83 if not serialize: +84 if all(isinstance(entry, (int, np.integer, float, np.floating)) for entry in df[column]): +85 if any([np.isnan(entry) for entry in df[column]]): +86 warnings.warn("nan value in column " + column + " will be replaced by None", UserWarning) +87 +88 out = _serialize_df(df, gz=False) +89 +90 if not fname.endswith('.csv'): +91 fname += '.csv' +92 +93 if gz is True: +94 if not fname.endswith('.gz'): +95 fname += '.gz' +96 out.to_csv(fname, index=False, compression='gzip') +97 else: +98 out.to_csv(fname, index=False)
93def load_df(fname, auto_gamma=False, gz=True): - 94 """Imports a pandas DataFrame from a csv.(gz) file in which Obs objects are serialized as json strings. - 95 - 96 Parameters - 97 ---------- - 98 fname : str - 99 Filename of the input file. -100 auto_gamma : bool -101 If True applies the gamma_method to all imported Obs objects with the default parameters for -102 the error analysis. Default False. -103 gz : bool -104 If True, assumes that data is gzipped. If False, assumes JSON file. -105 -106 Returns -107 ------- -108 data : pandas.DataFrame -109 Dataframe with the content of the sqlite database. -110 """ -111 if not fname.endswith('.csv') and not fname.endswith('.gz'): -112 fname += '.csv' +101def load_df(fname, auto_gamma=False, gz=True): +102 """Imports a pandas DataFrame from a csv.(gz) file in which Obs objects are serialized as json strings. +103 +104 Parameters +105 ---------- +106 fname : str +107 Filename of the input file. +108 auto_gamma : bool +109 If True applies the gamma_method to all imported Obs objects with the default parameters for +110 the error analysis. Default False. +111 gz : bool +112 If True, assumes that data is gzipped. If False, assumes JSON file. 113 -114 if gz is True: -115 if not fname.endswith('.gz'): -116 fname += '.gz' -117 with gzip.open(fname) as f: -118 re_import = pd.read_csv(f) -119 else: -120 if fname.endswith('.gz'): -121 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) -122 re_import = pd.read_csv(fname) -123 -124 return _deserialize_df(re_import, auto_gamma=auto_gamma) +114 Returns +115 ------- +116 data : pandas.DataFrame +117 Dataframe with the content of the sqlite database. +118 """ +119 if not fname.endswith('.csv') and not fname.endswith('.gz'): +120 fname += '.csv' +121 +122 if gz is True: +123 if not fname.endswith('.gz'): +124 fname += '.gz' +125 with gzip.open(fname) as f: +126 re_import = pd.read_csv(f, keep_default_na=False) +127 else: +128 if fname.endswith('.gz'): +129 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) +130 re_import = pd.read_csv(fname, keep_default_na=False) +131 +132 return _deserialize_df(re_import, auto_gamma=auto_gamma)