# pyerrors.input.json API documentation

# pyerrors.input.json

View Source

      1import rapidjson as json
      2import gzip
      3import getpass
      4import socket
      5import datetime
      6import platform
      7import warnings
      8import re
      9import numpy as np
     10from ..obs import Obs
     11from ..covobs import Covobs
     12from ..correlators import Corr
     13from ..misc import _assert_equal_properties
     14from .. import version as pyerrorsversion
     15
     16
     17def create_json_string(ol, description='', indent=1):
     18    """Generate the string for the export of a list of Obs or structures containing Obs
     19    to a .json(.gz) file
     20
     21    Parameters
     22    ----------
     23    ol : list
     24        List of objects that will be exported. At the moment, these objects can be
     25        either of: Obs, list, numpy.ndarray, Corr.
     26        All Obs inside a structure have to be defined on the same set of configurations.
     27    description : str
     28        Optional string that describes the contents of the json file.
     29    indent : int
     30        Specify the indentation level of the json file. None or 0 is permissible and
     31        saves disk space.
     32
     33    Returns
     34    -------
     35    json_string : str
     36        String for export to .json(.gz) file
     37    """
     38
     39    def _gen_data_d_from_list(ol):
     40        dl = []
     41        No = len(ol)
     42        for name in ol[0].mc_names:
     43            ed = {}
     44            ed['id'] = name
     45            ed['replica'] = []
     46            for r_name in ol[0].e_content[name]:
     47                rd = {}
     48                rd['name'] = r_name
     49                rd['deltas'] = []
     50                offsets = [o.r_values[r_name] - o.value for o in ol]
     51                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
     52                for i in range(len(ol[0].idl[r_name])):
     53                    rd['deltas'].append([ol[0].idl[r_name][i]])
     54                    rd['deltas'][-1] += deltas[i].tolist()
     55                ed['replica'].append(rd)
     56            dl.append(ed)
     57        return dl
     58
     59    def _gen_cdata_d_from_list(ol):
     60        dl = []
     61        for name in ol[0].cov_names:
     62            ed = {}
     63            ed['id'] = name
     64            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
     65            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
     66            ncov = ol[0].covobs[name].cov.shape[0]
     67            ed['grad'] = []
     68            for i in range(ncov):
     69                ed['grad'].append([])
     70                for o in ol:
     71                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
     72            dl.append(ed)
     73        return dl
     74
     75    def write_Obs_to_dict(o):
     76        d = {}
     77        d['type'] = 'Obs'
     78        d['layout'] = '1'
     79        if o.tag:
     80            d['tag'] = [o.tag]
     81        if o.reweighted:
     82            d['reweighted'] = o.reweighted
     83        d['value'] = [o.value]
     84        data = _gen_data_d_from_list([o])
     85        if len(data) > 0:
     86            d['data'] = data
     87        cdata = _gen_cdata_d_from_list([o])
     88        if len(cdata) > 0:
     89            d['cdata'] = cdata
     90        return d
     91
     92    def write_List_to_dict(ol):
     93        _assert_equal_properties(ol)
     94        d = {}
     95        d['type'] = 'List'
     96        d['layout'] = '%d' % len(ol)
     97        taglist = [o.tag for o in ol]
     98        if np.any([tag is not None for tag in taglist]):
     99            d['tag'] = taglist
    100        if ol[0].reweighted:
    101            d['reweighted'] = ol[0].reweighted
    102        d['value'] = [o.value for o in ol]
    103        data = _gen_data_d_from_list(ol)
    104        if len(data) > 0:
    105            d['data'] = data
    106        cdata = _gen_cdata_d_from_list(ol)
    107        if len(cdata) > 0:
    108            d['cdata'] = cdata
    109        return d
    110
    111    def write_Array_to_dict(oa):
    112        ol = np.ravel(oa)
    113        _assert_equal_properties(ol)
    114        d = {}
    115        d['type'] = 'Array'
    116        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
    117        taglist = [o.tag for o in ol]
    118        if np.any([tag is not None for tag in taglist]):
    119            d['tag'] = taglist
    120        if ol[0].reweighted:
    121            d['reweighted'] = ol[0].reweighted
    122        d['value'] = [o.value for o in ol]
    123        data = _gen_data_d_from_list(ol)
    124        if len(data) > 0:
    125            d['data'] = data
    126        cdata = _gen_cdata_d_from_list(ol)
    127        if len(cdata) > 0:
    128            d['cdata'] = cdata
    129        return d
    130
    131    def _nan_Obs_like(obs):
    132        samples = []
    133        names = []
    134        idl = []
    135        for key, value in obs.idl.items():
    136            samples.append([np.nan] * len(value))
    137            names.append(key)
    138            idl.append(value)
    139        my_obs = Obs(samples, names, idl)
    140        my_obs._covobs = obs._covobs
    141        for name in obs._covobs:
    142            my_obs.names.append(name)
    143        my_obs.reweighted = obs.reweighted
    144        return my_obs
    145
    146    def write_Corr_to_dict(my_corr):
    147        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
    148        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
    149        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
    150        content = [o if o is not None else dummy_array for o in my_corr.content]
    151        dat = write_Array_to_dict(np.array(content, dtype=object))
    152        dat['type'] = 'Corr'
    153        corr_meta_data = str(my_corr.tag)
    154        if 'tag' in dat.keys():
    155            dat['tag'].append(corr_meta_data)
    156        else:
    157            dat['tag'] = [corr_meta_data]
    158        taglist = dat['tag']
    159        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
    160        dat['tag']['tag'] = taglist
    161        if my_corr.prange is not None:
    162            dat['tag']['prange'] = my_corr.prange
    163        return dat
    164
    165    if not isinstance(ol, list):
    166        ol = [ol]
    167
    168    d = {}
    169    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
    170    d['version'] = '1.1'
    171    d['who'] = getpass.getuser()
    172    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
    173    d['host'] = socket.gethostname() + ', ' + platform.platform()
    174
    175    if description:
    176        d['description'] = description
    177
    178    d['obsdata'] = []
    179    for io in ol:
    180        if isinstance(io, Obs):
    181            d['obsdata'].append(write_Obs_to_dict(io))
    182        elif isinstance(io, list):
    183            d['obsdata'].append(write_List_to_dict(io))
    184        elif isinstance(io, np.ndarray):
    185            d['obsdata'].append(write_Array_to_dict(io))
    186        elif isinstance(io, Corr):
    187            d['obsdata'].append(write_Corr_to_dict(io))
    188        else:
    189            raise Exception("Unkown datatype.")
    190
    191    def _jsonifier(obj):
    192        if isinstance(obj, dict):
    193            result = {}
    194            for key in obj:
    195                if key is True:
    196                    result['true'] = obj[key]
    197                elif key is False:
    198                    result['false'] = obj[key]
    199                elif key is None:
    200                    result['null'] = obj[key]
    201                elif isinstance(key, (int, float, np.floating, np.integer)):
    202                    result[str(key)] = obj[key]
    203                else:
    204                    raise TypeError('keys must be str, int, float, bool or None')
    205            return result
    206        elif isinstance(obj, np.integer):
    207            return int(obj)
    208        elif isinstance(obj, np.floating):
    209            return float(obj)
    210        else:
    211            raise ValueError('%r is not JSON serializable' % (obj,))
    212
    213    if indent:
    214        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY)
    215    else:
    216        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)
    217
    218
    219def dump_to_json(ol, fname, description='', indent=1, gz=True):
    220    """Export a list of Obs or structures containing Obs to a .json(.gz) file.
    221    Dict keys that are not JSON-serializable such as floats are converted to strings.
    222
    223    Parameters
    224    ----------
    225    ol : list
    226        List of objects that will be exported. At the moment, these objects can be
    227        either of: Obs, list, numpy.ndarray, Corr.
    228        All Obs inside a structure have to be defined on the same set of configurations.
    229    fname : str
    230        Filename of the output file.
    231    description : str
    232        Optional string that describes the contents of the json file.
    233    indent : int
    234        Specify the indentation level of the json file. None or 0 is permissible and
    235        saves disk space.
    236    gz : bool
    237        If True, the output is a gzipped json. If False, the output is a json file.
    238
    239    Returns
    240    -------
    241    Null
    242    """
    243
    244    jsonstring = create_json_string(ol, description, indent)
    245
    246    if not fname.endswith('.json') and not fname.endswith('.gz'):
    247        fname += '.json'
    248
    249    if gz:
    250        if not fname.endswith('.gz'):
    251            fname += '.gz'
    252
    253        fp = gzip.open(fname, 'wb')
    254        fp.write(jsonstring.encode('utf-8'))
    255    else:
    256        fp = open(fname, 'w', encoding='utf-8')
    257        fp.write(jsonstring)
    258    fp.close()
    259
    260
    261def _parse_json_dict(json_dict, verbose=True, full_output=False):
    262    """Reconstruct a list of Obs or structures containing Obs from a dict that
    263    was built out of a json string.
    264
    265    The following structures are supported: Obs, list, numpy.ndarray, Corr
    266    If the list contains only one element, it is unpacked from the list.
    267
    268    Parameters
    269    ----------
    270    json_string : str
    271        json string containing the data.
    272    verbose : bool
    273        Print additional information that was written to the file.
    274    full_output : bool
    275        If True, a dict containing auxiliary information and the data is returned.
    276        If False, only the data is returned.
    277
    278    Returns
    279    -------
    280    result : list[Obs]
    281        reconstructed list of observables from the json string
    282    or
    283    result : Obs
    284        only one observable if the list only has one entry
    285    or
    286    result : dict
    287        if full_output=True
    288    """
    289
    290    def _gen_obsd_from_datad(d):
    291        retd = {}
    292        if d:
    293            retd['names'] = []
    294            retd['idl'] = []
    295            retd['deltas'] = []
    296            for ens in d:
    297                for rep in ens['replica']:
    298                    rep_name = rep['name']
    299                    if len(rep_name) > len(ens["id"]):
    300                        if rep_name[len(ens["id"])] != "|":
    301                            tmp_list = list(rep_name)
    302                            tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):]
    303                            rep_name = ''.join(tmp_list)
    304                    retd['names'].append(rep_name)
    305                    retd['idl'].append([di[0] for di in rep['deltas']])
    306                    retd['deltas'].append(np.array([di[1:] for di in rep['deltas']]))
    307        return retd
    308
    309    def _gen_covobsd_from_cdatad(d):
    310        retd = {}
    311        for ens in d:
    312            retl = []
    313            name = ens['id']
    314            layouts = ens.get('layout', '1').strip()
    315            layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
    316            cov = np.reshape(ens['cov'], layout)
    317            grad = ens['grad']
    318            nobs = len(grad[0])
    319            for i in range(nobs):
    320                retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]})
    321            retd[name] = retl
    322        return retd
    323
    324    def get_Obs_from_dict(o):
    325        layouts = o.get('layout', '1').strip()
    326        if layouts != '1':
    327            raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning)
    328
    329        values = o['value']
    330        od = _gen_obsd_from_datad(o.get('data', {}))
    331        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    332
    333        if od:
    334            ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl'])
    335            ret._value = values[0]
    336        else:
    337            ret = Obs([], [], means=[])
    338            ret._value = values[0]
    339        for name in cd:
    340            co = cd[name][0]
    341            ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    342            ret.names.append(co['name'])
    343
    344        ret.reweighted = o.get('reweighted', False)
    345        ret.tag = o.get('tag', [None])[0]
    346        return ret
    347
    348    def get_List_from_dict(o):
    349        layouts = o.get('layout', '1').strip()
    350        layout = int(layouts)
    351        values = o['value']
    352        od = _gen_obsd_from_datad(o.get('data', {}))
    353        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    354
    355        ret = []
    356        taglist = o.get('tag', layout * [None])
    357        for i in range(layout):
    358            if od:
    359                ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl']))
    360                ret[-1]._value = values[i]
    361            else:
    362                ret.append(Obs([], [], means=[]))
    363                ret[-1]._value = values[i]
    364                print('Created Obs with means= ', values[i])
    365            for name in cd:
    366                co = cd[name][i]
    367                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    368                ret[-1].names.append(co['name'])
    369
    370            ret[-1].reweighted = o.get('reweighted', False)
    371            ret[-1].tag = taglist[i]
    372        return ret
    373
    374    def get_Array_from_dict(o):
    375        layouts = o.get('layout', '1').strip()
    376        layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
    377        N = np.prod(layout)
    378        values = o['value']
    379        od = _gen_obsd_from_datad(o.get('data', {}))
    380        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
    381
    382        ret = []
    383        taglist = o.get('tag', N * [None])
    384        for i in range(N):
    385            if od:
    386                ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl']))
    387                ret[-1]._value = values[i]
    388            else:
    389                ret.append(Obs([], [], means=[]))
    390                ret[-1]._value = values[i]
    391            for name in cd:
    392                co = cd[name][i]
    393                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
    394                ret[-1].names.append(co['name'])
    395            ret[-1].reweighted = o.get('reweighted', False)
    396            ret[-1].tag = taglist[i]
    397        return np.reshape(ret, layout)
    398
    399    def get_Corr_from_dict(o):
    400        if isinstance(o.get('tag'), list):  # supports the old way
    401            taglist = o.get('tag')  # This had to be modified to get the taglist from the dictionary
    402            temp_prange = None
    403        elif isinstance(o.get('tag'), dict):
    404            tagdic = o.get('tag')
    405            taglist = tagdic['tag']
    406            if 'prange' in tagdic:
    407                temp_prange = tagdic['prange']
    408            else:
    409                temp_prange = None
    410        else:
    411            raise Exception("The tag is not a list or dict")
    412
    413        corr_tag = taglist[-1]
    414        tmp_o = o
    415        tmp_o['tag'] = taglist[:-1]
    416        if len(tmp_o['tag']) == 0:
    417            del tmp_o['tag']
    418        dat = get_Array_from_dict(tmp_o)
    419        my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)])
    420        if corr_tag != 'None':
    421            my_corr.tag = corr_tag
    422
    423        my_corr.prange = temp_prange
    424        return my_corr
    425
    426    prog = json_dict.get('program', '')
    427    version = json_dict.get('version', '')
    428    who = json_dict.get('who', '')
    429    date = json_dict.get('date', '')
    430    host = json_dict.get('host', '')
    431    if prog and verbose:
    432        print('Data has been written using %s.' % (prog))
    433    if version and verbose:
    434        print('Format version %s' % (version))
    435    if np.any([who, date, host] and verbose):
    436        print('Written by %s on %s on host %s' % (who, date, host))
    437    description = json_dict.get('description', '')
    438    if description and verbose:
    439        print()
    440        print('Description: ', description)
    441    obsdata = json_dict['obsdata']
    442    ol = []
    443    for io in obsdata:
    444        if io['type'] == 'Obs':
    445            ol.append(get_Obs_from_dict(io))
    446        elif io['type'] == 'List':
    447            ol.append(get_List_from_dict(io))
    448        elif io['type'] == 'Array':
    449            ol.append(get_Array_from_dict(io))
    450        elif io['type'] == 'Corr':
    451            ol.append(get_Corr_from_dict(io))
    452        else:
    453            raise Exception("Unknown datatype.")
    454
    455    if full_output:
    456        retd = {}
    457        retd['program'] = prog
    458        retd['version'] = version
    459        retd['who'] = who
    460        retd['date'] = date
    461        retd['host'] = host
    462        retd['description'] = description
    463        retd['obsdata'] = ol
    464
    465        return retd
    466    else:
    467        if len(obsdata) == 1:
    468            ol = ol[0]
    469
    470        return ol
    471
    472
    473def import_json_string(json_string, verbose=True, full_output=False):
    474    """Reconstruct a list of Obs or structures containing Obs from a json string.
    475
    476    The following structures are supported: Obs, list, numpy.ndarray, Corr
    477    If the list contains only one element, it is unpacked from the list.
    478
    479    Parameters
    480    ----------
    481    json_string : str
    482        json string containing the data.
    483    verbose : bool
    484        Print additional information that was written to the file.
    485    full_output : bool
    486        If True, a dict containing auxiliary information and the data is returned.
    487        If False, only the data is returned.
    488
    489    Returns
    490    -------
    491    result : list[Obs]
    492        reconstructed list of observables from the json string
    493    or
    494    result : Obs
    495        only one observable if the list only has one entry
    496    or
    497    result : dict
    498        if full_output=True
    499    """
    500    return _parse_json_dict(json.loads(json_string), verbose, full_output)
    501
    502
    503def load_json(fname, verbose=True, gz=True, full_output=False):
    504    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
    505
    506    The following structures are supported: Obs, list, numpy.ndarray, Corr
    507    If the list contains only one element, it is unpacked from the list.
    508
    509    Parameters
    510    ----------
    511    fname : str
    512        Filename of the input file.
    513    verbose : bool
    514        Print additional information that was written to the file.
    515    gz : bool
    516        If True, assumes that data is gzipped. If False, assumes JSON file.
    517    full_output : bool
    518        If True, a dict containing auxiliary information and the data is returned.
    519        If False, only the data is returned.
    520
    521    Returns
    522    -------
    523    result : list[Obs]
    524        reconstructed list of observables from the json string
    525    or
    526    result : Obs
    527        only one observable if the list only has one entry
    528    or
    529    result : dict
    530        if full_output=True
    531    """
    532    if not fname.endswith('.json') and not fname.endswith('.gz'):
    533        fname += '.json'
    534    if gz:
    535        if not fname.endswith('.gz'):
    536            fname += '.gz'
    537        with gzip.open(fname, 'r') as fin:
    538            d = json.load(fin)
    539    else:
    540        if fname.endswith('.gz'):
    541            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
    542        with open(fname, 'r', encoding='utf-8') as fin:
    543            d = json.loads(fin.read())
    544
    545    return _parse_json_dict(d, verbose, full_output)
    546
    547
    548def _ol_from_dict(ind, reps='DICTOBS'):
    549    """Convert a dictionary of Obs objects to a list and a dictionary that contains
    550    placeholders instead of the Obs objects.
    551
    552    Parameters
    553    ----------
    554    ind : dict
    555        Dict of JSON valid structures and objects that will be exported.
    556        At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr.
    557        All Obs inside a structure have to be defined on the same set of configurations.
    558    reps : str
    559        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    560    """
    561
    562    obstypes = (Obs, Corr, np.ndarray)
    563
    564    if not reps.isalnum():
    565        raise Exception('Placeholder string has to be alphanumeric!')
    566    ol = []
    567    counter = 0
    568
    569    def dict_replace_obs(d):
    570        nonlocal ol
    571        nonlocal counter
    572        x = {}
    573        for k, v in d.items():
    574            if isinstance(v, dict):
    575                v = dict_replace_obs(v)
    576            elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]):
    577                v = obslist_replace_obs(v)
    578            elif isinstance(v, list):
    579                v = list_replace_obs(v)
    580            elif isinstance(v, obstypes):
    581                ol.append(v)
    582                v = reps + '%d' % (counter)
    583                counter += 1
    584            elif isinstance(v, str):
    585                if bool(re.match(r'%s[0-9]+' % (reps), v)):
    586                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be safely exported.' % (v, reps))
    587            x[k] = v
    588        return x
    589
    590    def list_replace_obs(li):
    591        nonlocal ol
    592        nonlocal counter
    593        x = []
    594        for e in li:
    595            if isinstance(e, list):
    596                e = list_replace_obs(e)
    597            elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]):
    598                e = obslist_replace_obs(e)
    599            elif isinstance(e, dict):
    600                e = dict_replace_obs(e)
    601            elif isinstance(e, obstypes):
    602                ol.append(e)
    603                e = reps + '%d' % (counter)
    604                counter += 1
    605            elif isinstance(e, str):
    606                if bool(re.match(r'%s[0-9]+' % (reps), e)):
    607                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be safely exported.' % (e, reps))
    608            x.append(e)
    609        return x
    610
    611    def obslist_replace_obs(li):
    612        nonlocal ol
    613        nonlocal counter
    614        il = []
    615        for e in li:
    616            il.append(e)
    617
    618        ol.append(il)
    619        x = reps + '%d' % (counter)
    620        counter += 1
    621        return x
    622
    623    nd = dict_replace_obs(ind)
    624
    625    return ol, nd
    626
    627
    628def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
    629    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
    630
    631    Parameters
    632    ----------
    633    od : dict
    634        Dict of JSON valid structures and objects that will be exported.
    635        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    636        All Obs inside a structure have to be defined on the same set of configurations.
    637    fname : str
    638        Filename of the output file.
    639    description : str
    640        Optional string that describes the contents of the json file.
    641    indent : int
    642        Specify the indentation level of the json file. None or 0 is permissible and
    643        saves disk space.
    644    reps : str
    645        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    646    gz : bool
    647        If True, the output is a gzipped json. If False, the output is a json file.
    648
    649    Returns
    650    -------
    651    None
    652    """
    653
    654    if not isinstance(od, dict):
    655        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
    656
    657    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
    658                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
    659                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
    660                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
    661
    662    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
    663    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
    664
    665    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
    666
    667
    668def _od_from_list_and_dict(ol, ind, reps='DICTOBS'):
    669    """Parse a list of Obs or structures containing Obs and an accompanying
    670    dict, where the structures have been replaced by placeholders to a
    671    dict that contains the structures.
    672
    673    The following structures are supported: Obs, list, numpy.ndarray, Corr
    674
    675    Parameters
    676    ----------
    677    ol : list
    678        List of objects -
    679        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    680        All Obs inside a structure have to be defined on the same set of configurations.
    681    ind : dict
    682        Dict that defines the structure of the resulting dict and contains placeholders
    683    reps : str
    684        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    685    """
    686    if not reps.isalnum():
    687        raise Exception('Placeholder string has to be alphanumeric!')
    688
    689    counter = 0
    690
    691    def dict_replace_string(d):
    692        nonlocal counter
    693        nonlocal ol
    694        x = {}
    695        for k, v in d.items():
    696            if isinstance(v, dict):
    697                v = dict_replace_string(v)
    698            elif isinstance(v, list):
    699                v = list_replace_string(v)
    700            elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)):
    701                index = int(v[len(reps):])
    702                v = ol[index]
    703                counter += 1
    704            x[k] = v
    705        return x
    706
    707    def list_replace_string(li):
    708        nonlocal counter
    709        nonlocal ol
    710        x = []
    711        for e in li:
    712            if isinstance(e, list):
    713                e = list_replace_string(e)
    714            elif isinstance(e, dict):
    715                e = dict_replace_string(e)
    716            elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)):
    717                index = int(e[len(reps):])
    718                e = ol[index]
    719                counter += 1
    720            x.append(e)
    721        return x
    722
    723    nd = dict_replace_string(ind)
    724
    725    if counter == 0:
    726        raise Exception('No placeholder has been replaced! Check if reps is set correctly.')
    727
    728    return nd
    729
    730
    731def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
    732    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
    733
    734    The following structures are supported: Obs, list, numpy.ndarray, Corr
    735
    736    Parameters
    737    ----------
    738    fname : str
    739        Filename of the input file.
    740    verbose : bool
    741        Print additional information that was written to the file.
    742    gz : bool
    743        If True, assumes that data is gzipped. If False, assumes JSON file.
    744    full_output : bool
    745        If True, a dict containing auxiliary information and the data is returned.
    746        If False, only the data is returned.
    747    reps : str
    748        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    749
    750    Returns
    751    -------
    752    data : Obs / list / Corr
    753        Read data
    754    or
    755    data : dict
    756        Read data and meta-data
    757    """
    758    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
    759    description = indata['description']['description']
    760    indict = indata['description']['OBSDICT']
    761    ol = indata['obsdata']
    762    od = _od_from_list_and_dict(ol, indict, reps=reps)
    763
    764    if full_output:
    765        indata['description'] = description
    766        indata['obsdata'] = od
    767        return indata
    768    else:
    769        return od

def create_json_string(ol, description='', indent=1): View Source

     18def create_json_string(ol, description='', indent=1):
     19    """Generate the string for the export of a list of Obs or structures containing Obs
     20    to a .json(.gz) file
     21
     22    Parameters
     23    ----------
     24    ol : list
     25        List of objects that will be exported. At the moment, these objects can be
     26        either of: Obs, list, numpy.ndarray, Corr.
     27        All Obs inside a structure have to be defined on the same set of configurations.
     28    description : str
     29        Optional string that describes the contents of the json file.
     30    indent : int
     31        Specify the indentation level of the json file. None or 0 is permissible and
     32        saves disk space.
     33
     34    Returns
     35    -------
     36    json_string : str
     37        String for export to .json(.gz) file
     38    """
     39
     40    def _gen_data_d_from_list(ol):
     41        dl = []
     42        No = len(ol)
     43        for name in ol[0].mc_names:
     44            ed = {}
     45            ed['id'] = name
     46            ed['replica'] = []
     47            for r_name in ol[0].e_content[name]:
     48                rd = {}
     49                rd['name'] = r_name
     50                rd['deltas'] = []
     51                offsets = [o.r_values[r_name] - o.value for o in ol]
     52                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
     53                for i in range(len(ol[0].idl[r_name])):
     54                    rd['deltas'].append([ol[0].idl[r_name][i]])
     55                    rd['deltas'][-1] += deltas[i].tolist()
     56                ed['replica'].append(rd)
     57            dl.append(ed)
     58        return dl
     59
     60    def _gen_cdata_d_from_list(ol):
     61        dl = []
     62        for name in ol[0].cov_names:
     63            ed = {}
     64            ed['id'] = name
     65            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
     66            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
     67            ncov = ol[0].covobs[name].cov.shape[0]
     68            ed['grad'] = []
     69            for i in range(ncov):
     70                ed['grad'].append([])
     71                for o in ol:
     72                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
     73            dl.append(ed)
     74        return dl
     75
     76    def write_Obs_to_dict(o):
     77        d = {}
     78        d['type'] = 'Obs'
     79        d['layout'] = '1'
     80        if o.tag:
     81            d['tag'] = [o.tag]
     82        if o.reweighted:
     83            d['reweighted'] = o.reweighted
     84        d['value'] = [o.value]
     85        data = _gen_data_d_from_list([o])
     86        if len(data) > 0:
     87            d['data'] = data
     88        cdata = _gen_cdata_d_from_list([o])
     89        if len(cdata) > 0:
     90            d['cdata'] = cdata
     91        return d
     92
     93    def write_List_to_dict(ol):
     94        _assert_equal_properties(ol)
     95        d = {}
     96        d['type'] = 'List'
     97        d['layout'] = '%d' % len(ol)
     98        taglist = [o.tag for o in ol]
     99        if np.any([tag is not None for tag in taglist]):
    100            d['tag'] = taglist
    101        if ol[0].reweighted:
    102            d['reweighted'] = ol[0].reweighted
    103        d['value'] = [o.value for o in ol]
    104        data = _gen_data_d_from_list(ol)
    105        if len(data) > 0:
    106            d['data'] = data
    107        cdata = _gen_cdata_d_from_list(ol)
    108        if len(cdata) > 0:
    109            d['cdata'] = cdata
    110        return d
    111
    112    def write_Array_to_dict(oa):
    113        ol = np.ravel(oa)
    114        _assert_equal_properties(ol)
    115        d = {}
    116        d['type'] = 'Array'
    117        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
    118        taglist = [o.tag for o in ol]
    119        if np.any([tag is not None for tag in taglist]):
    120            d['tag'] = taglist
    121        if ol[0].reweighted:
    122            d['reweighted'] = ol[0].reweighted
    123        d['value'] = [o.value for o in ol]
    124        data = _gen_data_d_from_list(ol)
    125        if len(data) > 0:
    126            d['data'] = data
    127        cdata = _gen_cdata_d_from_list(ol)
    128        if len(cdata) > 0:
    129            d['cdata'] = cdata
    130        return d
    131
    132    def _nan_Obs_like(obs):
    133        samples = []
    134        names = []
    135        idl = []
    136        for key, value in obs.idl.items():
    137            samples.append([np.nan] * len(value))
    138            names.append(key)
    139            idl.append(value)
    140        my_obs = Obs(samples, names, idl)
    141        my_obs._covobs = obs._covobs
    142        for name in obs._covobs:
    143            my_obs.names.append(name)
    144        my_obs.reweighted = obs.reweighted
    145        return my_obs
    146
    147    def write_Corr_to_dict(my_corr):
    148        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
    149        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
    150        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
    151        content = [o if o is not None else dummy_array for o in my_corr.content]
    152        dat = write_Array_to_dict(np.array(content, dtype=object))
    153        dat['type'] = 'Corr'
    154        corr_meta_data = str(my_corr.tag)
    155        if 'tag' in dat.keys():
    156            dat['tag'].append(corr_meta_data)
    157        else:
    158            dat['tag'] = [corr_meta_data]
    159        taglist = dat['tag']
    160        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
    161        dat['tag']['tag'] = taglist
    162        if my_corr.prange is not None:
    163            dat['tag']['prange'] = my_corr.prange
    164        return dat
    165
    166    if not isinstance(ol, list):
    167        ol = [ol]
    168
    169    d = {}
    170    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
    171    d['version'] = '1.1'
    172    d['who'] = getpass.getuser()
    173    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
    174    d['host'] = socket.gethostname() + ', ' + platform.platform()
    175
    176    if description:
    177        d['description'] = description
    178
    179    d['obsdata'] = []
    180    for io in ol:
    181        if isinstance(io, Obs):
    182            d['obsdata'].append(write_Obs_to_dict(io))
    183        elif isinstance(io, list):
    184            d['obsdata'].append(write_List_to_dict(io))
    185        elif isinstance(io, np.ndarray):
    186            d['obsdata'].append(write_Array_to_dict(io))
    187        elif isinstance(io, Corr):
    188            d['obsdata'].append(write_Corr_to_dict(io))
    189        else:
    190            raise Exception("Unkown datatype.")
    191
    192    def _jsonifier(obj):
    193        if isinstance(obj, dict):
    194            result = {}
    195            for key in obj:
    196                if key is True:
    197                    result['true'] = obj[key]
    198                elif key is False:
    199                    result['false'] = obj[key]
    200                elif key is None:
    201                    result['null'] = obj[key]
    202                elif isinstance(key, (int, float, np.floating, np.integer)):
    203                    result[str(key)] = obj[key]
    204                else:
    205                    raise TypeError('keys must be str, int, float, bool or None')
    206            return result
    207        elif isinstance(obj, np.integer):
    208            return int(obj)
    209        elif isinstance(obj, np.floating):
    210            return float(obj)
    211        else:
    212            raise ValueError('%r is not JSON serializable' % (obj,))
    213
    214    if indent:
    215        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY)
    216    else:
    217        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)

Generate the string for the export of a list of Obs or structures containing Obs to a pyerrors.input.json(.gz) file

###### Parameters

- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.

###### Returns

- json_string (str): String for export to pyerrors.input.json(.gz) file

def dump_to_json(ol, fname, description='', indent=1, gz=True): View Source

    220def dump_to_json(ol, fname, description='', indent=1, gz=True):
    221    """Export a list of Obs or structures containing Obs to a .json(.gz) file.
    222    Dict keys that are not JSON-serializable such as floats are converted to strings.
    223
    224    Parameters
    225    ----------
    226    ol : list
    227        List of objects that will be exported. At the moment, these objects can be
    228        either of: Obs, list, numpy.ndarray, Corr.
    229        All Obs inside a structure have to be defined on the same set of configurations.
    230    fname : str
    231        Filename of the output file.
    232    description : str
    233        Optional string that describes the contents of the json file.
    234    indent : int
    235        Specify the indentation level of the json file. None or 0 is permissible and
    236        saves disk space.
    237    gz : bool
    238        If True, the output is a gzipped json. If False, the output is a json file.
    239
    240    Returns
    241    -------
    242    Null
    243    """
    244
    245    jsonstring = create_json_string(ol, description, indent)
    246
    247    if not fname.endswith('.json') and not fname.endswith('.gz'):
    248        fname += '.json'
    249
    250    if gz:
    251        if not fname.endswith('.gz'):
    252            fname += '.gz'
    253
    254        fp = gzip.open(fname, 'wb')
    255        fp.write(jsonstring.encode('utf-8'))
    256    else:
    257        fp = open(fname, 'w', encoding='utf-8')
    258        fp.write(jsonstring)
    259    fp.close()

Export a list of Obs or structures containing Obs to a pyerrors.input.json(.gz) file. Dict keys that are not JSON-serializable such as floats are converted to strings.

###### Parameters

- ol (list): List of objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.

###### Returns

- Null

def import_json_string(json_string, verbose=True, full_output=False): View Source

    474def import_json_string(json_string, verbose=True, full_output=False):
    475    """Reconstruct a list of Obs or structures containing Obs from a json string.
    476
    477    The following structures are supported: Obs, list, numpy.ndarray, Corr
    478    If the list contains only one element, it is unpacked from the list.
    479
    480    Parameters
    481    ----------
    482    json_string : str
    483        json string containing the data.
    484    verbose : bool
    485        Print additional information that was written to the file.
    486    full_output : bool
    487        If True, a dict containing auxiliary information and the data is returned.
    488        If False, only the data is returned.
    489
    490    Returns
    491    -------
    492    result : list[Obs]
    493        reconstructed list of observables from the json string
    494    or
    495    result : Obs
    496        only one observable if the list only has one entry
    497    or
    498    result : dict
    499        if full_output=True
    500    """
    501    return _parse_json_dict(json.loads(json_string), verbose, full_output)

Reconstruct a list of Obs or structures containing Obs from a json string.

The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.

###### Parameters

- json_string (str): json string containing the data.
- verbose (bool): Print additional information that was written to the file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.

###### Returns

- result (list[Obs]): reconstructed list of observables from the json string
- or
- result (Obs): only one observable if the list only has one entry
- or
- result (dict): if full_output=True

def load_json(fname, verbose=True, gz=True, full_output=False): View Source

    504def load_json(fname, verbose=True, gz=True, full_output=False):
    505    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
    506
    507    The following structures are supported: Obs, list, numpy.ndarray, Corr
    508    If the list contains only one element, it is unpacked from the list.
    509
    510    Parameters
    511    ----------
    512    fname : str
    513        Filename of the input file.
    514    verbose : bool
    515        Print additional information that was written to the file.
    516    gz : bool
    517        If True, assumes that data is gzipped. If False, assumes JSON file.
    518    full_output : bool
    519        If True, a dict containing auxiliary information and the data is returned.
    520        If False, only the data is returned.
    521
    522    Returns
    523    -------
    524    result : list[Obs]
    525        reconstructed list of observables from the json string
    526    or
    527    result : Obs
    528        only one observable if the list only has one entry
    529    or
    530    result : dict
    531        if full_output=True
    532    """
    533    if not fname.endswith('.json') and not fname.endswith('.gz'):
    534        fname += '.json'
    535    if gz:
    536        if not fname.endswith('.gz'):
    537            fname += '.gz'
    538        with gzip.open(fname, 'r') as fin:
    539            d = json.load(fin)
    540    else:
    541        if fname.endswith('.gz'):
    542            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
    543        with open(fname, 'r', encoding='utf-8') as fin:
    544            d = json.loads(fin.read())
    545
    546    return _parse_json_dict(d, verbose, full_output)

Import a list of Obs or structures containing Obs from a pyerrors.input.json(.gz) file.

The following structures are supported: Obs, list, numpy.ndarray, Corr If the list contains only one element, it is unpacked from the list.

###### Parameters

- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.

###### Returns

- result (list[Obs]): reconstructed list of observables from the json string
- or
- result (Obs): only one observable if the list only has one entry
- or
- result (dict): if full_output=True

def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): View Source

    629def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
    630    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
    631
    632    Parameters
    633    ----------
    634    od : dict
    635        Dict of JSON valid structures and objects that will be exported.
    636        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
    637        All Obs inside a structure have to be defined on the same set of configurations.
    638    fname : str
    639        Filename of the output file.
    640    description : str
    641        Optional string that describes the contents of the json file.
    642    indent : int
    643        Specify the indentation level of the json file. None or 0 is permissible and
    644        saves disk space.
    645    reps : str
    646        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
    647    gz : bool
    648        If True, the output is a gzipped json. If False, the output is a json file.
    649
    650    Returns
    651    -------
    652    None
    653    """
    654
    655    if not isinstance(od, dict):
    656        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
    657
    658    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
    659                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
    660                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
    661                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
    662
    663    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
    664    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
    665
    666    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)

Export a dict of Obs or structures containing Obs to a pyerrors.input.json(.gz) file

###### Parameters

- od (dict): Dict of JSON valid structures and objects that will be exported. At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. All Obs inside a structure have to be defined on the same set of configurations.
- fname (str): Filename of the output file.
- description (str): Optional string that describes the contents of the json file.
- indent (int): Specify the indentation level of the json file. None or 0 is permissible and saves disk space.
- reps (str): Specify the structure of the placeholder in exported dict to be reps[0-9]+.
- gz (bool): If True, the output is a gzipped json. If False, the output is a json file.

###### Returns

- None

def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): View Source

    732def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
    733    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
    734
    735    The following structures are supported: Obs, list, numpy.ndarray, Corr
    736
    737    Parameters
    738    ----------
    739    fname : str
    740        Filename of the input file.
    741    verbose : bool
    742        Print additional information that was written to the file.
    743    gz : bool
    744        If True, assumes that data is gzipped. If False, assumes JSON file.
    745    full_output : bool
    746        If True, a dict containing auxiliary information and the data is returned.
    747        If False, only the data is returned.
    748    reps : str
    749        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
    750
    751    Returns
    752    -------
    753    data : Obs / list / Corr
    754        Read data
    755    or
    756    data : dict
    757        Read data and meta-data
    758    """
    759    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
    760    description = indata['description']['description']
    761    indict = indata['description']['OBSDICT']
    762    ol = indata['obsdata']
    763    od = _od_from_list_and_dict(ol, indict, reps=reps)
    764
    765    if full_output:
    766        indata['description'] = description
    767        indata['obsdata'] = od
    768        return indata
    769    else:
    770        return od

Import a dict of Obs or structures containing Obs from a pyerrors.input.json(.gz) file.

The following structures are supported: Obs, list, numpy.ndarray, Corr

###### Parameters

- fname (str): Filename of the input file.
- verbose (bool): Print additional information that was written to the file.
- gz (bool): If True, assumes that data is gzipped. If False, assumes JSON file.
- full_output (bool): If True, a dict containing auxiliary information and the data is returned. If False, only the data is returned.
- reps (str): Specify the structure of the placeholder in imported dict to be reps[0-9]+.

###### Returns

- data (Obs / list / Corr): Read data
- or
- data (dict): Read data and meta-data