pyerrors.input.json

  1import rapidjson as json
  2import gzip
  3import getpass
  4import socket
  5import datetime
  6import platform
  7import warnings
  8import re
  9import numpy as np
 10from ..obs import Obs
 11from ..covobs import Covobs
 12from ..correlators import Corr
 13from ..misc import _assert_equal_properties
 14from .. import version as pyerrorsversion
 15
 16
 17def create_json_string(ol, description='', indent=1):
 18    """Generate the string for the export of a list of Obs or structures containing Obs
 19    to a .json(.gz) file
 20
 21    Parameters
 22    ----------
 23    ol : list
 24        List of objects that will be exported. At the moment, these objects can be
 25        either of: Obs, list, numpy.ndarray, Corr.
 26        All Obs inside a structure have to be defined on the same set of configurations.
 27    description : str
 28        Optional string that describes the contents of the json file.
 29    indent : int
 30        Specify the indentation level of the json file. None or 0 is permissible and
 31        saves disk space.
 32
 33    Returns
 34    -------
 35    json_string : str
 36        String for export to .json(.gz) file
 37    """
 38
 39    def _gen_data_d_from_list(ol):
 40        dl = []
 41        No = len(ol)
 42        for name in ol[0].mc_names:
 43            ed = {}
 44            ed['id'] = name
 45            ed['replica'] = []
 46            for r_name in ol[0].e_content[name]:
 47                rd = {}
 48                rd['name'] = r_name
 49                rd['deltas'] = []
 50                offsets = [o.r_values[r_name] - o.value for o in ol]
 51                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
 52                for i in range(len(ol[0].idl[r_name])):
 53                    rd['deltas'].append([ol[0].idl[r_name][i]])
 54                    rd['deltas'][-1] += deltas[i].tolist()
 55                ed['replica'].append(rd)
 56            dl.append(ed)
 57        return dl
 58
 59    def _gen_cdata_d_from_list(ol):
 60        dl = []
 61        for name in ol[0].cov_names:
 62            ed = {}
 63            ed['id'] = name
 64            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
 65            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
 66            ncov = ol[0].covobs[name].cov.shape[0]
 67            ed['grad'] = []
 68            for i in range(ncov):
 69                ed['grad'].append([])
 70                for o in ol:
 71                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
 72            dl.append(ed)
 73        return dl
 74
 75    def write_Obs_to_dict(o):
 76        d = {}
 77        d['type'] = 'Obs'
 78        d['layout'] = '1'
 79        if o.tag:
 80            d['tag'] = [o.tag]
 81        if o.reweighted:
 82            d['reweighted'] = o.reweighted
 83        d['value'] = [o.value]
 84        data = _gen_data_d_from_list([o])
 85        if len(data) > 0:
 86            d['data'] = data
 87        cdata = _gen_cdata_d_from_list([o])
 88        if len(cdata) > 0:
 89            d['cdata'] = cdata
 90        return d
 91
 92    def write_List_to_dict(ol):
 93        _assert_equal_properties(ol)
 94        d = {}
 95        d['type'] = 'List'
 96        d['layout'] = '%d' % len(ol)
 97        taglist = [o.tag for o in ol]
 98        if np.any([tag is not None for tag in taglist]):
 99            d['tag'] = taglist
100        if ol[0].reweighted:
101            d['reweighted'] = ol[0].reweighted
102        d['value'] = [o.value for o in ol]
103        data = _gen_data_d_from_list(ol)
104        if len(data) > 0:
105            d['data'] = data
106        cdata = _gen_cdata_d_from_list(ol)
107        if len(cdata) > 0:
108            d['cdata'] = cdata
109        return d
110
111    def write_Array_to_dict(oa):
112        ol = np.ravel(oa)
113        _assert_equal_properties(ol)
114        d = {}
115        d['type'] = 'Array'
116        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
117        taglist = [o.tag for o in ol]
118        if np.any([tag is not None for tag in taglist]):
119            d['tag'] = taglist
120        if ol[0].reweighted:
121            d['reweighted'] = ol[0].reweighted
122        d['value'] = [o.value for o in ol]
123        data = _gen_data_d_from_list(ol)
124        if len(data) > 0:
125            d['data'] = data
126        cdata = _gen_cdata_d_from_list(ol)
127        if len(cdata) > 0:
128            d['cdata'] = cdata
129        return d
130
131    def _nan_Obs_like(obs):
132        samples = []
133        names = []
134        idl = []
135        for key, value in obs.idl.items():
136            samples.append(np.array([np.nan] * len(value)))
137            names.append(key)
138            idl.append(value)
139        my_obs = Obs(samples, names, idl, means=[np.nan for n in names])
140        my_obs._value = np.nan
141        my_obs._covobs = obs._covobs
142        for name in obs._covobs:
143            my_obs.names.append(name)
144        my_obs.reweighted = obs.reweighted
145        return my_obs
146
147    def write_Corr_to_dict(my_corr):
148        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
149        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
150        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
151        content = [o if o is not None else dummy_array for o in my_corr.content]
152        dat = write_Array_to_dict(np.array(content, dtype=object))
153        dat['type'] = 'Corr'
154        corr_meta_data = str(my_corr.tag)
155        if 'tag' in dat.keys():
156            dat['tag'].append(corr_meta_data)
157        else:
158            dat['tag'] = [corr_meta_data]
159        taglist = dat['tag']
160        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
161        dat['tag']['tag'] = taglist
162        if my_corr.prange is not None:
163            dat['tag']['prange'] = my_corr.prange
164        return dat
165
166    if not isinstance(ol, list):
167        ol = [ol]
168
169    d = {}
170    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
171    d['version'] = '1.1'
172    d['who'] = getpass.getuser()
173    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
174    d['host'] = socket.gethostname() + ', ' + platform.platform()
175
176    if description:
177        d['description'] = description
178
179    d['obsdata'] = []
180    for io in ol:
181        if isinstance(io, Obs):
182            d['obsdata'].append(write_Obs_to_dict(io))
183        elif isinstance(io, list):
184            d['obsdata'].append(write_List_to_dict(io))
185        elif isinstance(io, np.ndarray):
186            d['obsdata'].append(write_Array_to_dict(io))
187        elif isinstance(io, Corr):
188            d['obsdata'].append(write_Corr_to_dict(io))
189        else:
190            raise Exception("Unkown datatype.")
191
192    def _jsonifier(obj):
193        if isinstance(obj, dict):
194            result = {}
195            for key in obj:
196                if key is True:
197                    result['true'] = obj[key]
198                elif key is False:
199                    result['false'] = obj[key]
200                elif key is None:
201                    result['null'] = obj[key]
202                elif isinstance(key, (int, float, np.floating, np.integer)):
203                    result[str(key)] = obj[key]
204                else:
205                    raise TypeError('keys must be str, int, float, bool or None')
206            return result
207        elif isinstance(obj, np.integer):
208            return int(obj)
209        elif isinstance(obj, np.floating):
210            return float(obj)
211        else:
212            raise ValueError('%r is not JSON serializable' % (obj,))
213
214    if indent:
215        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY)
216    else:
217        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)
218
219
220def dump_to_json(ol, fname, description='', indent=1, gz=True):
221    """Export a list of Obs or structures containing Obs to a .json(.gz) file.
222    Dict keys that are not JSON-serializable such as floats are converted to strings.
223
224    Parameters
225    ----------
226    ol : list
227        List of objects that will be exported. At the moment, these objects can be
228        either of: Obs, list, numpy.ndarray, Corr.
229        All Obs inside a structure have to be defined on the same set of configurations.
230    fname : str
231        Filename of the output file.
232    description : str
233        Optional string that describes the contents of the json file.
234    indent : int
235        Specify the indentation level of the json file. None or 0 is permissible and
236        saves disk space.
237    gz : bool
238        If True, the output is a gzipped json. If False, the output is a json file.
239
240    Returns
241    -------
242    Null
243    """
244
245    jsonstring = create_json_string(ol, description, indent)
246
247    if not fname.endswith('.json') and not fname.endswith('.gz'):
248        fname += '.json'
249
250    if gz:
251        if not fname.endswith('.gz'):
252            fname += '.gz'
253
254        fp = gzip.open(fname, 'wb')
255        fp.write(jsonstring.encode('utf-8'))
256    else:
257        fp = open(fname, 'w', encoding='utf-8')
258        fp.write(jsonstring)
259    fp.close()
260
261
262def _parse_json_dict(json_dict, verbose=True, full_output=False):
263    """Reconstruct a list of Obs or structures containing Obs from a dict that
264    was built out of a json string.
265
266    The following structures are supported: Obs, list, numpy.ndarray, Corr
267    If the list contains only one element, it is unpacked from the list.
268
269    Parameters
270    ----------
271    json_string : str
272        json string containing the data.
273    verbose : bool
274        Print additional information that was written to the file.
275    full_output : bool
276        If True, a dict containing auxiliary information and the data is returned.
277        If False, only the data is returned.
278
279    Returns
280    -------
281    result : list[Obs]
282        reconstructed list of observables from the json string
283    or
284    result : Obs
285        only one observable if the list only has one entry
286    or
287    result : dict
288        if full_output=True
289    """
290
291    def _gen_obsd_from_datad(d):
292        retd = {}
293        if d:
294            retd['names'] = []
295            retd['idl'] = []
296            retd['deltas'] = []
297            for ens in d:
298                for rep in ens['replica']:
299                    rep_name = rep['name']
300                    if len(rep_name) > len(ens["id"]):
301                        if rep_name[len(ens["id"])] != "|":
302                            tmp_list = list(rep_name)
303                            tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):]
304                            rep_name = ''.join(tmp_list)
305                    retd['names'].append(rep_name)
306                    retd['idl'].append([di[0] for di in rep['deltas']])
307                    retd['deltas'].append(np.array([di[1:] for di in rep['deltas']]))
308        return retd
309
310    def _gen_covobsd_from_cdatad(d):
311        retd = {}
312        for ens in d:
313            retl = []
314            name = ens['id']
315            layouts = ens.get('layout', '1').strip()
316            layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
317            cov = np.reshape(ens['cov'], layout)
318            grad = ens['grad']
319            nobs = len(grad[0])
320            for i in range(nobs):
321                retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]})
322            retd[name] = retl
323        return retd
324
325    def get_Obs_from_dict(o):
326        layouts = o.get('layout', '1').strip()
327        if layouts != '1':
328            raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning)
329
330        values = o['value']
331        od = _gen_obsd_from_datad(o.get('data', {}))
332        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
333
334        if od:
335            r_offsets = [np.average([ddi[0] for ddi in di]) for di in od['deltas']]
336            ret = Obs([np.array([ddi[0] for ddi in od['deltas'][i]]) - r_offsets[i] for i in range(len(od['deltas']))], od['names'], idl=od['idl'], means=[ro + values[0] for ro in r_offsets])
337            ret._value = values[0]
338        else:
339            ret = Obs([], [], means=[])
340            ret._value = values[0]
341        for name in cd:
342            co = cd[name][0]
343            ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
344            ret.names.append(co['name'])
345
346        ret.reweighted = o.get('reweighted', False)
347        ret.tag = o.get('tag', [None])[0]
348        return ret
349
350    def get_List_from_dict(o):
351        layouts = o.get('layout', '1').strip()
352        layout = int(layouts)
353        values = o['value']
354        od = _gen_obsd_from_datad(o.get('data', {}))
355        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
356
357        ret = []
358        taglist = o.get('tag', layout * [None])
359        for i in range(layout):
360            if od:
361                r_offsets = np.array([np.average(di[:, i]) for di in od['deltas']])
362                ret.append(Obs([od['deltas'][j][:, i] - r_offsets[j] for j in range(len(od['deltas']))], od['names'], idl=od['idl'], means=[ro + values[i] for ro in r_offsets]))
363                ret[-1]._value = values[i]
364            else:
365                ret.append(Obs([], [], means=[]))
366                ret[-1]._value = values[i]
367                print('Created Obs with means= ', values[i])
368            for name in cd:
369                co = cd[name][i]
370                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
371                ret[-1].names.append(co['name'])
372
373            ret[-1].reweighted = o.get('reweighted', False)
374            ret[-1].tag = taglist[i]
375        return ret
376
377    def get_Array_from_dict(o):
378        layouts = o.get('layout', '1').strip()
379        layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
380        N = np.prod(layout)
381        values = o['value']
382        od = _gen_obsd_from_datad(o.get('data', {}))
383        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
384
385        ret = []
386        taglist = o.get('tag', N * [None])
387        for i in range(N):
388            if od:
389                r_offsets = np.array([np.average(di[:, i]) for di in od['deltas']])
390                ret.append(Obs([od['deltas'][j][:, i] - r_offsets[j] for j in range(len(od['deltas']))], od['names'], idl=od['idl'], means=[ro + values[i] for ro in r_offsets]))
391                ret[-1]._value = values[i]
392            else:
393                ret.append(Obs([], [], means=[]))
394                ret[-1]._value = values[i]
395            for name in cd:
396                co = cd[name][i]
397                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
398                ret[-1].names.append(co['name'])
399            ret[-1].reweighted = o.get('reweighted', False)
400            ret[-1].tag = taglist[i]
401        return np.reshape(ret, layout)
402
403    def get_Corr_from_dict(o):
404        if isinstance(o.get('tag'), list):  # supports the old way
405            taglist = o.get('tag')  # This had to be modified to get the taglist from the dictionary
406            temp_prange = None
407        elif isinstance(o.get('tag'), dict):
408            tagdic = o.get('tag')
409            taglist = tagdic['tag']
410            if 'prange' in tagdic:
411                temp_prange = tagdic['prange']
412            else:
413                temp_prange = None
414        else:
415            raise Exception("The tag is not a list or dict")
416
417        corr_tag = taglist[-1]
418        tmp_o = o
419        tmp_o['tag'] = taglist[:-1]
420        if len(tmp_o['tag']) == 0:
421            del tmp_o['tag']
422        dat = get_Array_from_dict(tmp_o)
423        my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)])
424        if corr_tag != 'None':
425            my_corr.tag = corr_tag
426
427        my_corr.prange = temp_prange
428        return my_corr
429
430    prog = json_dict.get('program', '')
431    version = json_dict.get('version', '')
432    who = json_dict.get('who', '')
433    date = json_dict.get('date', '')
434    host = json_dict.get('host', '')
435    if prog and verbose:
436        print('Data has been written using %s.' % (prog))
437    if version and verbose:
438        print('Format version %s' % (version))
439    if np.any([who, date, host] and verbose):
440        print('Written by %s on %s on host %s' % (who, date, host))
441    description = json_dict.get('description', '')
442    if description and verbose:
443        print()
444        print('Description: ', description)
445    obsdata = json_dict['obsdata']
446    ol = []
447    for io in obsdata:
448        if io['type'] == 'Obs':
449            ol.append(get_Obs_from_dict(io))
450        elif io['type'] == 'List':
451            ol.append(get_List_from_dict(io))
452        elif io['type'] == 'Array':
453            ol.append(get_Array_from_dict(io))
454        elif io['type'] == 'Corr':
455            ol.append(get_Corr_from_dict(io))
456        else:
457            raise Exception("Unknown datatype.")
458
459    if full_output:
460        retd = {}
461        retd['program'] = prog
462        retd['version'] = version
463        retd['who'] = who
464        retd['date'] = date
465        retd['host'] = host
466        retd['description'] = description
467        retd['obsdata'] = ol
468
469        return retd
470    else:
471        if len(obsdata) == 1:
472            ol = ol[0]
473
474        return ol
475
476
477def import_json_string(json_string, verbose=True, full_output=False):
478    """Reconstruct a list of Obs or structures containing Obs from a json string.
479
480    The following structures are supported: Obs, list, numpy.ndarray, Corr
481    If the list contains only one element, it is unpacked from the list.
482
483    Parameters
484    ----------
485    json_string : str
486        json string containing the data.
487    verbose : bool
488        Print additional information that was written to the file.
489    full_output : bool
490        If True, a dict containing auxiliary information and the data is returned.
491        If False, only the data is returned.
492
493    Returns
494    -------
495    result : list[Obs]
496        reconstructed list of observables from the json string
497    or
498    result : Obs
499        only one observable if the list only has one entry
500    or
501    result : dict
502        if full_output=True
503    """
504    return _parse_json_dict(json.loads(json_string), verbose, full_output)
505
506
507def load_json(fname, verbose=True, gz=True, full_output=False):
508    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
509
510    The following structures are supported: Obs, list, numpy.ndarray, Corr
511    If the list contains only one element, it is unpacked from the list.
512
513    Parameters
514    ----------
515    fname : str
516        Filename of the input file.
517    verbose : bool
518        Print additional information that was written to the file.
519    gz : bool
520        If True, assumes that data is gzipped. If False, assumes JSON file.
521    full_output : bool
522        If True, a dict containing auxiliary information and the data is returned.
523        If False, only the data is returned.
524
525    Returns
526    -------
527    result : list[Obs]
528        reconstructed list of observables from the json string
529    or
530    result : Obs
531        only one observable if the list only has one entry
532    or
533    result : dict
534        if full_output=True
535    """
536    if not fname.endswith('.json') and not fname.endswith('.gz'):
537        fname += '.json'
538    if gz:
539        if not fname.endswith('.gz'):
540            fname += '.gz'
541        with gzip.open(fname, 'r') as fin:
542            d = json.load(fin)
543    else:
544        if fname.endswith('.gz'):
545            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
546        with open(fname, 'r', encoding='utf-8') as fin:
547            d = json.loads(fin.read())
548
549    return _parse_json_dict(d, verbose, full_output)
550
551
552def _ol_from_dict(ind, reps='DICTOBS'):
553    """Convert a dictionary of Obs objects to a list and a dictionary that contains
554    placeholders instead of the Obs objects.
555
556    Parameters
557    ----------
558    ind : dict
559        Dict of JSON valid structures and objects that will be exported.
560        At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr.
561        All Obs inside a structure have to be defined on the same set of configurations.
562    reps : str
563        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
564    """
565
566    obstypes = (Obs, Corr, np.ndarray)
567
568    if not reps.isalnum():
569        raise Exception('Placeholder string has to be alphanumeric!')
570    ol = []
571    counter = 0
572
573    def dict_replace_obs(d):
574        nonlocal counter
575        x = {}
576        for k, v in d.items():
577            if isinstance(v, dict):
578                v = dict_replace_obs(v)
579            elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]):
580                v = obslist_replace_obs(v)
581            elif isinstance(v, list):
582                v = list_replace_obs(v)
583            elif isinstance(v, obstypes):
584                ol.append(v)
585                v = reps + '%d' % (counter)
586                counter += 1
587            elif isinstance(v, str):
588                if bool(re.match(r'%s[0-9]+' % (reps), v)):
589                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be safely exported.' % (v, reps))
590            x[k] = v
591        return x
592
593    def list_replace_obs(li):
594        nonlocal counter
595        x = []
596        for e in li:
597            if isinstance(e, list):
598                e = list_replace_obs(e)
599            elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]):
600                e = obslist_replace_obs(e)
601            elif isinstance(e, dict):
602                e = dict_replace_obs(e)
603            elif isinstance(e, obstypes):
604                ol.append(e)
605                e = reps + '%d' % (counter)
606                counter += 1
607            elif isinstance(e, str):
608                if bool(re.match(r'%s[0-9]+' % (reps), e)):
609                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be safely exported.' % (e, reps))
610            x.append(e)
611        return x
612
613    def obslist_replace_obs(li):
614        nonlocal counter
615        il = []
616        for e in li:
617            il.append(e)
618
619        ol.append(il)
620        x = reps + '%d' % (counter)
621        counter += 1
622        return x
623
624    nd = dict_replace_obs(ind)
625
626    return ol, nd
627
628
629def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
630    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
631
632    Parameters
633    ----------
634    od : dict
635        Dict of JSON valid structures and objects that will be exported.
636        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
637        All Obs inside a structure have to be defined on the same set of configurations.
638    fname : str
639        Filename of the output file.
640    description : str
641        Optional string that describes the contents of the json file.
642    indent : int
643        Specify the indentation level of the json file. None or 0 is permissible and
644        saves disk space.
645    reps : str
646        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
647    gz : bool
648        If True, the output is a gzipped json. If False, the output is a json file.
649
650    Returns
651    -------
652    None
653    """
654
655    if not isinstance(od, dict):
656        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
657
658    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
659                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
660                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
661                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
662
663    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
664    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
665
666    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
667
668
669def _od_from_list_and_dict(ol, ind, reps='DICTOBS'):
670    """Parse a list of Obs or structures containing Obs and an accompanying
671    dict, where the structures have been replaced by placeholders to a
672    dict that contains the structures.
673
674    The following structures are supported: Obs, list, numpy.ndarray, Corr
675
676    Parameters
677    ----------
678    ol : list
679        List of objects -
680        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
681        All Obs inside a structure have to be defined on the same set of configurations.
682    ind : dict
683        Dict that defines the structure of the resulting dict and contains placeholders
684    reps : str
685        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
686    """
687    if not reps.isalnum():
688        raise Exception('Placeholder string has to be alphanumeric!')
689
690    counter = 0
691
692    def dict_replace_string(d):
693        nonlocal counter
694        x = {}
695        for k, v in d.items():
696            if isinstance(v, dict):
697                v = dict_replace_string(v)
698            elif isinstance(v, list):
699                v = list_replace_string(v)
700            elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)):
701                index = int(v[len(reps):])
702                v = ol[index]
703                counter += 1
704            x[k] = v
705        return x
706
707    def list_replace_string(li):
708        nonlocal counter
709        x = []
710        for e in li:
711            if isinstance(e, list):
712                e = list_replace_string(e)
713            elif isinstance(e, dict):
714                e = dict_replace_string(e)
715            elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)):
716                index = int(e[len(reps):])
717                e = ol[index]
718                counter += 1
719            x.append(e)
720        return x
721
722    nd = dict_replace_string(ind)
723
724    if counter == 0:
725        raise Exception('No placeholder has been replaced! Check if reps is set correctly.')
726
727    return nd
728
729
730def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
731    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
732
733    The following structures are supported: Obs, list, numpy.ndarray, Corr
734
735    Parameters
736    ----------
737    fname : str
738        Filename of the input file.
739    verbose : bool
740        Print additional information that was written to the file.
741    gz : bool
742        If True, assumes that data is gzipped. If False, assumes JSON file.
743    full_output : bool
744        If True, a dict containing auxiliary information and the data is returned.
745        If False, only the data is returned.
746    reps : str
747        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
748
749    Returns
750    -------
751    data : Obs / list / Corr
752        Read data
753    or
754    data : dict
755        Read data and meta-data
756    """
757    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
758    description = indata['description']['description']
759    indict = indata['description']['OBSDICT']
760    ol = indata['obsdata']
761    od = _od_from_list_and_dict(ol, indict, reps=reps)
762
763    if full_output:
764        indata['description'] = description
765        indata['obsdata'] = od
766        return indata
767    else:
768        return od