pyerrors.input.json

  1import datetime
  2import getpass
  3import gzip
  4import platform
  5import re
  6import socket
  7import warnings
  8
  9import numpy as np
 10import rapidjson as json
 11
 12from .. import version as pyerrorsversion
 13from ..correlators import Corr
 14from ..covobs import Covobs
 15from ..misc import _assert_equal_properties
 16from ..obs import Obs
 17
 18
 19def create_json_string(ol, description='', indent=1):
 20    """Generate the string for the export of a list of Obs or structures containing Obs
 21    to a .json(.gz) file
 22
 23    Parameters
 24    ----------
 25    ol : list
 26        List of objects that will be exported. At the moment, these objects can be
 27        either of: Obs, list, numpy.ndarray, Corr.
 28        All Obs inside a structure have to be defined on the same set of configurations.
 29    description : str
 30        Optional string that describes the contents of the json file.
 31    indent : int
 32        Specify the indentation level of the json file. None or 0 is permissible and
 33        saves disk space.
 34
 35    Returns
 36    -------
 37    json_string : str
 38        String for export to .json(.gz) file
 39    """
 40
 41    def _gen_data_d_from_list(ol):
 42        dl = []
 43        No = len(ol)
 44        for name in ol[0].mc_names:
 45            ed = {}
 46            ed['id'] = name
 47            ed['replica'] = []
 48            for r_name in ol[0].e_content[name]:
 49                rd = {}
 50                rd['name'] = r_name
 51                rd['deltas'] = []
 52                offsets = [o.r_values[r_name] - o.value for o in ol]
 53                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
 54                for i in range(len(ol[0].idl[r_name])):
 55                    rd['deltas'].append([ol[0].idl[r_name][i]])
 56                    rd['deltas'][-1] += deltas[i].tolist()
 57                ed['replica'].append(rd)
 58            dl.append(ed)
 59        return dl
 60
 61    def _gen_cdata_d_from_list(ol):
 62        dl = []
 63        for name in ol[0].cov_names:
 64            ed = {}
 65            ed['id'] = name
 66            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
 67            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
 68            ncov = ol[0].covobs[name].cov.shape[0]
 69            ed['grad'] = []
 70            for i in range(ncov):
 71                ed['grad'].append([])
 72                for o in ol:
 73                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
 74            dl.append(ed)
 75        return dl
 76
 77    def write_Obs_to_dict(o):
 78        d = {}
 79        d['type'] = 'Obs'
 80        d['layout'] = '1'
 81        if o.tag:
 82            d['tag'] = [o.tag]
 83        if o.reweighted:
 84            d['reweighted'] = o.reweighted
 85        d['value'] = [o.value]
 86        data = _gen_data_d_from_list([o])
 87        if len(data) > 0:
 88            d['data'] = data
 89        cdata = _gen_cdata_d_from_list([o])
 90        if len(cdata) > 0:
 91            d['cdata'] = cdata
 92        return d
 93
 94    def write_List_to_dict(ol):
 95        _assert_equal_properties(ol)
 96        d = {}
 97        d['type'] = 'List'
 98        d['layout'] = f'{len(ol)}'
 99        taglist = [o.tag for o in ol]
100        if np.any([tag is not None for tag in taglist]):
101            d['tag'] = taglist
102        if ol[0].reweighted:
103            d['reweighted'] = ol[0].reweighted
104        d['value'] = [o.value for o in ol]
105        data = _gen_data_d_from_list(ol)
106        if len(data) > 0:
107            d['data'] = data
108        cdata = _gen_cdata_d_from_list(ol)
109        if len(cdata) > 0:
110            d['cdata'] = cdata
111        return d
112
113    def write_Array_to_dict(oa):
114        ol = np.ravel(oa)
115        _assert_equal_properties(ol)
116        d = {}
117        d['type'] = 'Array'
118        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
119        taglist = [o.tag for o in ol]
120        if np.any([tag is not None for tag in taglist]):
121            d['tag'] = taglist
122        if ol[0].reweighted:
123            d['reweighted'] = ol[0].reweighted
124        d['value'] = [o.value for o in ol]
125        data = _gen_data_d_from_list(ol)
126        if len(data) > 0:
127            d['data'] = data
128        cdata = _gen_cdata_d_from_list(ol)
129        if len(cdata) > 0:
130            d['cdata'] = cdata
131        return d
132
133    def _nan_Obs_like(obs):
134        samples = []
135        names = []
136        idl = []
137        for key, value in obs.idl.items():
138            samples.append(np.array([np.nan] * len(value)))
139            names.append(key)
140            idl.append(value)
141        my_obs = Obs(samples, names, idl, means=[np.nan for n in names])
142        my_obs._value = np.nan
143        my_obs._covobs = obs._covobs
144        for name in obs._covobs:
145            my_obs.names.append(name)
146        my_obs.reweighted = obs.reweighted
147        return my_obs
148
149    def write_Corr_to_dict(my_corr):
150        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
151        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
152        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
153        content = [o if o is not None else dummy_array for o in my_corr.content]
154        dat = write_Array_to_dict(np.array(content, dtype=object))
155        dat['type'] = 'Corr'
156        corr_meta_data = str(my_corr.tag)
157        if 'tag' in dat.keys():
158            dat['tag'].append(corr_meta_data)
159        else:
160            dat['tag'] = [corr_meta_data]
161        taglist = dat['tag']
162        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
163        dat['tag']['tag'] = taglist
164        if my_corr.prange is not None:
165            dat['tag']['prange'] = my_corr.prange
166        return dat
167
168    if not isinstance(ol, list):
169        ol = [ol]
170
171    d = {}
172    d['program'] = f'pyerrors {pyerrorsversion.__version__}'
173    d['version'] = '1.1'
174    d['who'] = getpass.getuser()
175    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
176    d['host'] = socket.gethostname() + ', ' + platform.platform()
177
178    if description:
179        d['description'] = description
180
181    d['obsdata'] = []
182    for io in ol:
183        if isinstance(io, Obs):
184            d['obsdata'].append(write_Obs_to_dict(io))
185        elif isinstance(io, list):
186            d['obsdata'].append(write_List_to_dict(io))
187        elif isinstance(io, np.ndarray):
188            d['obsdata'].append(write_Array_to_dict(io))
189        elif isinstance(io, Corr):
190            d['obsdata'].append(write_Corr_to_dict(io))
191        else:
192            raise Exception("Unkown datatype.")
193
194    def _jsonifier(obj):
195        if isinstance(obj, dict):
196            result = {}
197            for key in obj:
198                if key is True:
199                    result['true'] = obj[key]
200                elif key is False:
201                    result['false'] = obj[key]
202                elif key is None:
203                    result['null'] = obj[key]
204                elif isinstance(key, (int, float, np.floating, np.integer)):
205                    result[str(key)] = obj[key]
206                else:
207                    raise TypeError('keys must be str, int, float, bool or None')
208            return result
209        elif isinstance(obj, np.integer):
210            return int(obj)
211        elif isinstance(obj, np.floating):
212            return float(obj)
213        else:
214            raise ValueError(f'{obj!r} is not JSON serializable')
215
216    if indent:
217        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY)
218    else:
219        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)
220
221
222def dump_to_json(ol, fname, description='', indent=1, gz=True):
223    """Export a list of Obs or structures containing Obs to a .json(.gz) file.
224    Dict keys that are not JSON-serializable such as floats are converted to strings.
225
226    Parameters
227    ----------
228    ol : list
229        List of objects that will be exported. At the moment, these objects can be
230        either of: Obs, list, numpy.ndarray, Corr.
231        All Obs inside a structure have to be defined on the same set of configurations.
232    fname : str
233        Filename of the output file.
234    description : str
235        Optional string that describes the contents of the json file.
236    indent : int
237        Specify the indentation level of the json file. None or 0 is permissible and
238        saves disk space.
239    gz : bool
240        If True, the output is a gzipped json. If False, the output is a json file.
241
242    Returns
243    -------
244    Null
245    """
246
247    jsonstring = create_json_string(ol, description, indent)
248
249    if not fname.endswith('.json') and not fname.endswith('.gz'):
250        fname += '.json'
251
252    if gz:
253        if not fname.endswith('.gz'):
254            fname += '.gz'
255
256        fp = gzip.open(fname, 'wb')
257        fp.write(jsonstring.encode('utf-8'))
258    else:
259        fp = open(fname, 'w', encoding='utf-8')
260        fp.write(jsonstring)
261    fp.close()
262
263
264def _parse_json_dict(json_dict, verbose=True, full_output=False):
265    """Reconstruct a list of Obs or structures containing Obs from a dict that
266    was built out of a json string.
267
268    The following structures are supported: Obs, list, numpy.ndarray, Corr
269    If the list contains only one element, it is unpacked from the list.
270
271    Parameters
272    ----------
273    json_string : str
274        json string containing the data.
275    verbose : bool
276        Print additional information that was written to the file.
277    full_output : bool
278        If True, a dict containing auxiliary information and the data is returned.
279        If False, only the data is returned.
280
281    Returns
282    -------
283    result : list[Obs]
284        reconstructed list of observables from the json string
285    or
286    result : Obs
287        only one observable if the list only has one entry
288    or
289    result : dict
290        if full_output=True
291    """
292
293    def _gen_obsd_from_datad(d):
294        retd = {}
295        if d:
296            retd['names'] = []
297            retd['idl'] = []
298            retd['deltas'] = []
299            for ens in d:
300                for rep in ens['replica']:
301                    rep_name = rep['name']
302                    if len(rep_name) > len(ens["id"]):
303                        if rep_name[len(ens["id"])] != "|":
304                            tmp_list = list(rep_name)
305                            tmp_list = [*tmp_list[:len(ens["id"])], "|", *tmp_list[len(ens["id"]):]]
306                            rep_name = ''.join(tmp_list)
307                    retd['names'].append(rep_name)
308                    retd['idl'].append([di[0] for di in rep['deltas']])
309                    retd['deltas'].append(np.array([di[1:] for di in rep['deltas']]))
310        return retd
311
312    def _gen_covobsd_from_cdatad(d):
313        retd = {}
314        for ens in d:
315            retl = []
316            name = ens['id']
317            layouts = ens.get('layout', '1').strip()
318            layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
319            cov = np.reshape(ens['cov'], layout)
320            grad = ens['grad']
321            nobs = len(grad[0])
322            for i in range(nobs):
323                retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]})
324            retd[name] = retl
325        return retd
326
327    def get_Obs_from_dict(o):
328        layouts = o.get('layout', '1').strip()
329        if layouts != '1':
330            raise Exception(f"layout is {layouts} has to be 1 for type Obs.", RuntimeWarning)
331
332        values = o['value']
333        od = _gen_obsd_from_datad(o.get('data', {}))
334        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
335
336        if od:
337            r_offsets = [np.average([ddi[0] for ddi in di]) for di in od['deltas']]
338            ret = Obs([np.array([ddi[0] for ddi in od['deltas'][i]]) - r_offsets[i] for i in range(len(od['deltas']))], od['names'], idl=od['idl'], means=[ro + values[0] for ro in r_offsets])
339            ret._value = values[0]
340        else:
341            ret = Obs([], [], means=[])
342            ret._value = values[0]
343        for name in cd:
344            co = cd[name][0]
345            ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
346            ret.names.append(co['name'])
347
348        ret.reweighted = o.get('reweighted', False)
349        ret.tag = o.get('tag', [None])[0]
350        return ret
351
352    def get_List_from_dict(o):
353        layouts = o.get('layout', '1').strip()
354        layout = int(layouts)
355        values = o['value']
356        od = _gen_obsd_from_datad(o.get('data', {}))
357        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
358
359        ret = []
360        taglist = o.get('tag', layout * [None])
361        for i in range(layout):
362            if od:
363                r_offsets = np.array([np.average(di[:, i]) for di in od['deltas']])
364                ret.append(Obs([od['deltas'][j][:, i] - r_offsets[j] for j in range(len(od['deltas']))], od['names'], idl=od['idl'], means=[ro + values[i] for ro in r_offsets]))
365                ret[-1]._value = values[i]
366            else:
367                ret.append(Obs([], [], means=[]))
368                ret[-1]._value = values[i]
369                print('Created Obs with means= ', values[i])
370            for name in cd:
371                co = cd[name][i]
372                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
373                ret[-1].names.append(co['name'])
374
375            ret[-1].reweighted = o.get('reweighted', False)
376            ret[-1].tag = taglist[i]
377        return ret
378
379    def get_Array_from_dict(o):
380        layouts = o.get('layout', '1').strip()
381        layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
382        N = np.prod(layout)
383        values = o['value']
384        od = _gen_obsd_from_datad(o.get('data', {}))
385        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
386
387        ret = []
388        taglist = o.get('tag', N * [None])
389        for i in range(N):
390            if od:
391                r_offsets = np.array([np.average(di[:, i]) for di in od['deltas']])
392                ret.append(Obs([od['deltas'][j][:, i] - r_offsets[j] for j in range(len(od['deltas']))], od['names'], idl=od['idl'], means=[ro + values[i] for ro in r_offsets]))
393                ret[-1]._value = values[i]
394            else:
395                ret.append(Obs([], [], means=[]))
396                ret[-1]._value = values[i]
397            for name in cd:
398                co = cd[name][i]
399                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
400                ret[-1].names.append(co['name'])
401            ret[-1].reweighted = o.get('reweighted', False)
402            ret[-1].tag = taglist[i]
403        return np.reshape(ret, layout)
404
405    def get_Corr_from_dict(o):
406        if isinstance(o.get('tag'), list):  # supports the old way
407            taglist = o.get('tag')  # This had to be modified to get the taglist from the dictionary
408            temp_prange = None
409        elif isinstance(o.get('tag'), dict):
410            tagdic = o.get('tag')
411            taglist = tagdic['tag']
412            if 'prange' in tagdic:
413                temp_prange = tagdic['prange']
414            else:
415                temp_prange = None
416        else:
417            raise Exception("The tag is not a list or dict")
418
419        corr_tag = taglist[-1]
420        tmp_o = o
421        tmp_o['tag'] = taglist[:-1]
422        if len(tmp_o['tag']) == 0:
423            del tmp_o['tag']
424        dat = get_Array_from_dict(tmp_o)
425        my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)])
426        if corr_tag != 'None':
427            my_corr.tag = corr_tag
428
429        my_corr.prange = temp_prange
430        return my_corr
431
432    prog = json_dict.get('program', '')
433    version = json_dict.get('version', '')
434    who = json_dict.get('who', '')
435    date = json_dict.get('date', '')
436    host = json_dict.get('host', '')
437    if prog and verbose:
438        print(f'Data has been written using {prog}.')
439    if version and verbose:
440        print(f'Format version {version}')
441    if np.any([who, date, host] and verbose):
442        print(f'Written by {who} on {date} on host {host}')
443    description = json_dict.get('description', '')
444    if description and verbose:
445        print()
446        print('Description: ', description)
447    obsdata = json_dict['obsdata']
448    ol = []
449    for io in obsdata:
450        if io['type'] == 'Obs':
451            ol.append(get_Obs_from_dict(io))
452        elif io['type'] == 'List':
453            ol.append(get_List_from_dict(io))
454        elif io['type'] == 'Array':
455            ol.append(get_Array_from_dict(io))
456        elif io['type'] == 'Corr':
457            ol.append(get_Corr_from_dict(io))
458        else:
459            raise Exception("Unknown datatype.")
460
461    if full_output:
462        retd = {}
463        retd['program'] = prog
464        retd['version'] = version
465        retd['who'] = who
466        retd['date'] = date
467        retd['host'] = host
468        retd['description'] = description
469        retd['obsdata'] = ol
470
471        return retd
472    else:
473        if len(obsdata) == 1:
474            ol = ol[0]
475
476        return ol
477
478
479def import_json_string(json_string, verbose=True, full_output=False):
480    """Reconstruct a list of Obs or structures containing Obs from a json string.
481
482    The following structures are supported: Obs, list, numpy.ndarray, Corr
483    If the list contains only one element, it is unpacked from the list.
484
485    Parameters
486    ----------
487    json_string : str
488        json string containing the data.
489    verbose : bool
490        Print additional information that was written to the file.
491    full_output : bool
492        If True, a dict containing auxiliary information and the data is returned.
493        If False, only the data is returned.
494
495    Returns
496    -------
497    result : list[Obs]
498        reconstructed list of observables from the json string
499    or
500    result : Obs
501        only one observable if the list only has one entry
502    or
503    result : dict
504        if full_output=True
505    """
506    return _parse_json_dict(json.loads(json_string), verbose, full_output)
507
508
509def load_json(fname, verbose=True, gz=True, full_output=False):
510    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
511
512    The following structures are supported: Obs, list, numpy.ndarray, Corr
513    If the list contains only one element, it is unpacked from the list.
514
515    Parameters
516    ----------
517    fname : str
518        Filename of the input file.
519    verbose : bool
520        Print additional information that was written to the file.
521    gz : bool
522        If True, assumes that data is gzipped. If False, assumes JSON file.
523    full_output : bool
524        If True, a dict containing auxiliary information and the data is returned.
525        If False, only the data is returned.
526
527    Returns
528    -------
529    result : list[Obs]
530        reconstructed list of observables from the json string
531    or
532    result : Obs
533        only one observable if the list only has one entry
534    or
535    result : dict
536        if full_output=True
537    """
538    if not fname.endswith('.json') and not fname.endswith('.gz'):
539        fname += '.json'
540    if gz:
541        if not fname.endswith('.gz'):
542            fname += '.gz'
543        with gzip.open(fname, 'r') as fin:
544            d = json.load(fin)
545    else:
546        if fname.endswith('.gz'):
547            warnings.warn(f"Trying to read from {fname} without unzipping!", UserWarning, stacklevel=2)
548        with open(fname, encoding='utf-8') as fin:
549            d = json.loads(fin.read())
550
551    return _parse_json_dict(d, verbose, full_output)
552
553
554def _ol_from_dict(ind, reps='DICTOBS'):
555    """Convert a dictionary of Obs objects to a list and a dictionary that contains
556    placeholders instead of the Obs objects.
557
558    Parameters
559    ----------
560    ind : dict
561        Dict of JSON valid structures and objects that will be exported.
562        At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr.
563        All Obs inside a structure have to be defined on the same set of configurations.
564    reps : str
565        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
566    """
567
568    obstypes = (Obs, Corr, np.ndarray)
569
570    if not reps.isalnum():
571        raise ValueError('Placeholder string has to be alphanumeric!')
572    ol = []
573    counter = 0
574
575    def dict_replace_obs(d):
576        nonlocal counter
577        x = {}
578        for k, v in d.items():
579            if isinstance(v, dict):
580                v = dict_replace_obs(v)
581            elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]):
582                v = obslist_replace_obs(v)
583            elif isinstance(v, list):
584                v = list_replace_obs(v)
585            elif isinstance(v, obstypes):
586                ol.append(v)
587                v = reps + f'{counter}'
588                counter += 1
589            elif isinstance(v, str):
590                if bool(re.match(rf'{reps}[0-9]+', v)):
591                    raise ValueError(f'Dict contains string {v} that matches the placeholder! {reps} Cannot be safely exported.')
592            x[k] = v
593        return x
594
595    def list_replace_obs(li):
596        nonlocal counter
597        x = []
598        for e in li:
599            if isinstance(e, list):
600                e = list_replace_obs(e)
601            elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]):
602                e = obslist_replace_obs(e)
603            elif isinstance(e, dict):
604                e = dict_replace_obs(e)
605            elif isinstance(e, obstypes):
606                ol.append(e)
607                e = reps + f'{counter}'
608                counter += 1
609            elif isinstance(e, str):
610                if bool(re.match(rf'{reps}[0-9]+', e)):
611                    raise ValueError(f'Dict contains string {e} that matches the placeholder! {reps} Cannot be safely exported.')
612            x.append(e)
613        return x
614
615    def obslist_replace_obs(li):
616        nonlocal counter
617        il = []
618        for e in li:
619            il.append(e)
620
621        ol.append(il)
622        x = reps + f'{counter}'
623        counter += 1
624        return x
625
626    nd = dict_replace_obs(ind)
627
628    return ol, nd
629
630
631def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
632    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
633
634    Parameters
635    ----------
636    od : dict
637        Dict of JSON valid structures and objects that will be exported.
638        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
639        All Obs inside a structure have to be defined on the same set of configurations.
640    fname : str
641        Filename of the output file.
642    description : str
643        Optional string that describes the contents of the json file.
644    indent : int
645        Specify the indentation level of the json file. None or 0 is permissible and
646        saves disk space.
647    reps : str
648        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
649    gz : bool
650        If True, the output is a gzipped json. If False, the output is a json file.
651
652    Returns
653    -------
654    None
655    """
656
657    if not isinstance(od, dict):
658        raise TypeError('od has to be a dictionary. Did you want to use dump_to_json?')
659
660    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
661                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
662                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
663                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
664
665    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
666    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
667
668    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
669
670
671def _od_from_list_and_dict(ol, ind, reps='DICTOBS'):
672    """Parse a list of Obs or structures containing Obs and an accompanying
673    dict, where the structures have been replaced by placeholders to a
674    dict that contains the structures.
675
676    The following structures are supported: Obs, list, numpy.ndarray, Corr
677
678    Parameters
679    ----------
680    ol : list
681        List of objects -
682        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
683        All Obs inside a structure have to be defined on the same set of configurations.
684    ind : dict
685        Dict that defines the structure of the resulting dict and contains placeholders
686    reps : str
687        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
688    """
689    if not reps.isalnum():
690        raise ValueError('Placeholder string has to be alphanumeric!')
691
692    counter = 0
693
694    def dict_replace_string(d):
695        nonlocal counter
696        x = {}
697        for k, v in d.items():
698            if isinstance(v, dict):
699                v = dict_replace_string(v)
700            elif isinstance(v, list):
701                v = list_replace_string(v)
702            elif isinstance(v, str) and bool(re.match(rf'{reps}[0-9]+', v)):
703                index = int(v[len(reps):])
704                v = ol[index]
705                counter += 1
706            x[k] = v
707        return x
708
709    def list_replace_string(li):
710        nonlocal counter
711        x = []
712        for e in li:
713            if isinstance(e, list):
714                e = list_replace_string(e)
715            elif isinstance(e, dict):
716                e = dict_replace_string(e)
717            elif isinstance(e, str) and bool(re.match(rf'{reps}[0-9]+', e)):
718                index = int(e[len(reps):])
719                e = ol[index]
720                counter += 1
721            x.append(e)
722        return x
723
724    nd = dict_replace_string(ind)
725
726    if counter == 0:
727        raise ValueError('No placeholder has been replaced! Check if reps is set correctly.')
728
729    return nd
730
731
732def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
733    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
734
735    The following structures are supported: Obs, list, numpy.ndarray, Corr
736
737    Parameters
738    ----------
739    fname : str
740        Filename of the input file.
741    verbose : bool
742        Print additional information that was written to the file.
743    gz : bool
744        If True, assumes that data is gzipped. If False, assumes JSON file.
745    full_output : bool
746        If True, a dict containing auxiliary information and the data is returned.
747        If False, only the data is returned.
748    reps : str
749        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
750
751    Returns
752    -------
753    data : Obs / list / Corr
754        Read data
755    or
756    data : dict
757        Read data and meta-data
758    """
759    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
760    description = indata['description']['description']
761    indict = indata['description']['OBSDICT']
762    ol = indata['obsdata']
763    od = _od_from_list_and_dict(ol, indict, reps=reps)
764
765    if full_output:
766        indata['description'] = description
767        indata['obsdata'] = od
768        return indata
769    else:
770        return od