pyerrors.input.json

  1import rapidjson as json
  2import gzip
  3import getpass
  4import socket
  5import datetime
  6import platform
  7import warnings
  8import re
  9import numpy as np
 10from ..obs import Obs
 11from ..covobs import Covobs
 12from ..correlators import Corr
 13from ..misc import _assert_equal_properties
 14from .. import version as pyerrorsversion
 15
 16
 17def create_json_string(ol, description='', indent=1):
 18    """Generate the string for the export of a list of Obs or structures containing Obs
 19    to a .json(.gz) file
 20
 21    Parameters
 22    ----------
 23    ol : list
 24        List of objects that will be exported. At the moment, these objects can be
 25        either of: Obs, list, numpy.ndarray, Corr.
 26        All Obs inside a structure have to be defined on the same set of configurations.
 27    description : str
 28        Optional string that describes the contents of the json file.
 29    indent : int
 30        Specify the indentation level of the json file. None or 0 is permissible and
 31        saves disk space.
 32    """
 33
 34    def _gen_data_d_from_list(ol):
 35        dl = []
 36        No = len(ol)
 37        for name in ol[0].mc_names:
 38            ed = {}
 39            ed['id'] = name
 40            ed['replica'] = []
 41            for r_name in ol[0].e_content[name]:
 42                rd = {}
 43                rd['name'] = r_name
 44                if ol[0].is_merged.get(r_name, False):
 45                    rd['is_merged'] = True
 46                rd['deltas'] = []
 47                offsets = [o.r_values[r_name] - o.value for o in ol]
 48                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
 49                for i in range(len(ol[0].idl[r_name])):
 50                    rd['deltas'].append([ol[0].idl[r_name][i]])
 51                    rd['deltas'][-1] += deltas[i].tolist()
 52                ed['replica'].append(rd)
 53            dl.append(ed)
 54        return dl
 55
 56    def _gen_cdata_d_from_list(ol):
 57        dl = []
 58        for name in ol[0].cov_names:
 59            ed = {}
 60            ed['id'] = name
 61            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
 62            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
 63            ncov = ol[0].covobs[name].cov.shape[0]
 64            ed['grad'] = []
 65            for i in range(ncov):
 66                ed['grad'].append([])
 67                for o in ol:
 68                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
 69            dl.append(ed)
 70        return dl
 71
 72    def write_Obs_to_dict(o):
 73        d = {}
 74        d['type'] = 'Obs'
 75        d['layout'] = '1'
 76        if o.tag:
 77            d['tag'] = [o.tag]
 78        if o.reweighted:
 79            d['reweighted'] = o.reweighted
 80        d['value'] = [o.value]
 81        data = _gen_data_d_from_list([o])
 82        if len(data) > 0:
 83            d['data'] = data
 84        cdata = _gen_cdata_d_from_list([o])
 85        if len(cdata) > 0:
 86            d['cdata'] = cdata
 87        return d
 88
 89    def write_List_to_dict(ol):
 90        _assert_equal_properties(ol)
 91        d = {}
 92        d['type'] = 'List'
 93        d['layout'] = '%d' % len(ol)
 94        taglist = [o.tag for o in ol]
 95        if np.any([tag is not None for tag in taglist]):
 96            d['tag'] = taglist
 97        if ol[0].reweighted:
 98            d['reweighted'] = ol[0].reweighted
 99        d['value'] = [o.value for o in ol]
100        data = _gen_data_d_from_list(ol)
101        if len(data) > 0:
102            d['data'] = data
103        cdata = _gen_cdata_d_from_list(ol)
104        if len(cdata) > 0:
105            d['cdata'] = cdata
106        return d
107
108    def write_Array_to_dict(oa):
109        ol = np.ravel(oa)
110        _assert_equal_properties(ol)
111        d = {}
112        d['type'] = 'Array'
113        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
114        taglist = [o.tag for o in ol]
115        if np.any([tag is not None for tag in taglist]):
116            d['tag'] = taglist
117        if ol[0].reweighted:
118            d['reweighted'] = ol[0].reweighted
119        d['value'] = [o.value for o in ol]
120        data = _gen_data_d_from_list(ol)
121        if len(data) > 0:
122            d['data'] = data
123        cdata = _gen_cdata_d_from_list(ol)
124        if len(cdata) > 0:
125            d['cdata'] = cdata
126        return d
127
128    def _nan_Obs_like(obs):
129        samples = []
130        names = []
131        idl = []
132        for key, value in obs.idl.items():
133            samples.append([np.nan] * len(value))
134            names.append(key)
135            idl.append(value)
136        my_obs = Obs(samples, names, idl)
137        my_obs._covobs = obs._covobs
138        for name in obs._covobs:
139            my_obs.names.append(name)
140        my_obs.reweighted = obs.reweighted
141        my_obs.is_merged = obs.is_merged
142        return my_obs
143
144    def write_Corr_to_dict(my_corr):
145        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
146        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
147        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
148        content = [o if o is not None else dummy_array for o in my_corr.content]
149        dat = write_Array_to_dict(np.array(content, dtype=object))
150        dat['type'] = 'Corr'
151        corr_meta_data = str(my_corr.tag)
152        if 'tag' in dat.keys():
153            dat['tag'].append(corr_meta_data)
154        else:
155            dat['tag'] = [corr_meta_data]
156        taglist = dat['tag']
157        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
158        dat['tag']['tag'] = taglist
159        if my_corr.prange is not None:
160            dat['tag']['prange'] = my_corr.prange
161        return dat
162
163    if not isinstance(ol, list):
164        ol = [ol]
165
166    d = {}
167    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
168    d['version'] = '1.1'
169    d['who'] = getpass.getuser()
170    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
171    d['host'] = socket.gethostname() + ', ' + platform.platform()
172
173    if description:
174        d['description'] = description
175
176    d['obsdata'] = []
177    for io in ol:
178        if isinstance(io, Obs):
179            d['obsdata'].append(write_Obs_to_dict(io))
180        elif isinstance(io, list):
181            d['obsdata'].append(write_List_to_dict(io))
182        elif isinstance(io, np.ndarray):
183            d['obsdata'].append(write_Array_to_dict(io))
184        elif isinstance(io, Corr):
185            d['obsdata'].append(write_Corr_to_dict(io))
186        else:
187            raise Exception("Unkown datatype.")
188
189    def _jsonifier(o):
190        if isinstance(o, np.int64):
191            return int(o)
192        raise TypeError('%r is not JSON serializable' % o)
193
194    if indent:
195        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_SINGLE_LINE_ARRAY)
196    else:
197        return json.dumps(d, indent=indent, ensure_ascii=False, default=_jsonifier, write_mode=json.WM_COMPACT)
198
199
200def dump_to_json(ol, fname, description='', indent=1, gz=True):
201    """Export a list of Obs or structures containing Obs to a .json(.gz) file
202
203    Parameters
204    ----------
205    ol : list
206        List of objects that will be exported. At the moment, these objects can be
207        either of: Obs, list, numpy.ndarray, Corr.
208        All Obs inside a structure have to be defined on the same set of configurations.
209    fname : str
210        Filename of the output file.
211    description : str
212        Optional string that describes the contents of the json file.
213    indent : int
214        Specify the indentation level of the json file. None or 0 is permissible and
215        saves disk space.
216    gz : bool
217        If True, the output is a gzipped json. If False, the output is a json file.
218    """
219
220    jsonstring = create_json_string(ol, description, indent)
221
222    if not fname.endswith('.json') and not fname.endswith('.gz'):
223        fname += '.json'
224
225    if gz:
226        if not fname.endswith('.gz'):
227            fname += '.gz'
228
229        fp = gzip.open(fname, 'wb')
230        fp.write(jsonstring.encode('utf-8'))
231    else:
232        fp = open(fname, 'w', encoding='utf-8')
233        fp.write(jsonstring)
234    fp.close()
235
236
237def _parse_json_dict(json_dict, verbose=True, full_output=False):
238    """Reconstruct a list of Obs or structures containing Obs from a dict that
239    was built out of a json string.
240
241    The following structures are supported: Obs, list, numpy.ndarray, Corr
242    If the list contains only one element, it is unpacked from the list.
243
244    Parameters
245    ----------
246    json_string : str
247        json string containing the data.
248    verbose : bool
249        Print additional information that was written to the file.
250    full_output : bool
251        If True, a dict containing auxiliary information and the data is returned.
252        If False, only the data is returned.
253    """
254
255    def _gen_obsd_from_datad(d):
256        retd = {}
257        if d:
258            retd['names'] = []
259            retd['idl'] = []
260            retd['deltas'] = []
261            retd['is_merged'] = {}
262            for ens in d:
263                for rep in ens['replica']:
264                    rep_name = rep['name']
265                    if len(rep_name) > len(ens["id"]):
266                        if rep_name[len(ens["id"])] != "|":
267                            tmp_list = list(rep_name)
268                            tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):]
269                            rep_name = ''.join(tmp_list)
270                    retd['names'].append(rep_name)
271                    retd['idl'].append([di[0] for di in rep['deltas']])
272                    retd['deltas'].append(np.array([di[1:] for di in rep['deltas']]))
273                    retd['is_merged'][rep_name] = rep.get('is_merged', False)
274        return retd
275
276    def _gen_covobsd_from_cdatad(d):
277        retd = {}
278        for ens in d:
279            retl = []
280            name = ens['id']
281            layouts = ens.get('layout', '1').strip()
282            layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
283            cov = np.reshape(ens['cov'], layout)
284            grad = ens['grad']
285            nobs = len(grad[0])
286            for i in range(nobs):
287                retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]})
288            retd[name] = retl
289        return retd
290
291    def get_Obs_from_dict(o):
292        layouts = o.get('layout', '1').strip()
293        if layouts != '1':
294            raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning)
295
296        values = o['value']
297        od = _gen_obsd_from_datad(o.get('data', {}))
298        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
299
300        if od:
301            ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl'])
302            ret._value = values[0]
303            ret.is_merged = od['is_merged']
304        else:
305            ret = Obs([], [], means=[])
306            ret._value = values[0]
307        for name in cd:
308            co = cd[name][0]
309            ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
310            ret.names.append(co['name'])
311
312        ret.reweighted = o.get('reweighted', False)
313        ret.tag = o.get('tag', [None])[0]
314        return ret
315
316    def get_List_from_dict(o):
317        layouts = o.get('layout', '1').strip()
318        layout = int(layouts)
319        values = o['value']
320        od = _gen_obsd_from_datad(o.get('data', {}))
321        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
322
323        ret = []
324        taglist = o.get('tag', layout * [None])
325        for i in range(layout):
326            if od:
327                ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl']))
328                ret[-1]._value = values[i]
329                ret[-1].is_merged = od['is_merged']
330            else:
331                ret.append(Obs([], [], means=[]))
332                ret[-1]._value = values[i]
333                print('Created Obs with means= ', values[i])
334            for name in cd:
335                co = cd[name][i]
336                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
337                ret[-1].names.append(co['name'])
338
339            ret[-1].reweighted = o.get('reweighted', False)
340            ret[-1].tag = taglist[i]
341        return ret
342
343    def get_Array_from_dict(o):
344        layouts = o.get('layout', '1').strip()
345        layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
346        N = np.prod(layout)
347        values = o['value']
348        od = _gen_obsd_from_datad(o.get('data', {}))
349        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
350
351        ret = []
352        taglist = o.get('tag', N * [None])
353        for i in range(N):
354            if od:
355                ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl']))
356                ret[-1]._value = values[i]
357                ret[-1].is_merged = od['is_merged']
358            else:
359                ret.append(Obs([], [], means=[]))
360                ret[-1]._value = values[i]
361            for name in cd:
362                co = cd[name][i]
363                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
364                ret[-1].names.append(co['name'])
365            ret[-1].reweighted = o.get('reweighted', False)
366            ret[-1].tag = taglist[i]
367        return np.reshape(ret, layout)
368
369    def get_Corr_from_dict(o):
370        if isinstance(o.get('tag'), list):  # supports the old way
371            taglist = o.get('tag')  # This had to be modified to get the taglist from the dictionary
372            temp_prange = None
373        elif isinstance(o.get('tag'), dict):
374            tagdic = o.get('tag')
375            taglist = tagdic['tag']
376            if 'prange' in tagdic:
377                temp_prange = tagdic['prange']
378            else:
379                temp_prange = None
380        else:
381            raise Exception("The tag is not a list or dict")
382
383        corr_tag = taglist[-1]
384        tmp_o = o
385        tmp_o['tag'] = taglist[:-1]
386        if len(tmp_o['tag']) == 0:
387            del tmp_o['tag']
388        dat = get_Array_from_dict(tmp_o)
389        my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)])
390        if corr_tag != 'None':
391            my_corr.tag = corr_tag
392
393        my_corr.prange = temp_prange
394        return my_corr
395
396    prog = json_dict.get('program', '')
397    version = json_dict.get('version', '')
398    who = json_dict.get('who', '')
399    date = json_dict.get('date', '')
400    host = json_dict.get('host', '')
401    if prog and verbose:
402        print('Data has been written using %s.' % (prog))
403    if version and verbose:
404        print('Format version %s' % (version))
405    if np.any([who, date, host] and verbose):
406        print('Written by %s on %s on host %s' % (who, date, host))
407    description = json_dict.get('description', '')
408    if description and verbose:
409        print()
410        print('Description: ', description)
411    obsdata = json_dict['obsdata']
412    ol = []
413    for io in obsdata:
414        if io['type'] == 'Obs':
415            ol.append(get_Obs_from_dict(io))
416        elif io['type'] == 'List':
417            ol.append(get_List_from_dict(io))
418        elif io['type'] == 'Array':
419            ol.append(get_Array_from_dict(io))
420        elif io['type'] == 'Corr':
421            ol.append(get_Corr_from_dict(io))
422        else:
423            raise Exception("Unknown datatype.")
424
425    if full_output:
426        retd = {}
427        retd['program'] = prog
428        retd['version'] = version
429        retd['who'] = who
430        retd['date'] = date
431        retd['host'] = host
432        retd['description'] = description
433        retd['obsdata'] = ol
434
435        return retd
436    else:
437        if len(obsdata) == 1:
438            ol = ol[0]
439
440        return ol
441
442
443def import_json_string(json_string, verbose=True, full_output=False):
444    """Reconstruct a list of Obs or structures containing Obs from a json string.
445
446    The following structures are supported: Obs, list, numpy.ndarray, Corr
447    If the list contains only one element, it is unpacked from the list.
448
449    Parameters
450    ----------
451    json_string : str
452        json string containing the data.
453    verbose : bool
454        Print additional information that was written to the file.
455    full_output : bool
456        If True, a dict containing auxiliary information and the data is returned.
457        If False, only the data is returned.
458    """
459
460    return _parse_json_dict(json.loads(json_string), verbose, full_output)
461
462
463def load_json(fname, verbose=True, gz=True, full_output=False):
464    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
465
466    The following structures are supported: Obs, list, numpy.ndarray, Corr
467    If the list contains only one element, it is unpacked from the list.
468
469    Parameters
470    ----------
471    fname : str
472        Filename of the input file.
473    verbose : bool
474        Print additional information that was written to the file.
475    gz : bool
476        If True, assumes that data is gzipped. If False, assumes JSON file.
477    full_output : bool
478        If True, a dict containing auxiliary information and the data is returned.
479        If False, only the data is returned.
480    """
481    if not fname.endswith('.json') and not fname.endswith('.gz'):
482        fname += '.json'
483    if gz:
484        if not fname.endswith('.gz'):
485            fname += '.gz'
486        with gzip.open(fname, 'r') as fin:
487            d = json.load(fin)
488    else:
489        if fname.endswith('.gz'):
490            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
491        with open(fname, 'r', encoding='utf-8') as fin:
492            d = json.loads(fin.read())
493
494    return _parse_json_dict(d, verbose, full_output)
495
496
497def _ol_from_dict(ind, reps='DICTOBS'):
498    """Convert a dictionary of Obs objects to a list and a dictionary that contains
499    placeholders instead of the Obs objects.
500
501    Parameters
502    ----------
503    ind : dict
504        Dict of JSON valid structures and objects that will be exported.
505        At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr.
506        All Obs inside a structure have to be defined on the same set of configurations.
507    reps : str
508        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
509    """
510
511    obstypes = (Obs, Corr, np.ndarray)
512
513    if not reps.isalnum():
514        raise Exception('Placeholder string has to be alphanumeric!')
515    ol = []
516    counter = 0
517
518    def dict_replace_obs(d):
519        nonlocal ol
520        nonlocal counter
521        x = {}
522        for k, v in d.items():
523            if isinstance(v, dict):
524                v = dict_replace_obs(v)
525            elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]):
526                v = obslist_replace_obs(v)
527            elif isinstance(v, list):
528                v = list_replace_obs(v)
529            elif isinstance(v, obstypes):
530                ol.append(v)
531                v = reps + '%d' % (counter)
532                counter += 1
533            elif isinstance(v, str):
534                if bool(re.match(r'%s[0-9]+' % (reps), v)):
535                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps))
536            x[k] = v
537        return x
538
539    def list_replace_obs(li):
540        nonlocal ol
541        nonlocal counter
542        x = []
543        for e in li:
544            if isinstance(e, list):
545                e = list_replace_obs(e)
546            elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]):
547                e = obslist_replace_obs(e)
548            elif isinstance(e, dict):
549                e = dict_replace_obs(e)
550            elif isinstance(e, obstypes):
551                ol.append(e)
552                e = reps + '%d' % (counter)
553                counter += 1
554            elif isinstance(e, str):
555                if bool(re.match(r'%s[0-9]+' % (reps), e)):
556                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps))
557            x.append(e)
558        return x
559
560    def obslist_replace_obs(li):
561        nonlocal ol
562        nonlocal counter
563        il = []
564        for e in li:
565            il.append(e)
566
567        ol.append(il)
568        x = reps + '%d' % (counter)
569        counter += 1
570        return x
571
572    nd = dict_replace_obs(ind)
573
574    return ol, nd
575
576
577def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
578    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
579
580    Parameters
581    ----------
582    od : dict
583        Dict of JSON valid structures and objects that will be exported.
584        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
585        All Obs inside a structure have to be defined on the same set of configurations.
586    fname : str
587        Filename of the output file.
588    description : str
589        Optional string that describes the contents of the json file.
590    indent : int
591        Specify the indentation level of the json file. None or 0 is permissible and
592        saves disk space.
593    reps : str
594        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
595    gz : bool
596        If True, the output is a gzipped json. If False, the output is a json file.
597    """
598
599    if not isinstance(od, dict):
600        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
601
602    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
603                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
604                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
605                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
606
607    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
608    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
609
610    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
611
612
613def _od_from_list_and_dict(ol, ind, reps='DICTOBS'):
614    """Parse a list of Obs or structures containing Obs and an accompanying
615    dict, where the structures have been replaced by placeholders to a
616    dict that contains the structures.
617
618    The following structures are supported: Obs, list, numpy.ndarray, Corr
619
620    Parameters
621    ----------
622    ol : list
623        List of objects -
624        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
625        All Obs inside a structure have to be defined on the same set of configurations.
626    ind : dict
627        Dict that defines the structure of the resulting dict and contains placeholders
628    reps : str
629        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
630    """
631    if not reps.isalnum():
632        raise Exception('Placeholder string has to be alphanumeric!')
633
634    counter = 0
635
636    def dict_replace_string(d):
637        nonlocal counter
638        nonlocal ol
639        x = {}
640        for k, v in d.items():
641            if isinstance(v, dict):
642                v = dict_replace_string(v)
643            elif isinstance(v, list):
644                v = list_replace_string(v)
645            elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)):
646                index = int(v[len(reps):])
647                v = ol[index]
648                counter += 1
649            x[k] = v
650        return x
651
652    def list_replace_string(li):
653        nonlocal counter
654        nonlocal ol
655        x = []
656        for e in li:
657            if isinstance(e, list):
658                e = list_replace_string(e)
659            elif isinstance(e, dict):
660                e = dict_replace_string(e)
661            elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)):
662                index = int(e[len(reps):])
663                e = ol[index]
664                counter += 1
665            x.append(e)
666        return x
667
668    nd = dict_replace_string(ind)
669
670    if counter == 0:
671        raise Exception('No placeholder has been replaced! Check if reps is set correctly.')
672
673    return nd
674
675
676def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
677    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
678
679    The following structures are supported: Obs, list, numpy.ndarray, Corr
680
681    Parameters
682    ----------
683    fname : str
684        Filename of the input file.
685    verbose : bool
686        Print additional information that was written to the file.
687    gz : bool
688        If True, assumes that data is gzipped. If False, assumes JSON file.
689    full_output : bool
690        If True, a dict containing auxiliary information and the data is returned.
691        If False, only the data is returned.
692    reps : str
693        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
694    """
695    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
696    description = indata['description']['description']
697    indict = indata['description']['OBSDICT']
698    ol = indata['obsdata']
699    od = _od_from_list_and_dict(ol, indict, reps=reps)
700
701    if full_output:
702        indata['description'] = description
703        indata['obsdata'] = od
704        return indata
705    else:
706        return od