pyerrors.input.json

  1import rapidjson as json
  2import gzip
  3import getpass
  4import socket
  5import datetime
  6import platform
  7import warnings
  8import re
  9import numpy as np
 10from ..obs import Obs
 11from ..covobs import Covobs
 12from ..correlators import Corr
 13from ..misc import _assert_equal_properties
 14from .. import version as pyerrorsversion
 15
 16
 17def create_json_string(ol, description='', indent=1):
 18    """Generate the string for the export of a list of Obs or structures containing Obs
 19    to a .json(.gz) file
 20
 21    Parameters
 22    ----------
 23    ol : list
 24        List of objects that will be exported. At the moment, these objects can be
 25        either of: Obs, list, numpy.ndarray, Corr.
 26        All Obs inside a structure have to be defined on the same set of configurations.
 27    description : str
 28        Optional string that describes the contents of the json file.
 29    indent : int
 30        Specify the indentation level of the json file. None or 0 is permissible and
 31        saves disk space.
 32    """
 33
 34    def _gen_data_d_from_list(ol):
 35        dl = []
 36        No = len(ol)
 37        for name in ol[0].mc_names:
 38            ed = {}
 39            ed['id'] = name
 40            ed['replica'] = []
 41            for r_name in ol[0].e_content[name]:
 42                rd = {}
 43                rd['name'] = r_name
 44                if ol[0].is_merged.get(r_name, False):
 45                    rd['is_merged'] = True
 46                rd['deltas'] = []
 47                offsets = [o.r_values[r_name] - o.value for o in ol]
 48                deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)])
 49                for i in range(len(ol[0].idl[r_name])):
 50                    rd['deltas'].append([ol[0].idl[r_name][i]])
 51                    rd['deltas'][-1] += deltas[i].tolist()
 52                ed['replica'].append(rd)
 53            dl.append(ed)
 54        return dl
 55
 56    def _gen_cdata_d_from_list(ol):
 57        dl = []
 58        for name in ol[0].cov_names:
 59            ed = {}
 60            ed['id'] = name
 61            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
 62            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
 63            ncov = ol[0].covobs[name].cov.shape[0]
 64            ed['grad'] = []
 65            for i in range(ncov):
 66                ed['grad'].append([])
 67                for o in ol:
 68                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
 69            dl.append(ed)
 70        return dl
 71
 72    def write_Obs_to_dict(o):
 73        d = {}
 74        d['type'] = 'Obs'
 75        d['layout'] = '1'
 76        if o.tag:
 77            d['tag'] = [o.tag]
 78        if o.reweighted:
 79            d['reweighted'] = o.reweighted
 80        d['value'] = [o.value]
 81        data = _gen_data_d_from_list([o])
 82        if len(data) > 0:
 83            d['data'] = data
 84        cdata = _gen_cdata_d_from_list([o])
 85        if len(cdata) > 0:
 86            d['cdata'] = cdata
 87        return d
 88
 89    def write_List_to_dict(ol):
 90        _assert_equal_properties(ol)
 91        d = {}
 92        d['type'] = 'List'
 93        d['layout'] = '%d' % len(ol)
 94        taglist = [o.tag for o in ol]
 95        if np.any([tag is not None for tag in taglist]):
 96            d['tag'] = taglist
 97        if ol[0].reweighted:
 98            d['reweighted'] = ol[0].reweighted
 99        d['value'] = [o.value for o in ol]
100        data = _gen_data_d_from_list(ol)
101        if len(data) > 0:
102            d['data'] = data
103        cdata = _gen_cdata_d_from_list(ol)
104        if len(cdata) > 0:
105            d['cdata'] = cdata
106        return d
107
108    def write_Array_to_dict(oa):
109        ol = np.ravel(oa)
110        _assert_equal_properties(ol)
111        d = {}
112        d['type'] = 'Array'
113        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
114        taglist = [o.tag for o in ol]
115        if np.any([tag is not None for tag in taglist]):
116            d['tag'] = taglist
117        if ol[0].reweighted:
118            d['reweighted'] = ol[0].reweighted
119        d['value'] = [o.value for o in ol]
120        data = _gen_data_d_from_list(ol)
121        if len(data) > 0:
122            d['data'] = data
123        cdata = _gen_cdata_d_from_list(ol)
124        if len(cdata) > 0:
125            d['cdata'] = cdata
126        return d
127
128    def _nan_Obs_like(obs):
129        samples = []
130        names = []
131        idl = []
132        for key, value in obs.idl.items():
133            samples.append([np.nan] * len(value))
134            names.append(key)
135            idl.append(value)
136        my_obs = Obs(samples, names, idl)
137        my_obs._covobs = obs._covobs
138        for name in obs._covobs:
139            my_obs.names.append(name)
140        my_obs.reweighted = obs.reweighted
141        my_obs.is_merged = obs.is_merged
142        return my_obs
143
144    def write_Corr_to_dict(my_corr):
145        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
146        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
147        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
148        content = [o if o is not None else dummy_array for o in my_corr.content]
149        dat = write_Array_to_dict(np.array(content, dtype=object))
150        dat['type'] = 'Corr'
151        corr_meta_data = str(my_corr.tag)
152        if 'tag' in dat.keys():
153            dat['tag'].append(corr_meta_data)
154        else:
155            dat['tag'] = [corr_meta_data]
156        taglist = dat['tag']
157        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
158        dat['tag']['tag'] = taglist
159        if my_corr.prange is not None:
160            dat['tag']['prange'] = my_corr.prange
161        return dat
162
163    if not isinstance(ol, list):
164        ol = [ol]
165
166    d = {}
167    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
168    d['version'] = '1.1'
169    d['who'] = getpass.getuser()
170    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
171    d['host'] = socket.gethostname() + ', ' + platform.platform()
172
173    if description:
174        d['description'] = description
175
176    d['obsdata'] = []
177    for io in ol:
178        if isinstance(io, Obs):
179            d['obsdata'].append(write_Obs_to_dict(io))
180        elif isinstance(io, list):
181            d['obsdata'].append(write_List_to_dict(io))
182        elif isinstance(io, np.ndarray):
183            d['obsdata'].append(write_Array_to_dict(io))
184        elif isinstance(io, Corr):
185            d['obsdata'].append(write_Corr_to_dict(io))
186        else:
187            raise Exception("Unkown datatype.")
188
189    if indent:
190        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY)
191    else:
192        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT)
193
194
195def dump_to_json(ol, fname, description='', indent=1, gz=True):
196    """Export a list of Obs or structures containing Obs to a .json(.gz) file
197
198    Parameters
199    ----------
200    ol : list
201        List of objects that will be exported. At the moment, these objects can be
202        either of: Obs, list, numpy.ndarray, Corr.
203        All Obs inside a structure have to be defined on the same set of configurations.
204    fname : str
205        Filename of the output file.
206    description : str
207        Optional string that describes the contents of the json file.
208    indent : int
209        Specify the indentation level of the json file. None or 0 is permissible and
210        saves disk space.
211    gz : bool
212        If True, the output is a gzipped json. If False, the output is a json file.
213    """
214
215    jsonstring = create_json_string(ol, description, indent)
216
217    if not fname.endswith('.json') and not fname.endswith('.gz'):
218        fname += '.json'
219
220    if gz:
221        if not fname.endswith('.gz'):
222            fname += '.gz'
223
224        fp = gzip.open(fname, 'wb')
225        fp.write(jsonstring.encode('utf-8'))
226    else:
227        fp = open(fname, 'w', encoding='utf-8')
228        fp.write(jsonstring)
229    fp.close()
230
231
232def _parse_json_dict(json_dict, verbose=True, full_output=False):
233    """Reconstruct a list of Obs or structures containing Obs from a dict that
234    was built out of a json string.
235
236    The following structures are supported: Obs, list, numpy.ndarray, Corr
237    If the list contains only one element, it is unpacked from the list.
238
239    Parameters
240    ----------
241    json_string : str
242        json string containing the data.
243    verbose : bool
244        Print additional information that was written to the file.
245    full_output : bool
246        If True, a dict containing auxiliary information and the data is returned.
247        If False, only the data is returned.
248    """
249
250    def _gen_obsd_from_datad(d):
251        retd = {}
252        if d:
253            retd['names'] = []
254            retd['idl'] = []
255            retd['deltas'] = []
256            retd['is_merged'] = {}
257            for ens in d:
258                for rep in ens['replica']:
259                    rep_name = rep['name']
260                    if len(rep_name) > len(ens["id"]):
261                        if rep_name[len(ens["id"])] != "|":
262                            tmp_list = list(rep_name)
263                            tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):]
264                            rep_name = ''.join(tmp_list)
265                    retd['names'].append(rep_name)
266                    retd['idl'].append([di[0] for di in rep['deltas']])
267                    retd['deltas'].append(np.array([di[1:] for di in rep['deltas']]))
268                    retd['is_merged'][rep_name] = rep.get('is_merged', False)
269        return retd
270
271    def _gen_covobsd_from_cdatad(d):
272        retd = {}
273        for ens in d:
274            retl = []
275            name = ens['id']
276            layouts = ens.get('layout', '1').strip()
277            layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
278            cov = np.reshape(ens['cov'], layout)
279            grad = ens['grad']
280            nobs = len(grad[0])
281            for i in range(nobs):
282                retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]})
283            retd[name] = retl
284        return retd
285
286    def get_Obs_from_dict(o):
287        layouts = o.get('layout', '1').strip()
288        if layouts != '1':
289            raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning)
290
291        values = o['value']
292        od = _gen_obsd_from_datad(o.get('data', {}))
293        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
294
295        if od:
296            ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl'])
297            ret._value = values[0]
298            ret.is_merged = od['is_merged']
299        else:
300            ret = Obs([], [], means=[])
301            ret._value = values[0]
302        for name in cd:
303            co = cd[name][0]
304            ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
305            ret.names.append(co['name'])
306
307        ret.reweighted = o.get('reweighted', False)
308        ret.tag = o.get('tag', [None])[0]
309        return ret
310
311    def get_List_from_dict(o):
312        layouts = o.get('layout', '1').strip()
313        layout = int(layouts)
314        values = o['value']
315        od = _gen_obsd_from_datad(o.get('data', {}))
316        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
317
318        ret = []
319        taglist = o.get('tag', layout * [None])
320        for i in range(layout):
321            if od:
322                ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl']))
323                ret[-1]._value = values[i]
324                ret[-1].is_merged = od['is_merged']
325            else:
326                ret.append(Obs([], [], means=[]))
327                ret[-1]._value = values[i]
328                print('Created Obs with means= ', values[i])
329            for name in cd:
330                co = cd[name][i]
331                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
332                ret[-1].names.append(co['name'])
333
334            ret[-1].reweighted = o.get('reweighted', False)
335            ret[-1].tag = taglist[i]
336        return ret
337
338    def get_Array_from_dict(o):
339        layouts = o.get('layout', '1').strip()
340        layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
341        N = np.prod(layout)
342        values = o['value']
343        od = _gen_obsd_from_datad(o.get('data', {}))
344        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
345
346        ret = []
347        taglist = o.get('tag', N * [None])
348        for i in range(N):
349            if od:
350                ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl']))
351                ret[-1]._value = values[i]
352                ret[-1].is_merged = od['is_merged']
353            else:
354                ret.append(Obs([], [], means=[]))
355                ret[-1]._value = values[i]
356            for name in cd:
357                co = cd[name][i]
358                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
359                ret[-1].names.append(co['name'])
360            ret[-1].reweighted = o.get('reweighted', False)
361            ret[-1].tag = taglist[i]
362        return np.reshape(ret, layout)
363
364    def get_Corr_from_dict(o):
365        if isinstance(o.get('tag'), list):  # supports the old way
366            taglist = o.get('tag')  # This had to be modified to get the taglist from the dictionary
367            temp_prange = None
368        elif isinstance(o.get('tag'), dict):
369            tagdic = o.get('tag')
370            taglist = tagdic['tag']
371            if 'prange' in tagdic:
372                temp_prange = tagdic['prange']
373            else:
374                temp_prange = None
375        else:
376            raise Exception("The tag is not a list or dict")
377
378        corr_tag = taglist[-1]
379        tmp_o = o
380        tmp_o['tag'] = taglist[:-1]
381        if len(tmp_o['tag']) == 0:
382            del tmp_o['tag']
383        dat = get_Array_from_dict(tmp_o)
384        my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)])
385        if corr_tag != 'None':
386            my_corr.tag = corr_tag
387
388        my_corr.prange = temp_prange
389        return my_corr
390
391    prog = json_dict.get('program', '')
392    version = json_dict.get('version', '')
393    who = json_dict.get('who', '')
394    date = json_dict.get('date', '')
395    host = json_dict.get('host', '')
396    if prog and verbose:
397        print('Data has been written using %s.' % (prog))
398    if version and verbose:
399        print('Format version %s' % (version))
400    if np.any([who, date, host] and verbose):
401        print('Written by %s on %s on host %s' % (who, date, host))
402    description = json_dict.get('description', '')
403    if description and verbose:
404        print()
405        print('Description: ', description)
406    obsdata = json_dict['obsdata']
407    ol = []
408    for io in obsdata:
409        if io['type'] == 'Obs':
410            ol.append(get_Obs_from_dict(io))
411        elif io['type'] == 'List':
412            ol.append(get_List_from_dict(io))
413        elif io['type'] == 'Array':
414            ol.append(get_Array_from_dict(io))
415        elif io['type'] == 'Corr':
416            ol.append(get_Corr_from_dict(io))
417        else:
418            raise Exception("Unkown datatype.")
419
420    if full_output:
421        retd = {}
422        retd['program'] = prog
423        retd['version'] = version
424        retd['who'] = who
425        retd['date'] = date
426        retd['host'] = host
427        retd['description'] = description
428        retd['obsdata'] = ol
429
430        return retd
431    else:
432        if len(obsdata) == 1:
433            ol = ol[0]
434
435        return ol
436
437
438def import_json_string(json_string, verbose=True, full_output=False):
439    """Reconstruct a list of Obs or structures containing Obs from a json string.
440
441    The following structures are supported: Obs, list, numpy.ndarray, Corr
442    If the list contains only one element, it is unpacked from the list.
443
444    Parameters
445    ----------
446    json_string : str
447        json string containing the data.
448    verbose : bool
449        Print additional information that was written to the file.
450    full_output : bool
451        If True, a dict containing auxiliary information and the data is returned.
452        If False, only the data is returned.
453    """
454
455    return _parse_json_dict(json.loads(json_string), verbose, full_output)
456
457
458def load_json(fname, verbose=True, gz=True, full_output=False):
459    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
460
461    The following structures are supported: Obs, list, numpy.ndarray, Corr
462    If the list contains only one element, it is unpacked from the list.
463
464    Parameters
465    ----------
466    fname : str
467        Filename of the input file.
468    verbose : bool
469        Print additional information that was written to the file.
470    gz : bool
471        If True, assumes that data is gzipped. If False, assumes JSON file.
472    full_output : bool
473        If True, a dict containing auxiliary information and the data is returned.
474        If False, only the data is returned.
475    """
476    if not fname.endswith('.json') and not fname.endswith('.gz'):
477        fname += '.json'
478    if gz:
479        if not fname.endswith('.gz'):
480            fname += '.gz'
481        with gzip.open(fname, 'r') as fin:
482            d = json.load(fin)
483    else:
484        if fname.endswith('.gz'):
485            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
486        with open(fname, 'r', encoding='utf-8') as fin:
487            d = json.loads(fin.read())
488
489    return _parse_json_dict(d, verbose, full_output)
490
491
492def _ol_from_dict(ind, reps='DICTOBS'):
493    """Convert a dictionary of Obs objects to a list and a dictionary that contains
494    placeholders instead of the Obs objects.
495
496    Parameters
497    ----------
498    ind : dict
499        Dict of JSON valid structures and objects that will be exported.
500        At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr.
501        All Obs inside a structure have to be defined on the same set of configurations.
502    reps : str
503        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
504    """
505
506    obstypes = (Obs, Corr, np.ndarray)
507
508    if not reps.isalnum():
509        raise Exception('Placeholder string has to be alphanumeric!')
510    ol = []
511    counter = 0
512
513    def dict_replace_obs(d):
514        nonlocal ol
515        nonlocal counter
516        x = {}
517        for k, v in d.items():
518            if isinstance(v, dict):
519                v = dict_replace_obs(v)
520            elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]):
521                v = obslist_replace_obs(v)
522            elif isinstance(v, list):
523                v = list_replace_obs(v)
524            elif isinstance(v, obstypes):
525                ol.append(v)
526                v = reps + '%d' % (counter)
527                counter += 1
528            elif isinstance(v, str):
529                if bool(re.match(r'%s[0-9]+' % (reps), v)):
530                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps))
531            x[k] = v
532        return x
533
534    def list_replace_obs(li):
535        nonlocal ol
536        nonlocal counter
537        x = []
538        for e in li:
539            if isinstance(e, list):
540                e = list_replace_obs(e)
541            elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]):
542                e = obslist_replace_obs(e)
543            elif isinstance(e, dict):
544                e = dict_replace_obs(e)
545            elif isinstance(e, obstypes):
546                ol.append(e)
547                e = reps + '%d' % (counter)
548                counter += 1
549            elif isinstance(e, str):
550                if bool(re.match(r'%s[0-9]+' % (reps), e)):
551                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps))
552            x.append(e)
553        return x
554
555    def obslist_replace_obs(li):
556        nonlocal ol
557        nonlocal counter
558        il = []
559        for e in li:
560            il.append(e)
561
562        ol.append(il)
563        x = reps + '%d' % (counter)
564        counter += 1
565        return x
566
567    nd = dict_replace_obs(ind)
568
569    return ol, nd
570
571
572def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
573    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
574
575    Parameters
576    ----------
577    od : dict
578        Dict of JSON valid structures and objects that will be exported.
579        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
580        All Obs inside a structure have to be defined on the same set of configurations.
581    fname : str
582        Filename of the output file.
583    description : str
584        Optional string that describes the contents of the json file.
585    indent : int
586        Specify the indentation level of the json file. None or 0 is permissible and
587        saves disk space.
588    reps : str
589        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
590    gz : bool
591        If True, the output is a gzipped json. If False, the output is a json file.
592    """
593
594    if not isinstance(od, dict):
595        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
596
597    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
598                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
599                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
600                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
601
602    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
603    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
604
605    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
606
607
608def _od_from_list_and_dict(ol, ind, reps='DICTOBS'):
609    """Parse a list of Obs or structures containing Obs and an accompanying
610    dict, where the structures have been replaced by placeholders to a
611    dict that contains the structures.
612
613    The following structures are supported: Obs, list, numpy.ndarray, Corr
614
615    Parameters
616    ----------
617    ol : list
618        List of objects -
619        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
620        All Obs inside a structure have to be defined on the same set of configurations.
621    ind : dict
622        Dict that defines the structure of the resulting dict and contains placeholders
623    reps : str
624        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
625    """
626    if not reps.isalnum():
627        raise Exception('Placeholder string has to be alphanumeric!')
628
629    counter = 0
630
631    def dict_replace_string(d):
632        nonlocal counter
633        nonlocal ol
634        x = {}
635        for k, v in d.items():
636            if isinstance(v, dict):
637                v = dict_replace_string(v)
638            elif isinstance(v, list):
639                v = list_replace_string(v)
640            elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)):
641                index = int(v[len(reps):])
642                v = ol[index]
643                counter += 1
644            x[k] = v
645        return x
646
647    def list_replace_string(li):
648        nonlocal counter
649        nonlocal ol
650        x = []
651        for e in li:
652            if isinstance(e, list):
653                e = list_replace_string(e)
654            elif isinstance(e, dict):
655                e = dict_replace_string(e)
656            elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)):
657                index = int(e[len(reps):])
658                e = ol[index]
659                counter += 1
660            x.append(e)
661        return x
662
663    nd = dict_replace_string(ind)
664
665    if counter == 0:
666        raise Exception('No placeholder has been replaced! Check if reps is set correctly.')
667
668    return nd
669
670
671def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
672    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
673
674    The following structures are supported: Obs, list, numpy.ndarray, Corr
675
676    Parameters
677    ----------
678    fname : str
679        Filename of the input file.
680    verbose : bool
681        Print additional information that was written to the file.
682    gz : bool
683        If True, assumes that data is gzipped. If False, assumes JSON file.
684    full_output : bool
685        If True, a dict containing auxiliary information and the data is returned.
686        If False, only the data is returned.
687    reps : str
688        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
689    """
690    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
691    description = indata['description']['description']
692    indict = indata['description']['OBSDICT']
693    ol = indata['obsdata']
694    od = _od_from_list_and_dict(ol, indict, reps=reps)
695
696    if full_output:
697        indata['description'] = description
698        indata['obsdata'] = od
699        return indata
700    else:
701        return od