pyerrors.input.json

  1import rapidjson as json
  2import gzip
  3import getpass
  4import socket
  5import datetime
  6import platform
  7import warnings
  8import re
  9import numpy as np
 10from ..obs import Obs
 11from ..covobs import Covobs
 12from ..correlators import Corr
 13from ..misc import _assert_equal_properties
 14from .. import version as pyerrorsversion
 15
 16
 17def create_json_string(ol, description='', indent=1):
 18    """Generate the string for the export of a list of Obs or structures containing Obs
 19    to a .json(.gz) file
 20
 21    Parameters
 22    ----------
 23    ol : list
 24        List of objects that will be exported. At the moment, these objects can be
 25        either of: Obs, list, numpy.ndarray, Corr.
 26        All Obs inside a structure have to be defined on the same set of configurations.
 27    description : str
 28        Optional string that describes the contents of the json file.
 29    indent : int
 30        Specify the indentation level of the json file. None or 0 is permissible and
 31        saves disk space.
 32    """
 33
 34    def _gen_data_d_from_list(ol):
 35        dl = []
 36        for name in ol[0].mc_names:
 37            ed = {}
 38            ed['id'] = name
 39            ed['replica'] = []
 40            for r_name in ol[0].e_content[name]:
 41                rd = {}
 42                rd['name'] = r_name
 43                if ol[0].is_merged.get(r_name, False):
 44                    rd['is_merged'] = True
 45                rd['deltas'] = []
 46                for i in range(len(ol[0].idl[r_name])):
 47                    rd['deltas'].append([ol[0].idl[r_name][i]])
 48                    for o in ol:
 49                        rd['deltas'][-1].append(o.deltas[r_name][i])
 50                ed['replica'].append(rd)
 51            dl.append(ed)
 52        return dl
 53
 54    def _gen_cdata_d_from_list(ol):
 55        dl = []
 56        for name in ol[0].cov_names:
 57            ed = {}
 58            ed['id'] = name
 59            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
 60            ed['cov'] = list(np.ravel(ol[0].covobs[name].cov))
 61            ncov = ol[0].covobs[name].cov.shape[0]
 62            ed['grad'] = []
 63            for i in range(ncov):
 64                ed['grad'].append([])
 65                for o in ol:
 66                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
 67            dl.append(ed)
 68        return dl
 69
 70    def write_Obs_to_dict(o):
 71        d = {}
 72        d['type'] = 'Obs'
 73        d['layout'] = '1'
 74        if o.tag:
 75            d['tag'] = [o.tag]
 76        if o.reweighted:
 77            d['reweighted'] = o.reweighted
 78        d['value'] = [o.value]
 79        data = _gen_data_d_from_list([o])
 80        if len(data) > 0:
 81            d['data'] = data
 82        cdata = _gen_cdata_d_from_list([o])
 83        if len(cdata) > 0:
 84            d['cdata'] = cdata
 85        return d
 86
 87    def write_List_to_dict(ol):
 88        _assert_equal_properties(ol)
 89        d = {}
 90        d['type'] = 'List'
 91        d['layout'] = '%d' % len(ol)
 92        taglist = [o.tag for o in ol]
 93        if np.any([tag is not None for tag in taglist]):
 94            d['tag'] = taglist
 95        if ol[0].reweighted:
 96            d['reweighted'] = ol[0].reweighted
 97        d['value'] = [o.value for o in ol]
 98        data = _gen_data_d_from_list(ol)
 99        if len(data) > 0:
100            d['data'] = data
101        cdata = _gen_cdata_d_from_list(ol)
102        if len(cdata) > 0:
103            d['cdata'] = cdata
104        return d
105
106    def write_Array_to_dict(oa):
107        ol = np.ravel(oa)
108        _assert_equal_properties(ol)
109        d = {}
110        d['type'] = 'Array'
111        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
112        taglist = [o.tag for o in ol]
113        if np.any([tag is not None for tag in taglist]):
114            d['tag'] = taglist
115        if ol[0].reweighted:
116            d['reweighted'] = ol[0].reweighted
117        d['value'] = [o.value for o in ol]
118        data = _gen_data_d_from_list(ol)
119        if len(data) > 0:
120            d['data'] = data
121        cdata = _gen_cdata_d_from_list(ol)
122        if len(cdata) > 0:
123            d['cdata'] = cdata
124        return d
125
126    def _nan_Obs_like(obs):
127        samples = []
128        names = []
129        idl = []
130        for key, value in obs.idl.items():
131            samples.append([np.nan] * len(value))
132            names.append(key)
133            idl.append(value)
134        my_obs = Obs(samples, names, idl)
135        my_obs._covobs = obs._covobs
136        for name in obs._covobs:
137            my_obs.names.append(name)
138        my_obs.reweighted = obs.reweighted
139        my_obs.is_merged = obs.is_merged
140        return my_obs
141
142    def write_Corr_to_dict(my_corr):
143        first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j))
144        dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object)
145        dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0])
146        content = [o if o is not None else dummy_array for o in my_corr.content]
147        dat = write_Array_to_dict(np.array(content, dtype=object))
148        dat['type'] = 'Corr'
149        corr_meta_data = str(my_corr.tag)
150        if 'tag' in dat.keys():
151            dat['tag'].append(corr_meta_data)
152        else:
153            dat['tag'] = [corr_meta_data]
154        taglist = dat['tag']
155        dat['tag'] = {}  # tag is now a dictionary, that contains the previous taglist in the key "tag"
156        dat['tag']['tag'] = taglist
157        if my_corr.prange is not None:
158            dat['tag']['prange'] = my_corr.prange
159        return dat
160
161    if not isinstance(ol, list):
162        ol = [ol]
163
164    d = {}
165    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
166    d['version'] = '1.0'
167    d['who'] = getpass.getuser()
168    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
169    d['host'] = socket.gethostname() + ', ' + platform.platform()
170
171    if description:
172        d['description'] = description
173
174    d['obsdata'] = []
175    for io in ol:
176        if isinstance(io, Obs):
177            d['obsdata'].append(write_Obs_to_dict(io))
178        elif isinstance(io, list):
179            d['obsdata'].append(write_List_to_dict(io))
180        elif isinstance(io, np.ndarray):
181            d['obsdata'].append(write_Array_to_dict(io))
182        elif isinstance(io, Corr):
183            d['obsdata'].append(write_Corr_to_dict(io))
184        else:
185            raise Exception("Unkown datatype.")
186
187    if indent:
188        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY)
189    else:
190        return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT)
191
192
193def dump_to_json(ol, fname, description='', indent=1, gz=True):
194    """Export a list of Obs or structures containing Obs to a .json(.gz) file
195
196    Parameters
197    ----------
198    ol : list
199        List of objects that will be exported. At the moment, these objects can be
200        either of: Obs, list, numpy.ndarray, Corr.
201        All Obs inside a structure have to be defined on the same set of configurations.
202    fname : str
203        Filename of the output file.
204    description : str
205        Optional string that describes the contents of the json file.
206    indent : int
207        Specify the indentation level of the json file. None or 0 is permissible and
208        saves disk space.
209    gz : bool
210        If True, the output is a gzipped json. If False, the output is a json file.
211    """
212
213    jsonstring = create_json_string(ol, description, indent)
214
215    if not fname.endswith('.json') and not fname.endswith('.gz'):
216        fname += '.json'
217
218    if gz:
219        if not fname.endswith('.gz'):
220            fname += '.gz'
221
222        fp = gzip.open(fname, 'wb')
223        fp.write(jsonstring.encode('utf-8'))
224    else:
225        fp = open(fname, 'w', encoding='utf-8')
226        fp.write(jsonstring)
227    fp.close()
228
229
230def _parse_json_dict(json_dict, verbose=True, full_output=False):
231    """Reconstruct a list of Obs or structures containing Obs from a dict that
232    was built out of a json string.
233
234    The following structures are supported: Obs, list, numpy.ndarray, Corr
235    If the list contains only one element, it is unpacked from the list.
236
237    Parameters
238    ----------
239    json_string : str
240        json string containing the data.
241    verbose : bool
242        Print additional information that was written to the file.
243    full_output : bool
244        If True, a dict containing auxiliary information and the data is returned.
245        If False, only the data is returned.
246    """
247
248    def _gen_obsd_from_datad(d):
249        retd = {}
250        if d:
251            retd['names'] = []
252            retd['idl'] = []
253            retd['deltas'] = []
254            retd['is_merged'] = {}
255            for ens in d:
256                for rep in ens['replica']:
257                    rep_name = rep['name']
258                    if len(rep_name) > len(ens["id"]):
259                        if rep_name[len(ens["id"])] != "|":
260                            tmp_list = list(rep_name)
261                            tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):]
262                            rep_name = ''.join(tmp_list)
263                    retd['names'].append(rep_name)
264                    retd['idl'].append([di[0] for di in rep['deltas']])
265                    retd['deltas'].append(np.array([di[1:] for di in rep['deltas']]))
266                    retd['is_merged'][rep_name] = rep.get('is_merged', False)
267        return retd
268
269    def _gen_covobsd_from_cdatad(d):
270        retd = {}
271        for ens in d:
272            retl = []
273            name = ens['id']
274            layouts = ens.get('layout', '1').strip()
275            layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
276            cov = np.reshape(ens['cov'], layout)
277            grad = ens['grad']
278            nobs = len(grad[0])
279            for i in range(nobs):
280                retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]})
281            retd[name] = retl
282        return retd
283
284    def get_Obs_from_dict(o):
285        layouts = o.get('layout', '1').strip()
286        if layouts != '1':
287            raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning)
288
289        values = o['value']
290        od = _gen_obsd_from_datad(o.get('data', {}))
291        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
292
293        if od:
294            ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl'])
295            ret.is_merged = od['is_merged']
296        else:
297            ret = Obs([], [], means=[])
298            ret._value = values[0]
299        for name in cd:
300            co = cd[name][0]
301            ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
302            ret.names.append(co['name'])
303
304        ret.reweighted = o.get('reweighted', False)
305        ret.tag = o.get('tag', [None])[0]
306        return ret
307
308    def get_List_from_dict(o):
309        layouts = o.get('layout', '1').strip()
310        layout = int(layouts)
311        values = o['value']
312        od = _gen_obsd_from_datad(o.get('data', {}))
313        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
314
315        ret = []
316        taglist = o.get('tag', layout * [None])
317        for i in range(layout):
318            if od:
319                ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl']))
320                ret[-1].is_merged = od['is_merged']
321            else:
322                ret.append(Obs([], [], means=[]))
323                ret[-1]._value = values[i]
324                print('Created Obs with means= ', values[i])
325            for name in cd:
326                co = cd[name][i]
327                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
328                ret[-1].names.append(co['name'])
329
330            ret[-1].reweighted = o.get('reweighted', False)
331            ret[-1].tag = taglist[i]
332        return ret
333
334    def get_Array_from_dict(o):
335        layouts = o.get('layout', '1').strip()
336        layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
337        N = np.prod(layout)
338        values = o['value']
339        od = _gen_obsd_from_datad(o.get('data', {}))
340        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
341
342        ret = []
343        taglist = o.get('tag', N * [None])
344        for i in range(N):
345            if od:
346                ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl']))
347                ret[-1].is_merged = od['is_merged']
348            else:
349                ret.append(Obs([], [], means=[]))
350                ret[-1]._value = values[i]
351            for name in cd:
352                co = cd[name][i]
353                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
354                ret[-1].names.append(co['name'])
355            ret[-1].reweighted = o.get('reweighted', False)
356            ret[-1].tag = taglist[i]
357        return np.reshape(ret, layout)
358
359    def get_Corr_from_dict(o):
360        if isinstance(o.get('tag'), list):  # supports the old way
361            taglist = o.get('tag')  # This had to be modified to get the taglist from the dictionary
362            temp_prange = None
363        elif isinstance(o.get('tag'), dict):
364            tagdic = o.get('tag')
365            taglist = tagdic['tag']
366            if 'prange' in tagdic:
367                temp_prange = tagdic['prange']
368            else:
369                temp_prange = None
370        else:
371            raise Exception("The tag is not a list or dict")
372
373        corr_tag = taglist[-1]
374        tmp_o = o
375        tmp_o['tag'] = taglist[:-1]
376        if len(tmp_o['tag']) == 0:
377            del tmp_o['tag']
378        dat = get_Array_from_dict(tmp_o)
379        my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)])
380        if corr_tag != 'None':
381            my_corr.tag = corr_tag
382
383        my_corr.prange = temp_prange
384        return my_corr
385
386    prog = json_dict.get('program', '')
387    version = json_dict.get('version', '')
388    who = json_dict.get('who', '')
389    date = json_dict.get('date', '')
390    host = json_dict.get('host', '')
391    if prog and verbose:
392        print('Data has been written using %s.' % (prog))
393    if version and verbose:
394        print('Format version %s' % (version))
395    if np.any([who, date, host] and verbose):
396        print('Written by %s on %s on host %s' % (who, date, host))
397    description = json_dict.get('description', '')
398    if description and verbose:
399        print()
400        print('Description: ', description)
401    obsdata = json_dict['obsdata']
402    ol = []
403    for io in obsdata:
404        if io['type'] == 'Obs':
405            ol.append(get_Obs_from_dict(io))
406        elif io['type'] == 'List':
407            ol.append(get_List_from_dict(io))
408        elif io['type'] == 'Array':
409            ol.append(get_Array_from_dict(io))
410        elif io['type'] == 'Corr':
411            ol.append(get_Corr_from_dict(io))
412        else:
413            raise Exception("Unkown datatype.")
414
415    if full_output:
416        retd = {}
417        retd['program'] = prog
418        retd['version'] = version
419        retd['who'] = who
420        retd['date'] = date
421        retd['host'] = host
422        retd['description'] = description
423        retd['obsdata'] = ol
424
425        return retd
426    else:
427        if len(obsdata) == 1:
428            ol = ol[0]
429
430        return ol
431
432
433def import_json_string(json_string, verbose=True, full_output=False):
434    """Reconstruct a list of Obs or structures containing Obs from a json string.
435
436    The following structures are supported: Obs, list, numpy.ndarray, Corr
437    If the list contains only one element, it is unpacked from the list.
438
439    Parameters
440    ----------
441    json_string : str
442        json string containing the data.
443    verbose : bool
444        Print additional information that was written to the file.
445    full_output : bool
446        If True, a dict containing auxiliary information and the data is returned.
447        If False, only the data is returned.
448    """
449
450    return _parse_json_dict(json.loads(json_string), verbose, full_output)
451
452
453def load_json(fname, verbose=True, gz=True, full_output=False):
454    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
455
456    The following structures are supported: Obs, list, numpy.ndarray, Corr
457    If the list contains only one element, it is unpacked from the list.
458
459    Parameters
460    ----------
461    fname : str
462        Filename of the input file.
463    verbose : bool
464        Print additional information that was written to the file.
465    gz : bool
466        If True, assumes that data is gzipped. If False, assumes JSON file.
467    full_output : bool
468        If True, a dict containing auxiliary information and the data is returned.
469        If False, only the data is returned.
470    """
471    if not fname.endswith('.json') and not fname.endswith('.gz'):
472        fname += '.json'
473    if gz:
474        if not fname.endswith('.gz'):
475            fname += '.gz'
476        with gzip.open(fname, 'r') as fin:
477            d = json.load(fin)
478    else:
479        if fname.endswith('.gz'):
480            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
481        with open(fname, 'r', encoding='utf-8') as fin:
482            d = json.loads(fin.read())
483
484    return _parse_json_dict(d, verbose, full_output)
485
486
487def _ol_from_dict(ind, reps='DICTOBS'):
488    """Convert a dictionary of Obs objects to a list and a dictionary that contains
489    placeholders instead of the Obs objects.
490
491    Parameters
492    ----------
493    ind : dict
494        Dict of JSON valid structures and objects that will be exported.
495        At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr.
496        All Obs inside a structure have to be defined on the same set of configurations.
497    reps : str
498        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
499    """
500
501    obstypes = (Obs, Corr, np.ndarray)
502
503    if not reps.isalnum():
504        raise Exception('Placeholder string has to be alphanumeric!')
505    ol = []
506    counter = 0
507
508    def dict_replace_obs(d):
509        nonlocal ol
510        nonlocal counter
511        x = {}
512        for k, v in d.items():
513            if isinstance(v, dict):
514                v = dict_replace_obs(v)
515            elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]):
516                v = obslist_replace_obs(v)
517            elif isinstance(v, list):
518                v = list_replace_obs(v)
519            elif isinstance(v, obstypes):
520                ol.append(v)
521                v = reps + '%d' % (counter)
522                counter += 1
523            elif isinstance(v, str):
524                if bool(re.match(r'%s[0-9]+' % (reps), v)):
525                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps))
526            x[k] = v
527        return x
528
529    def list_replace_obs(li):
530        nonlocal ol
531        nonlocal counter
532        x = []
533        for e in li:
534            if isinstance(e, list):
535                e = list_replace_obs(e)
536            elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]):
537                e = obslist_replace_obs(e)
538            elif isinstance(e, dict):
539                e = dict_replace_obs(e)
540            elif isinstance(e, obstypes):
541                ol.append(e)
542                e = reps + '%d' % (counter)
543                counter += 1
544            elif isinstance(e, str):
545                if bool(re.match(r'%s[0-9]+' % (reps), e)):
546                    raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps))
547            x.append(e)
548        return x
549
550    def obslist_replace_obs(li):
551        nonlocal ol
552        nonlocal counter
553        il = []
554        for e in li:
555            il.append(e)
556
557        ol.append(il)
558        x = reps + '%d' % (counter)
559        counter += 1
560        return x
561
562    nd = dict_replace_obs(ind)
563
564    return ol, nd
565
566
567def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
568    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
569
570    Parameters
571    ----------
572    od : dict
573        Dict of JSON valid structures and objects that will be exported.
574        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
575        All Obs inside a structure have to be defined on the same set of configurations.
576    fname : str
577        Filename of the output file.
578    description : str
579        Optional string that describes the contents of the json file.
580    indent : int
581        Specify the indentation level of the json file. None or 0 is permissible and
582        saves disk space.
583    reps : str
584        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
585    gz : bool
586        If True, the output is a gzipped json. If False, the output is a json file.
587    """
588
589    if not isinstance(od, dict):
590        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
591
592    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
593                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
594                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
595                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
596
597    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
598    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
599
600    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
601
602
603def _od_from_list_and_dict(ol, ind, reps='DICTOBS'):
604    """Parse a list of Obs or structures containing Obs and an accompanying
605    dict, where the structures have been replaced by placeholders to a
606    dict that contains the structures.
607
608    The following structures are supported: Obs, list, numpy.ndarray, Corr
609
610    Parameters
611    ----------
612    ol : list
613        List of objects -
614        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
615        All Obs inside a structure have to be defined on the same set of configurations.
616    ind : dict
617        Dict that defines the structure of the resulting dict and contains placeholders
618    reps : str
619        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
620    """
621    if not reps.isalnum():
622        raise Exception('Placeholder string has to be alphanumeric!')
623
624    counter = 0
625
626    def dict_replace_string(d):
627        nonlocal counter
628        nonlocal ol
629        x = {}
630        for k, v in d.items():
631            if isinstance(v, dict):
632                v = dict_replace_string(v)
633            elif isinstance(v, list):
634                v = list_replace_string(v)
635            elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)):
636                index = int(v[len(reps):])
637                v = ol[index]
638                counter += 1
639            x[k] = v
640        return x
641
642    def list_replace_string(li):
643        nonlocal counter
644        nonlocal ol
645        x = []
646        for e in li:
647            if isinstance(e, list):
648                e = list_replace_string(e)
649            elif isinstance(e, dict):
650                e = dict_replace_string(e)
651            elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)):
652                index = int(e[len(reps):])
653                e = ol[index]
654                counter += 1
655            x.append(e)
656        return x
657
658    nd = dict_replace_string(ind)
659
660    if counter == 0:
661        raise Exception('No placeholder has been replaced! Check if reps is set correctly.')
662
663    return nd
664
665
666def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
667    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
668
669    The following structures are supported: Obs, list, numpy.ndarray, Corr
670
671    Parameters
672    ----------
673    fname : str
674        Filename of the input file.
675    verbose : bool
676        Print additional information that was written to the file.
677    gz : bool
678        If True, assumes that data is gzipped. If False, assumes JSON file.
679    full_output : bool
680        If True, a dict containing auxiliary information and the data is returned.
681        If False, only the data is returned.
682    reps : str
683        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
684    """
685    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
686    description = indata['description']['description']
687    indict = indata['description']['OBSDICT']
688    ol = indata['obsdata']
689    od = _od_from_list_and_dict(ol, indict, reps=reps)
690
691    if full_output:
692        indata['description'] = description
693        indata['obsdata'] = od
694        return indata
695    else:
696        return od