Merge branch 'fjosw:develop' into feature/input_2.0

pyerrors/input/__init__.py

@@ -1,5 +1,6 @@
 from . import bdio
 from . import hadrons
-from . import sfcf
-from . import openQCD
+from . import json
 from . import misc
+from . import openQCD
+from . import sfcf

pyerrors/input/bdio.py

@@ -226,7 +226,7 @@ def write_ADerrors(obs_list, file_path, bdio_path='./libbdio.so', **kwargs):
         for key in keys:
             try:  # Try to convert key to integer
                 ids.append(int(key))
-            except:  # If not possible construct a hash
+            except Exception:  # If not possible construct a hash
                 ids.append(int(hashlib.sha256(key.encode('utf-8')).hexdigest(), 16) % 10 ** 8)
         print('ids', ids)
         nt = []

pyerrors/input/hadrons.py

@@ -1,25 +1,15 @@
-#!/usr/bin/env python
-# coding: utf-8
-
 import os
 import h5py
 import numpy as np
 from ..obs import Obs, CObs
 from ..correlators import Corr
-from ..npr import Npr_matrix
 
 
-def _get_files(path, filestem):
-    ls = []
-    for (dirpath, dirnames, filenames) in os.walk(path):
-        ls.extend(filenames)
-        break
+def _get_files(path, filestem, idl):
+    ls = os.listdir(path)
 
     # Clean up file list
-    files = []
-    for line in ls:
-        if line.startswith(filestem):
-            files.append(line)
+    files = list(filter(lambda x: x.startswith(filestem), ls))
 
     if not files:
         raise Exception('No files starting with', filestem, 'in folder', path)
@@ -30,18 +20,31 @@ def _get_files(path, filestem):
     # Sort according to configuration number
     files.sort(key=get_cnfg_number)
 
-    # Check that configurations are evenly spaced
     cnfg_numbers = []
+    filtered_files = []
     for line in files:
-        cnfg_numbers.append(get_cnfg_number(line))
+        no = get_cnfg_number(line)
+        if idl:
+            if no in list(idl):
+                filtered_files.append(line)
+                cnfg_numbers.append(no)
+        else:
+            filtered_files.append(line)
+            cnfg_numbers.append(no)
 
-    if not all(np.diff(cnfg_numbers) == np.diff(cnfg_numbers)[0]):
+    # Check that configurations are evenly spaced
+    dc = np.unique(np.diff(cnfg_numbers))
+    if np.any(dc < 0):
+        raise Exception("Unsorted files")
+    if len(dc) == 1:
+        idx = range(cnfg_numbers[0], cnfg_numbers[-1] + dc[0], dc[0])
+    else:
         raise Exception('Configurations are not evenly spaced.')
 
-    return files, cnfg_numbers
+    return filtered_files, idx
 
 
-def read_meson_hd5(path, filestem, ens_id, meson='meson_0', tree='meson'):
+def read_meson_hd5(path, filestem, ens_id, meson='meson_0', tree='meson', idl=None):
     """Read hadrons meson hdf5 file and extract the meson labeled 'meson'
 
     Parameters
@@ -59,9 +62,11 @@ def read_meson_hd5(path, filestem, ens_id, meson='meson_0', tree='meson'):
         Label of the upmost directory in the hdf5 file, default 'meson'
         for outputs of the Meson module. Can be altered to read input
         from other modules with similar structures.
+    idl : range
+        If specified only configurations in the given range are read in.
     """
 
-    files, cnfg_numbers = _get_files(path, filestem)
+    files, idx = _get_files(path, filestem, idl)
 
     corr_data = []
     infos = []
@@ -78,27 +83,69 @@ def read_meson_hd5(path, filestem, ens_id, meson='meson_0', tree='meson'):
 
     l_obs = []
     for c in corr_data.T:
-        l_obs.append(Obs([c], [ens_id], idl=[cnfg_numbers]))
+        l_obs.append(Obs([c], [ens_id], idl=[idx]))
 
     corr = Corr(l_obs)
     corr.tag = r", ".join(infos)
     return corr
 
 
-def read_ExternalLeg_hd5(path, filestem, ens_id, order='F'):
+class Npr_matrix(np.ndarray):
+
+    def __new__(cls, input_array, mom_in=None, mom_out=None):
+        obj = np.asarray(input_array).view(cls)
+        obj.mom_in = mom_in
+        obj.mom_out = mom_out
+        return obj
+
+    @property
+    def g5H(self):
+        """Gamma_5 hermitean conjugate
+
+        Uses the fact that the propagator is gamma5 hermitean, so just the
+        in and out momenta of the propagator are exchanged.
+        """
+        return Npr_matrix(self,
+                          mom_in=self.mom_out,
+                          mom_out=self.mom_in)
+
+    def _propagate_mom(self, other, name):
+        s_mom = getattr(self, name, None)
+        o_mom = getattr(other, name, None)
+        if s_mom is not None and o_mom is not None:
+            if not np.allclose(s_mom, o_mom):
+                raise Exception(name + ' does not match.')
+        return o_mom if o_mom is not None else s_mom
+
+    def __matmul__(self, other):
+        return self.__new__(Npr_matrix,
+                            super().__matmul__(other),
+                            self._propagate_mom(other, 'mom_in'),
+                            self._propagate_mom(other, 'mom_out'))
+
+    def __array_finalize__(self, obj):
+        if obj is None:
+            return
+        self.mom_in = getattr(obj, 'mom_in', None)
+        self.mom_out = getattr(obj, 'mom_out', None)
+
+
+def read_ExternalLeg_hd5(path, filestem, ens_id, idl=None):
     """Read hadrons ExternalLeg hdf5 file and output an array of CObs
 
     Parameters
-    -----------------
-    path -- path to the files to read
-    filestem -- namestem of the files to read
-    ens_id -- name of the ensemble, required for internal bookkeeping
-    order -- order in which the array is to be reshaped,
-             'F' for the first index changing fastest (9 4x4 matrices) default.
-             'C' for the last index changing fastest (16 3x3 matrices),
+    ----------
+    path : str
+        path to the files to read
+    filestem : str
+        namestem of the files to read
+    ens_id : str
+        name of the ensemble, required for internal bookkeeping
+    idl : range
+        If specified only configurations in the given range are read in.
     """
 
-    files, cnfg_numbers = _get_files(path, filestem)
+    files, idx = _get_files(path, filestem, idl)
 
     mom = None
 
@@ -107,9 +154,7 @@ def read_ExternalLeg_hd5(path, filestem, ens_id, order='F'):
         file = h5py.File(path + '/' + hd5_file, "r")
         raw_data = file['ExternalLeg/corr'][0][0].view('complex')
         corr_data.append(raw_data)
-        if mom is not None:
-            assert np.allclose(mom, np.array(str(file['ExternalLeg/info'].attrs['pIn'])[3:-2].strip().split(' '), dtype=int))
-        else:
+        if mom is None:
             mom = np.array(str(file['ExternalLeg/info'].attrs['pIn'])[3:-2].strip().split(' '), dtype=int)
         file.close()
     corr_data = np.array(corr_data)
@@ -118,28 +163,29 @@ def read_ExternalLeg_hd5(path, filestem, ens_id, order='F'):
 
     matrix = np.empty((rolled_array.shape[:-1]), dtype=object)
     for si, sj, ci, cj in np.ndindex(rolled_array.shape[:-1]):
-        real = Obs([rolled_array[si, sj, ci, cj].real], [ens_id], idl=[cnfg_numbers])
-        imag = Obs([rolled_array[si, sj, ci, cj].imag], [ens_id], idl=[cnfg_numbers])
+        real = Obs([rolled_array[si, sj, ci, cj].real], [ens_id], idl=[idx])
+        imag = Obs([rolled_array[si, sj, ci, cj].imag], [ens_id], idl=[idx])
         matrix[si, sj, ci, cj] = CObs(real, imag)
-        matrix[si, sj, ci, cj].gamma_method()
 
-    return Npr_matrix(matrix.swapaxes(1, 2).reshape((12, 12), order=order), mom_in=mom)
+    return Npr_matrix(matrix, mom_in=mom)
 
 
-def read_Bilinear_hd5(path, filestem, ens_id, order='F'):
+def read_Bilinear_hd5(path, filestem, ens_id, idl=None):
     """Read hadrons Bilinear hdf5 file and output an array of CObs
 
     Parameters
-    -----------------
-    path -- path to the files to read
-    filestem -- namestem of the files to read
-    ens_id -- name of the ensemble, required for internal bookkeeping
-    order -- order in which the array is to be reshaped,
-             'F' for the first index changing fastest (9 4x4 matrices) default.
-             'C' for the last index changing fastest (16 3x3 matrices),
+    ----------
+    path : str
+        path to the files to read
+    filestem : str
+        namestem of the files to read
+    ens_id : str
+        name of the ensemble, required for internal bookkeeping
+    idl : range
+        If specified only configurations in the given range are read in.
     """
 
-    files, cnfg_numbers = _get_files(path, filestem)
+    files, idx = _get_files(path, filestem, idl)
 
     mom_in = None
     mom_out = None
@@ -153,13 +199,9 @@ def read_Bilinear_hd5(path, filestem, ens_id, order='F'):
                 corr_data[name] = []
             raw_data = file['Bilinear/Bilinear_' + str(i) + '/corr'][0][0].view('complex')
             corr_data[name].append(raw_data)
-            if mom_in is not None:
-                assert np.allclose(mom_in, np.array(str(file['Bilinear/Bilinear_' + str(i) + '/info'].attrs['pIn'])[3:-2].strip().split(' '), dtype=int))
-            else:
+            if mom_in is None:
                 mom_in = np.array(str(file['Bilinear/Bilinear_' + str(i) + '/info'].attrs['pIn'])[3:-2].strip().split(' '), dtype=int)
-            if mom_out is not None:
-                assert np.allclose(mom_out, np.array(str(file['Bilinear/Bilinear_' + str(i) + '/info'].attrs['pOut'])[3:-2].strip().split(' '), dtype=int))
-            else:
+            if mom_out is None:
                 mom_out = np.array(str(file['Bilinear/Bilinear_' + str(i) + '/info'].attrs['pOut'])[3:-2].strip().split(' '), dtype=int)
 
         file.close()
@@ -173,11 +215,117 @@ def read_Bilinear_hd5(path, filestem, ens_id, order='F'):
 
         matrix = np.empty((rolled_array.shape[:-1]), dtype=object)
         for si, sj, ci, cj in np.ndindex(rolled_array.shape[:-1]):
-            real = Obs([rolled_array[si, sj, ci, cj].real], [ens_id], idl=[cnfg_numbers])
-            imag = Obs([rolled_array[si, sj, ci, cj].imag], [ens_id], idl=[cnfg_numbers])
+            real = Obs([rolled_array[si, sj, ci, cj].real], [ens_id], idl=[idx])
+            imag = Obs([rolled_array[si, sj, ci, cj].imag], [ens_id], idl=[idx])
             matrix[si, sj, ci, cj] = CObs(real, imag)
-            matrix[si, sj, ci, cj].gamma_method()
 
-        result_dict[key] = Npr_matrix(matrix.swapaxes(1, 2).reshape((12, 12), order=order), mom_in=mom_in, mom_out=mom_out)
+        result_dict[key] = Npr_matrix(matrix, mom_in=mom_in, mom_out=mom_out)
 
     return result_dict
+
+
+def read_Fourquark_hd5(path, filestem, ens_id, idl=None, vertices=["VA", "AV"]):
+    """Read hadrons FourquarkFullyConnected hdf5 file and output an array of CObs
+
+    Parameters
+    ----------
+    path : str
+        path to the files to read
+    filestem : str
+        namestem of the files to read
+    ens_id : str
+        name of the ensemble, required for internal bookkeeping
+    idl : range
+        If specified only configurations in the given range are read in.
+    vertices : list
+        Vertex functions to be extracted.
+    """
+
+    files, idx = _get_files(path, filestem, idl)
+
+    mom_in = None
+    mom_out = None
+
+    vertex_names = []
+    for vertex in vertices:
+        vertex_names += _get_lorentz_names(vertex)
+
+    corr_data = {}
+
+    tree = 'FourQuarkFullyConnected/FourQuarkFullyConnected_'
+
+    for hd5_file in files:
+        file = h5py.File(path + '/' + hd5_file, "r")
+
+        for i in range(32):
+            name = (file[tree + str(i) + '/info'].attrs['gammaA'][0].decode('UTF-8'), file[tree + str(i) + '/info'].attrs['gammaB'][0].decode('UTF-8'))
+            if name in vertex_names:
+                if name not in corr_data:
+                    corr_data[name] = []
+                raw_data = file[tree + str(i) + '/corr'][0][0].view('complex')
+                corr_data[name].append(raw_data)
+                if mom_in is None:
+                    mom_in = np.array(str(file[tree + str(i) + '/info'].attrs['pIn'])[3:-2].strip().split(' '), dtype=int)
+                if mom_out is None:
+                    mom_out = np.array(str(file[tree + str(i) + '/info'].attrs['pOut'])[3:-2].strip().split(' '), dtype=int)
+
+        file.close()
+
+    intermediate_dict = {}
+
+    for vertex in vertices:
+        lorentz_names = _get_lorentz_names(vertex)
+        for v_name in lorentz_names:
+            if vertex not in intermediate_dict:
+                intermediate_dict[vertex] = np.array(corr_data[v_name])
+            else:
+                intermediate_dict[vertex] += np.array(corr_data[v_name])
+
+    result_dict = {}
+
+    for key, data in intermediate_dict.items():
+
+        rolled_array = np.moveaxis(data, 0, 8)
+
+        matrix = np.empty((rolled_array.shape[:-1]), dtype=object)
+        for index in np.ndindex(rolled_array.shape[:-1]):
+            real = Obs([rolled_array[index].real], [ens_id], idl=[idx])
+            imag = Obs([rolled_array[index].imag], [ens_id], idl=[idx])
+            matrix[index] = CObs(real, imag)
+
+        result_dict[key] = Npr_matrix(matrix, mom_in=mom_in, mom_out=mom_out)
+
+    return result_dict
+
+
+def _get_lorentz_names(name):
+    assert len(name) == 2
+
+    res = []
+
+    if not set(name) <= set(['S', 'P', 'V', 'A', 'T']):
+        raise Exception("Name can only contain 'S', 'P', 'V', 'A' or 'T'")
+
+    if 'S' in name or 'P' in name:
+        if not set(name) <= set(['S', 'P']):
+            raise Exception("'" + name + "' is not a Lorentz scalar")
+
+        g_names = {'S': 'Identity',
+                   'P': 'Gamma5'}
+
+        res.append((g_names[name[0]], g_names[name[1]]))
+
+    elif 'T' in name:
+        if not set(name) <= set(['T']):
+            raise Exception("'" + name + "' is not a Lorentz scalar")
+        raise Exception("Tensor operators not yet implemented.")
+    else:
+        if not set(name) <= set(['V', 'A']):
+            raise Exception("'" + name + "' is not a Lorentz scalar")
+        lorentz_index = ['X', 'Y', 'Z', 'T']
+
+        for ind in lorentz_index:
+            res.append(('Gamma' + ind + (name[0] == 'A') * 'Gamma5',
+                        'Gamma' + ind + (name[1] == 'A') * 'Gamma5'))
+
+    return res

pyerrors/input/json.py

@@ -0,0 +1,453 @@
+import json
+import gzip
+import numpy as np
+import getpass
+import socket
+import datetime
+import platform
+import warnings
+from ..obs import Obs
+from ..covobs import Covobs
+from .. import version as pyerrorsversion
+
+
+def create_json_string(ol, description='', indent=1):
+    """Generate the string for the export of a list of Obs or structures containing Obs
+    to a .json(.gz) file
+
+    Parameters
+    ----------
+    ol : list
+        List of objects that will be exported. At the moments, these objects can be
+        either of: Obs, list, numpy.ndarray.
+        All Obs inside a structure have to be defined on the same set of configurations.
+    description : str
+        Optional string that describes the contents of the json file.
+    indent : int
+        Specify the indentation level of the json file. None or 0 is permissible and
+        saves disk space.
+    """
+
+    def _default(self, obj):
+        return str(obj)
+    my_encoder = json.JSONEncoder
+    _default.default = json.JSONEncoder().default
+    my_encoder.default = _default
+
+    class Deltalist:
+        def __init__(self, li):
+            self.cnfg = li[0]
+            self.deltas = li[1:]
+
+        def __repr__(self):
+            s = '[%d' % (self.cnfg)
+            for d in self.deltas:
+                s += ', %1.15e' % (d)
+            s += ']'
+            return s
+
+        def __str__(self):
+            return self.__repr__()
+
+    class Floatlist:
+        def __init__(self, li):
+            self.li = list(li)
+
+        def __repr__(self):
+            s = '['
+            for i in range(len(self.li)):
+                if i > 0:
+                    s += ', '
+                s += '%1.15e' % (self.li[i])
+            s += ']'
+            return s
+
+        def __str__(self):
+            return self.__repr__()
+
+    def _gen_data_d_from_list(ol):
+        dl = []
+        for name in ol[0].mc_names:
+            ed = {}
+            ed['id'] = name
+            ed['replica'] = []
+            for r_name in ol[0].e_content[name]:
+                rd = {}
+                rd['name'] = r_name
+                if ol[0].is_merged.get(r_name, False):
+                    rd['is_merged'] = True
+                rd['deltas'] = []
+                for i in range(len(ol[0].idl[r_name])):
+                    rd['deltas'].append([ol[0].idl[r_name][i]])
+                    for o in ol:
+                        rd['deltas'][-1].append(o.deltas[r_name][i])
+                    rd['deltas'][-1] = Deltalist(rd['deltas'][-1])
+                ed['replica'].append(rd)
+            dl.append(ed)
+        return dl
+
+    def _gen_cdata_d_from_list(ol):
+        dl = []
+        for name in ol[0].cov_names:
+            ed = {}
+            ed['id'] = name
+            ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',')
+            ed['cov'] = Floatlist(np.ravel(ol[0].covobs[name].cov))
+            ncov = ol[0].covobs[name].cov.shape[0]
+            ed['grad'] = []
+            for i in range(ncov):
+                ed['grad'].append([])
+                for o in ol:
+                    ed['grad'][-1].append(o.covobs[name].grad[i][0])
+                ed['grad'][-1] = Floatlist(ed['grad'][-1])
+            dl.append(ed)
+        return dl
+
+    def _assert_equal_properties(ol, otype=Obs):
+        for o in ol:
+            if not isinstance(o, otype):
+                raise Exception("Wrong data type in list.")
+        for o in ol[1:]:
+            if not ol[0].is_merged == o.is_merged:
+                raise Exception("All Obs in list have to be defined on the same set of configs.")
+            if not ol[0].reweighted == o.reweighted:
+                raise Exception("All Obs in list have to have the same property 'reweighted'.")
+            if not ol[0].e_content == o.e_content:
+                raise Exception("All Obs in list have to be defined on the same set of configs.")
+            if not ol[0].idl == o.idl:
+                raise Exception("All Obs in list have to be defined on the same set of configurations.")
+
+    def write_Obs_to_dict(o):
+        d = {}
+        d['type'] = 'Obs'
+        d['layout'] = '1'
+        if o.tag:
+            d['tag'] = [o.tag]
+        if o.reweighted:
+            d['reweighted'] = o.reweighted
+        d['value'] = [o.value]
+        data = _gen_data_d_from_list([o])
+        if len(data) > 0:
+            d['data'] = data
+        cdata = _gen_cdata_d_from_list([o])
+        if len(cdata) > 0:
+            d['cdata'] = cdata
+        return d
+
+    def write_List_to_dict(ol):
+        _assert_equal_properties(ol)
+        d = {}
+        d['type'] = 'List'
+        d['layout'] = '%d' % len(ol)
+        taglist = [o.tag for o in ol]
+        if np.any([tag is not None for tag in taglist]):
+            d['tag'] = taglist
+        if ol[0].reweighted:
+            d['reweighted'] = ol[0].reweighted
+        d['value'] = [o.value for o in ol]
+        data = _gen_data_d_from_list(ol)
+        if len(data) > 0:
+            d['data'] = data
+        cdata = _gen_cdata_d_from_list(ol)
+        if len(cdata) > 0:
+            d['cdata'] = cdata
+        return d
+
+    def write_Array_to_dict(oa):
+        ol = np.ravel(oa)
+        _assert_equal_properties(ol)
+        d = {}
+        d['type'] = 'Array'
+        d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',')
+        taglist = [o.tag for o in ol]
+        if np.any([tag is not None for tag in taglist]):
+            d['tag'] = taglist
+        if ol[0].reweighted:
+            d['reweighted'] = ol[0].reweighted
+        d['value'] = [o.value for o in ol]
+        data = _gen_data_d_from_list(ol)
+        if len(data) > 0:
+            d['data'] = data
+        cdata = _gen_cdata_d_from_list(ol)
+        if len(cdata) > 0:
+            d['cdata'] = cdata
+        return d
+
+    if not isinstance(ol, list):
+        ol = [ol]
+
+    d = {}
+    d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__)
+    d['version'] = '0.1'
+    d['who'] = getpass.getuser()
+    d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
+    d['host'] = socket.gethostname() + ', ' + platform.platform()
+
+    if description:
+        d['description'] = description
+    d['obsdata'] = []
+    for io in ol:
+        if isinstance(io, Obs):
+            d['obsdata'].append(write_Obs_to_dict(io))
+        elif isinstance(io, list):
+            d['obsdata'].append(write_List_to_dict(io))
+        elif isinstance(io, np.ndarray):
+            d['obsdata'].append(write_Array_to_dict(io))
+
+    jsonstring = json.dumps(d, indent=indent, cls=my_encoder, ensure_ascii=False)
+
+    def remove_quotationmarks(s):
+        """Workaround for un-quoting of delta lists, adds 5% of work
+           but is save, compared to a simple replace that could destroy the structure
+        """
+        deltas = False
+        split = s.split('\n')
+        for i in range(len(split)):
+            if '"deltas":' in split[i] or '"cov":' in split[i] or '"grad":' in split[i]:
+                deltas = True
+            if deltas:
+                split[i] = split[i].replace('"[', '[').replace(']"', ']')
+                if split[i][-1] == ']':
+                    deltas = False
+        return '\n'.join(split)
+
+    jsonstring = remove_quotationmarks(jsonstring)
+    return jsonstring
+
+
+def dump_to_json(ol, fname, description='', indent=1, gz=True):
+    """Export a list of Obs or structures containing Obs to a .json(.gz) file
+
+    Parameters
+    ----------
+    ol : list
+        List of objects that will be exported. At the moments, these objects can be
+        either of: Obs, list, numpy.ndarray.
+        All Obs inside a structure have to be defined on the same set of configurations.
+    fname : str
+        Filename of the output file.
+    description : str
+        Optional string that describes the contents of the json file.
+    indent : int
+        Specify the indentation level of the json file. None or 0 is permissible and
+        saves disk space.
+    gz : bool
+        If True, the output is a gzipped json. If False, the output is a json file.
+    """
+
+    jsonstring = create_json_string(ol, description, indent)
+
+    if not fname.endswith('.json') and not fname.endswith('.gz'):
+        fname += '.json'
+
+    if gz:
+        if not fname.endswith('.gz'):
+            fname += '.gz'
+
+        fp = gzip.open(fname, 'wb')
+        fp.write(jsonstring.encode('utf-8'))
+    else:
+        fp = open(fname, 'w', encoding='utf-8')
+        fp.write(jsonstring)
+    fp.close()
+
+
+def import_json_string(json_string, verbose=True, full_output=False):
+    """Reconstruct a list of Obs or structures containing Obs from a json string.
+
+    The following structures are supported: Obs, list, numpy.ndarray
+    If the list contains only one element, it is unpacked from the list.
+
+    Parameters
+    ----------
+    json_string : str
+        json string containing the data.
+    verbose : bool
+        Print additional information that was written to the file.
+    full_output : bool
+        If True, a dict containing auxiliary information and the data is returned.
+        If False, only the data is returned.
+    """
+
+    def _gen_obsd_from_datad(d):
+        retd = {}
+        if d:
+            retd['names'] = []
+            retd['idl'] = []
+            retd['deltas'] = []
+            retd['is_merged'] = {}
+            for ens in d:
+                for rep in ens['replica']:
+                    retd['names'].append(rep['name'])
+                    retd['idl'].append([di[0] for di in rep['deltas']])
+                    retd['deltas'].append(np.array([di[1:] for di in rep['deltas']]))
+                    retd['is_merged'][rep['name']] = rep.get('is_merged', False)
+        return retd
+
+    def _gen_covobsd_from_cdatad(d):
+        retd = {}
+        for ens in d:
+            retl = []
+            name = ens['id']
+            layouts = ens.get('layout', '1').strip()
+            layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
+            cov = np.reshape(ens['cov'], layout)
+            grad = ens['grad']
+            nobs = len(grad[0])
+            for i in range(nobs):
+                retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]})
+            retd[name] = retl
+        return retd
+
+    def get_Obs_from_dict(o):
+        layouts = o.get('layout', '1').strip()
+        if layouts != '1':
+            raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning)
+
+        values = o['value']
+        od = _gen_obsd_from_datad(o.get('data', {}))
+        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
+
+        if od:
+            ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl'])
+            ret.is_merged = od['is_merged']
+        else:
+            ret = Obs([], [])
+            ret._value = values[0]
+        for name in cd:
+            co = cd[name][0]
+            ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
+            ret.names.append(co['name'])
+
+        ret.reweighted = o.get('reweighted', False)
+        ret.tag = o.get('tag', [None])[0]
+        return ret
+
+    def get_List_from_dict(o):
+        layouts = o.get('layout', '1').strip()
+        layout = int(layouts)
+        values = o['value']
+        od = _gen_obsd_from_datad(o.get('data', {}))
+        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
+
+        ret = []
+        taglist = o.get('tag', layout * [None])
+        for i in range(layout):
+            if od:
+                ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl']))
+                ret[-1].is_merged = od['is_merged']
+            else:
+                ret.append(Obs([], []))
+                ret[-1]._value = values[i]
+                print('Created Obs with means= ', values[i])
+            for name in cd:
+                co = cd[name][i]
+                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
+                ret[-1].names.append(co['name'])
+
+            ret[-1].reweighted = o.get('reweighted', False)
+            ret[-1].tag = taglist[i]
+        return ret
+
+    def get_Array_from_dict(o):
+        layouts = o.get('layout', '1').strip()
+        layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0]
+        N = np.prod(layout)
+        values = o['value']
+        od = _gen_obsd_from_datad(o.get('data', {}))
+        cd = _gen_covobsd_from_cdatad(o.get('cdata', {}))
+
+        ret = []
+        taglist = o.get('tag', N * [None])
+        for i in range(N):
+            if od:
+                ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl']))
+                ret[-1].is_merged = od['is_merged']
+            else:
+                ret.append(Obs([], []))
+                ret[-1]._value = values[i]
+            for name in cd:
+                co = cd[name][i]
+                ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad'])
+                ret[-1].names.append(co['name'])
+            ret[-1].reweighted = o.get('reweighted', False)
+            ret[-1].tag = taglist[i]
+        return np.reshape(ret, layout)
+
+    json_dict = json.loads(json_string)
+
+    prog = json_dict.get('program', '')
+    version = json_dict.get('version', '')
+    who = json_dict.get('who', '')
+    date = json_dict.get('date', '')
+    host = json_dict.get('host', '')
+    if prog and verbose:
+        print('Data has been written using %s.' % (prog))
+    if version and verbose:
+        print('Format version %s' % (version))
+    if np.any([who, date, host] and verbose):
+        print('Written by %s on %s on host %s' % (who, date, host))
+    description = json_dict.get('description', '')
+    if description and verbose:
+        print()
+        print('Description: ', description)
+    obsdata = json_dict['obsdata']
+    ol = []
+    for io in obsdata:
+        if io['type'] == 'Obs':
+            ol.append(get_Obs_from_dict(io))
+        elif io['type'] == 'List':
+            ol.append(get_List_from_dict(io))
+        elif io['type'] == 'Array':
+            ol.append(get_Array_from_dict(io))
+
+    if full_output:
+        retd = {}
+        retd['program'] = prog
+        retd['version'] = version
+        retd['who'] = who
+        retd['date'] = date
+        retd['host'] = host
+        retd['description'] = description
+        retd['obsdata'] = ol
+
+        return retd
+    else:
+        if len(obsdata) == 1:
+            ol = ol[0]
+
+        return ol
+
+
+def load_json(fname, verbose=True, gz=True, full_output=False):
+    """Import a list of Obs or structures containing Obs from a .json.gz file.
+
+    The following structures are supported: Obs, list, numpy.ndarray
+    If the list contains only one element, it is unpacked from the list.
+
+    Parameters
+    ----------
+    fname : str
+        Filename of the input file.
+    verbose : bool
+        Print additional information that was written to the file.
+    gz : bool
+        If True, assumes that data is gzipped. If False, assumes JSON file.
+    full_output : bool
+        If True, a dict containing auxiliary information and the data is returned.
+        If False, only the data is returned.
+    """
+    if not fname.endswith('.json') and not fname.endswith('.gz'):
+        fname += '.json'
+    if gz:
+        if not fname.endswith('.gz'):
+            fname += '.gz'
+        with gzip.open(fname, 'r') as fin:
+            d = fin.read().decode('utf-8')
+    else:
+        if fname.endswith('.gz'):
+            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
+        with open(fname, 'r', encoding='utf-8') as fin:
+            d = fin.read()
+
+    return import_json_string(d, verbose, full_output)