Merge pull request #73 from s-kuberski/feature/json_io

Feature/json io
This commit is contained in:
Fabian Joswig 2022-02-16 11:56:44 +00:00 committed by GitHub
commit a8f029629a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23

View file

@ -6,6 +6,7 @@ import datetime
import platform import platform
import warnings import warnings
import re import re
import gc
import numpy as np import numpy as np
from ..obs import Obs from ..obs import Obs
from ..covobs import Covobs from ..covobs import Covobs
@ -38,6 +39,8 @@ def create_json_string(ol, description='', indent=1):
my_encoder.default = _default my_encoder.default = _default
class Deltalist: class Deltalist:
__slots__ = ['cnfg', 'deltas']
def __init__(self, li): def __init__(self, li):
self.cnfg = li[0] self.cnfg = li[0]
self.deltas = li[1:] self.deltas = li[1:]
@ -53,6 +56,8 @@ def create_json_string(ol, description='', indent=1):
return self.__repr__() return self.__repr__()
class Floatlist: class Floatlist:
__slots__ = ['li']
def __init__(self, li): def __init__(self, li):
self.li = list(li) self.li = list(li)
@ -222,14 +227,18 @@ def create_json_string(ol, description='', indent=1):
else: else:
raise Exception("Unkown datatype.") raise Exception("Unkown datatype.")
jsonstring = json.dumps(d, indent=indent, cls=my_encoder, ensure_ascii=False) jsonstring = ''
for chunk in my_encoder(indent=indent, ensure_ascii=False).iterencode(d):
jsonstring += chunk
def remove_quotationmarks(s): del d
gc.collect()
def remove_quotationmarks_split(split):
"""Workaround for un-quoting of delta lists, adds 5% of work """Workaround for un-quoting of delta lists, adds 5% of work
but is save, compared to a simple replace that could destroy the structure but is save, compared to a simple replace that could destroy the structure
""" """
deltas = False deltas = False
split = s.split('\n')
for i in range(len(split)): for i in range(len(split)):
if '"deltas":' in split[i] or '"cov":' in split[i] or '"grad":' in split[i]: if '"deltas":' in split[i] or '"cov":' in split[i] or '"grad":' in split[i]:
deltas = True deltas = True
@ -239,7 +248,8 @@ def create_json_string(ol, description='', indent=1):
deltas = False deltas = False
return '\n'.join(split) return '\n'.join(split)
jsonstring = remove_quotationmarks(jsonstring) jsonstring = jsonstring.split('\n')
jsonstring = remove_quotationmarks_split(jsonstring)
jsonstring = jsonstring.replace('nan', 'NaN') jsonstring = jsonstring.replace('nan', 'NaN')
return jsonstring return jsonstring
@ -281,8 +291,9 @@ def dump_to_json(ol, fname, description='', indent=1, gz=True):
fp.close() fp.close()
def import_json_string(json_string, verbose=True, full_output=False): def _parse_json_dict(json_dict, verbose=True, full_output=False):
"""Reconstruct a list of Obs or structures containing Obs from a json string. """Reconstruct a list of Obs or structures containing Obs from a dict that
was built out of a json string.
The following structures are supported: Obs, list, numpy.ndarray, Corr The following structures are supported: Obs, list, numpy.ndarray, Corr
If the list contains only one element, it is unpacked from the list. If the list contains only one element, it is unpacked from the list.
@ -436,8 +447,6 @@ def import_json_string(json_string, verbose=True, full_output=False):
my_corr.prange = temp_prange my_corr.prange = temp_prange
return my_corr return my_corr
json_dict = json.loads(json_string)
prog = json_dict.get('program', '') prog = json_dict.get('program', '')
version = json_dict.get('version', '') version = json_dict.get('version', '')
who = json_dict.get('who', '') who = json_dict.get('who', '')
@ -485,6 +494,26 @@ def import_json_string(json_string, verbose=True, full_output=False):
return ol return ol
def import_json_string(json_string, verbose=True, full_output=False):
"""Reconstruct a list of Obs or structures containing Obs from a json string.
The following structures are supported: Obs, list, numpy.ndarray, Corr
If the list contains only one element, it is unpacked from the list.
Parameters
----------
json_string : str
json string containing the data.
verbose : bool
Print additional information that was written to the file.
full_output : bool
If True, a dict containing auxiliary information and the data is returned.
If False, only the data is returned.
"""
return _parse_json_dict(json.loads(json_string), verbose, full_output)
def load_json(fname, verbose=True, gz=True, full_output=False): def load_json(fname, verbose=True, gz=True, full_output=False):
"""Import a list of Obs or structures containing Obs from a .json(.gz) file. """Import a list of Obs or structures containing Obs from a .json(.gz) file.
@ -509,14 +538,14 @@ def load_json(fname, verbose=True, gz=True, full_output=False):
if not fname.endswith('.gz'): if not fname.endswith('.gz'):
fname += '.gz' fname += '.gz'
with gzip.open(fname, 'r') as fin: with gzip.open(fname, 'r') as fin:
d = fin.read().decode('utf-8') d = json.load(fin)
else: else:
if fname.endswith('.gz'): if fname.endswith('.gz'):
warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
with open(fname, 'r', encoding='utf-8') as fin: with open(fname, 'r', encoding='utf-8') as fin:
d = fin.read() d = json.loads(fin.read())
return import_json_string(d, verbose, full_output) return _parse_json_dict(d, verbose, full_output)
def _ol_from_dict(ind, reps='DICTOBS'): def _ol_from_dict(ind, reps='DICTOBS'):