lattice/pyerrors
1
0
Fork 0
mirror of https://github.com/fjosw/pyerrors.git synced 2025-03-15 06:40:24 +01:00
Code Issues Projects Releases Packages Wiki Activity

Reduce RAM requirements when reading a JSON file

Browse source
This commit is contained in:
Simon Kuberski 2022-02-16 11:26:06 +01:00
parent bd20057f52
commit 005f54e0d1
1 changed files with 26 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
pyerrors/input/json.py
View file

@ -291,8 +291,9 @@ def dump_to_json(ol, fname, description='', indent=1, gz=True):
fp.close()
def import_json_string(json_string, verbose=True, full_output=False):
"""Reconstruct a list of Obs or structures containing Obs from a json string.
def _parse_json_dict(json_dict, verbose=True, full_output=False):
"""Reconstruct a list of Obs or structures containing Obs from a dict that
was built out of a json string.
The following structures are supported: Obs, list, numpy.ndarray, Corr
If the list contains only one element, it is unpacked from the list.
@ -446,8 +447,6 @@ def import_json_string(json_string, verbose=True, full_output=False):
my_corr.prange = temp_prange
return my_corr
json_dict = json.loads(json_string)
prog = json_dict.get('program', '')
version = json_dict.get('version', '')
who = json_dict.get('who', '')
@ -495,6 +494,26 @@ def import_json_string(json_string, verbose=True, full_output=False):
return ol
def import_json_string(json_string, verbose=True, full_output=False):
"""Reconstruct a list of Obs or structures containing Obs from a json string.
The following structures are supported: Obs, list, numpy.ndarray, Corr
If the list contains only one element, it is unpacked from the list.
Parameters
----------
json_string : str
json string containing the data.
verbose : bool
Print additional information that was written to the file.
full_output : bool
If True, a dict containing auxiliary information and the data is returned.
If False, only the data is returned.
"""
return _parse_json_dict(json.loads(json_string), verbose, full_output)
def load_json(fname, verbose=True, gz=True, full_output=False):
"""Import a list of Obs or structures containing Obs from a .json(.gz) file.
@ -519,14 +538,14 @@ def load_json(fname, verbose=True, gz=True, full_output=False):
if not fname.endswith('.gz'):
fname += '.gz'
with gzip.open(fname, 'r') as fin:
d = fin.read().decode('utf-8')
d = json.load(fin)
else:
if fname.endswith('.gz'):
warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
with open(fname, 'r', encoding='utf-8') as fin:
d = fin.read()
d = json.loads(fin.read())
return import_json_string(d, verbose, full_output)
return _parse_json_dict(d, verbose, full_output)
def _ol_from_dict(ind, reps='DICTOBS'):