mirror of
https://github.com/fjosw/pyerrors.git
synced 2025-03-15 06:40:24 +01:00
Reduce RAM requirements when reading a JSON file
This commit is contained in:
parent
bd20057f52
commit
005f54e0d1
1 changed files with 26 additions and 7 deletions
|
@ -291,8 +291,9 @@ def dump_to_json(ol, fname, description='', indent=1, gz=True):
|
|||
fp.close()
|
||||
|
||||
|
||||
def import_json_string(json_string, verbose=True, full_output=False):
|
||||
"""Reconstruct a list of Obs or structures containing Obs from a json string.
|
||||
def _parse_json_dict(json_dict, verbose=True, full_output=False):
|
||||
"""Reconstruct a list of Obs or structures containing Obs from a dict that
|
||||
was built out of a json string.
|
||||
|
||||
The following structures are supported: Obs, list, numpy.ndarray, Corr
|
||||
If the list contains only one element, it is unpacked from the list.
|
||||
|
@ -446,8 +447,6 @@ def import_json_string(json_string, verbose=True, full_output=False):
|
|||
my_corr.prange = temp_prange
|
||||
return my_corr
|
||||
|
||||
json_dict = json.loads(json_string)
|
||||
|
||||
prog = json_dict.get('program', '')
|
||||
version = json_dict.get('version', '')
|
||||
who = json_dict.get('who', '')
|
||||
|
@ -495,6 +494,26 @@ def import_json_string(json_string, verbose=True, full_output=False):
|
|||
return ol
|
||||
|
||||
|
||||
def import_json_string(json_string, verbose=True, full_output=False):
|
||||
"""Reconstruct a list of Obs or structures containing Obs from a json string.
|
||||
|
||||
The following structures are supported: Obs, list, numpy.ndarray, Corr
|
||||
If the list contains only one element, it is unpacked from the list.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
json_string : str
|
||||
json string containing the data.
|
||||
verbose : bool
|
||||
Print additional information that was written to the file.
|
||||
full_output : bool
|
||||
If True, a dict containing auxiliary information and the data is returned.
|
||||
If False, only the data is returned.
|
||||
"""
|
||||
|
||||
return _parse_json_dict(json.loads(json_string), verbose, full_output)
|
||||
|
||||
|
||||
def load_json(fname, verbose=True, gz=True, full_output=False):
|
||||
"""Import a list of Obs or structures containing Obs from a .json(.gz) file.
|
||||
|
||||
|
@ -519,14 +538,14 @@ def load_json(fname, verbose=True, gz=True, full_output=False):
|
|||
if not fname.endswith('.gz'):
|
||||
fname += '.gz'
|
||||
with gzip.open(fname, 'r') as fin:
|
||||
d = fin.read().decode('utf-8')
|
||||
d = json.load(fin)
|
||||
else:
|
||||
if fname.endswith('.gz'):
|
||||
warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
|
||||
with open(fname, 'r', encoding='utf-8') as fin:
|
||||
d = fin.read()
|
||||
d = json.loads(fin.read())
|
||||
|
||||
return import_json_string(d, verbose, full_output)
|
||||
return _parse_json_dict(d, verbose, full_output)
|
||||
|
||||
|
||||
def _ol_from_dict(ind, reps='DICTOBS'):
|
||||
|
|
Loading…
Add table
Reference in a new issue