From d740f8d3c4893ae96434b6daba783b597d50499b Mon Sep 17 00:00:00 2001 From: Simon Kuberski Date: Tue, 30 Nov 2021 16:26:46 +0100 Subject: [PATCH] utf-8 for plain json files and a bugfix for reading arrays --- pyerrors/input/json.py | 16 ++++++---------- tests/io_test.py | 10 +++++++++- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/pyerrors/input/json.py b/pyerrors/input/json.py index bc4deaee..d81a5f94 100644 --- a/pyerrors/input/json.py +++ b/pyerrors/input/json.py @@ -120,7 +120,7 @@ def create_json_string(ol, fname, description='', indent=1): _assert_equal_properties(ol) d = {} d['type'] = 'Array' - d['layout'] = str(oa.shape).lstrip('(').rstrip(')') + d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') if ol[0].tag: d['tag'] = ol[0].tag if isinstance(ol[0].tag, str): @@ -153,7 +153,7 @@ def create_json_string(ol, fname, description='', indent=1): elif isinstance(io, np.ndarray): d['obsdata'].append(write_Array_to_dict(io)) - jsonstring = json.dumps(d, indent=indent, cls=my_encoder) + jsonstring = json.dumps(d, indent=indent, cls=my_encoder, ensure_ascii=False) # workaround for un-indentation of delta lists jsonstring = jsonstring.replace(' "[', ' [').replace(']",', '],').replace(']"\n', ']\n') @@ -192,14 +192,10 @@ def dump_to_json(ol, fname, description='', indent=1, gz=True): fp = gzip.open(fname, 'wb') fp.write(jsonstring.encode('utf-8')) else: - fp = open(fname, 'w') + fp = open(fname, 'w', encoding='utf-8') fp.write(jsonstring) fp.close() - # this would be nicer, since it does not need a string but uses serialization (less memory!) - # with gzip.open(fname, 'wt', encoding='UTF-8') as zipfile: - # json.dump(d, zipfile, indent=indent) - def load_json(fname, verbose=True, gz=True, full_output=False): """Import a list of Obs or structures containing Obs from a .json.gz file. @@ -229,7 +225,7 @@ def load_json(fname, verbose=True, gz=True, full_output=False): for rep in ens['replica']: retd['names'].append(rep['name']) retd['idl'].append([di[0] for di in rep['deltas']]) - retd['deltas'].append([di[1:] for di in rep['deltas']]) + retd['deltas'].append(np.array([di[1:] for di in rep['deltas']])) retd['is_merged'][rep['name']] = rep.get('is_merged', False) retd['deltas'] = np.array(retd['deltas']) return retd @@ -286,7 +282,7 @@ def load_json(fname, verbose=True, gz=True, full_output=False): else: if fname.endswith('.gz'): warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) - with open(fname, 'r') as fin: + with open(fname, 'r', encoding='utf-8') as fin: d = json.loads(fin.read()) prog = d.get('program', '') @@ -303,7 +299,7 @@ def load_json(fname, verbose=True, gz=True, full_output=False): description = d.get('description', '') if description and verbose: print() - print(description) + print('Description: ', description) obsdata = d['obsdata'] ol = [] for io in obsdata: diff --git a/tests/io_test.py b/tests/io_test.py index de309fd0..77a0f474 100644 --- a/tests/io_test.py +++ b/tests/io_test.py @@ -19,7 +19,15 @@ def test_jsonio(): arr = np.array([o3, o5]) mat = np.array([[pe.pseudo_Obs(1.0, .1, 'mat'), pe.pseudo_Obs(0.3, .1, 'mat')], [pe.pseudo_Obs(0.2, .1, 'mat'), pe.pseudo_Obs(2.0, .4, 'mat')]]) - ol = [o4, do, testl, mat, arr, np.array([o])] + tt1 = pe.Obs([np.random.rand(100)], ['t|r1'], idl=[range(2, 202, 2)]) + tt2 = pe.Obs([np.random.rand(100)], ['t|r2'], idl=[range(2, 202, 2)]) + tt3 = pe.Obs([np.random.rand(102)], ['qe']) + + tt = tt1 + tt2 + tt3 + + tt.tag = 'Test Obs' + + ol = [o4, do, testl, mat, arr, np.array([o]), np.array([tt, tt]), [tt, tt]] fname = 'test_rw' jsonio.dump_to_json(ol, fname, indent=1)