From 87485f03fa7aaa6cfbb8b938f002796df247865a Mon Sep 17 00:00:00 2001 From: fjosw Date: Thu, 19 May 2022 08:58:09 +0000 Subject: [PATCH] Documentation updated --- docs/pyerrors/input/json.html | 1906 +++++++++++++++++---------------- 1 file changed, 955 insertions(+), 951 deletions(-) diff --git a/docs/pyerrors/input/json.html b/docs/pyerrors/input/json.html index c4690c2a..aa16fc19 100644 --- a/docs/pyerrors/input/json.html +++ b/docs/pyerrors/input/json.html @@ -121,667 +121,669 @@ 33 34 def _gen_data_d_from_list(ol): 35 dl = [] - 36 for name in ol[0].mc_names: - 37 ed = {} - 38 ed['id'] = name - 39 ed['replica'] = [] - 40 for r_name in ol[0].e_content[name]: - 41 rd = {} - 42 rd['name'] = r_name - 43 if ol[0].is_merged.get(r_name, False): - 44 rd['is_merged'] = True - 45 rd['deltas'] = [] - 46 for i in range(len(ol[0].idl[r_name])): - 47 rd['deltas'].append([ol[0].idl[r_name][i]]) - 48 for o in ol: - 49 rd['deltas'][-1].append(o.deltas[r_name][i]) - 50 ed['replica'].append(rd) - 51 dl.append(ed) - 52 return dl - 53 - 54 def _gen_cdata_d_from_list(ol): - 55 dl = [] - 56 for name in ol[0].cov_names: - 57 ed = {} - 58 ed['id'] = name - 59 ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') - 60 ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) - 61 ncov = ol[0].covobs[name].cov.shape[0] - 62 ed['grad'] = [] - 63 for i in range(ncov): - 64 ed['grad'].append([]) - 65 for o in ol: - 66 ed['grad'][-1].append(o.covobs[name].grad[i][0]) - 67 dl.append(ed) - 68 return dl - 69 - 70 def write_Obs_to_dict(o): - 71 d = {} - 72 d['type'] = 'Obs' - 73 d['layout'] = '1' - 74 if o.tag: - 75 d['tag'] = [o.tag] - 76 if o.reweighted: - 77 d['reweighted'] = o.reweighted - 78 d['value'] = [o.value] - 79 data = _gen_data_d_from_list([o]) - 80 if len(data) > 0: - 81 d['data'] = data - 82 cdata = _gen_cdata_d_from_list([o]) - 83 if len(cdata) > 0: - 84 d['cdata'] = cdata - 85 return d - 86 - 87 def write_List_to_dict(ol): - 88 _assert_equal_properties(ol) - 89 d = {} - 90 d['type'] = 'List' - 91 d['layout'] = '%d' % len(ol) - 92 taglist = [o.tag for o in ol] - 93 if np.any([tag is not None for tag in taglist]): - 94 d['tag'] = taglist - 95 if ol[0].reweighted: - 96 d['reweighted'] = ol[0].reweighted - 97 d['value'] = [o.value for o in ol] - 98 data = _gen_data_d_from_list(ol) - 99 if len(data) > 0: -100 d['data'] = data -101 cdata = _gen_cdata_d_from_list(ol) -102 if len(cdata) > 0: -103 d['cdata'] = cdata -104 return d -105 -106 def write_Array_to_dict(oa): -107 ol = np.ravel(oa) -108 _assert_equal_properties(ol) -109 d = {} -110 d['type'] = 'Array' -111 d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') -112 taglist = [o.tag for o in ol] -113 if np.any([tag is not None for tag in taglist]): -114 d['tag'] = taglist -115 if ol[0].reweighted: -116 d['reweighted'] = ol[0].reweighted -117 d['value'] = [o.value for o in ol] -118 data = _gen_data_d_from_list(ol) -119 if len(data) > 0: -120 d['data'] = data -121 cdata = _gen_cdata_d_from_list(ol) -122 if len(cdata) > 0: -123 d['cdata'] = cdata -124 return d -125 -126 def _nan_Obs_like(obs): -127 samples = [] -128 names = [] -129 idl = [] -130 for key, value in obs.idl.items(): -131 samples.append([np.nan] * len(value)) -132 names.append(key) -133 idl.append(value) -134 my_obs = Obs(samples, names, idl) -135 my_obs._covobs = obs._covobs -136 for name in obs._covobs: -137 my_obs.names.append(name) -138 my_obs.reweighted = obs.reweighted -139 my_obs.is_merged = obs.is_merged -140 return my_obs -141 -142 def write_Corr_to_dict(my_corr): -143 first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j)) -144 dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object) -145 dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0]) -146 content = [o if o is not None else dummy_array for o in my_corr.content] -147 dat = write_Array_to_dict(np.array(content, dtype=object)) -148 dat['type'] = 'Corr' -149 corr_meta_data = str(my_corr.tag) -150 if 'tag' in dat.keys(): -151 dat['tag'].append(corr_meta_data) -152 else: -153 dat['tag'] = [corr_meta_data] -154 taglist = dat['tag'] -155 dat['tag'] = {} # tag is now a dictionary, that contains the previous taglist in the key "tag" -156 dat['tag']['tag'] = taglist -157 if my_corr.prange is not None: -158 dat['tag']['prange'] = my_corr.prange -159 return dat -160 -161 if not isinstance(ol, list): -162 ol = [ol] -163 -164 d = {} -165 d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__) -166 d['version'] = '1.0' -167 d['who'] = getpass.getuser() -168 d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z') -169 d['host'] = socket.gethostname() + ', ' + platform.platform() -170 -171 if description: -172 d['description'] = description -173 -174 d['obsdata'] = [] -175 for io in ol: -176 if isinstance(io, Obs): -177 d['obsdata'].append(write_Obs_to_dict(io)) -178 elif isinstance(io, list): -179 d['obsdata'].append(write_List_to_dict(io)) -180 elif isinstance(io, np.ndarray): -181 d['obsdata'].append(write_Array_to_dict(io)) -182 elif isinstance(io, Corr): -183 d['obsdata'].append(write_Corr_to_dict(io)) -184 else: -185 raise Exception("Unkown datatype.") -186 -187 if indent: -188 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY) -189 else: -190 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT) -191 -192 -193def dump_to_json(ol, fname, description='', indent=1, gz=True): -194 """Export a list of Obs or structures containing Obs to a .json(.gz) file -195 -196 Parameters -197 ---------- -198 ol : list -199 List of objects that will be exported. At the moment, these objects can be -200 either of: Obs, list, numpy.ndarray, Corr. -201 All Obs inside a structure have to be defined on the same set of configurations. -202 fname : str -203 Filename of the output file. -204 description : str -205 Optional string that describes the contents of the json file. -206 indent : int -207 Specify the indentation level of the json file. None or 0 is permissible and -208 saves disk space. -209 gz : bool -210 If True, the output is a gzipped json. If False, the output is a json file. -211 """ -212 -213 jsonstring = create_json_string(ol, description, indent) + 36 No = len(ol) + 37 for name in ol[0].mc_names: + 38 ed = {} + 39 ed['id'] = name + 40 ed['replica'] = [] + 41 for r_name in ol[0].e_content[name]: + 42 rd = {} + 43 rd['name'] = r_name + 44 if ol[0].is_merged.get(r_name, False): + 45 rd['is_merged'] = True + 46 rd['deltas'] = [] + 47 offsets = [o.r_values[r_name] - o.value for o in ol] + 48 deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)]) + 49 for i in range(len(ol[0].idl[r_name])): + 50 rd['deltas'].append([ol[0].idl[r_name][i]]) + 51 rd['deltas'][-1] += deltas[i].tolist() + 52 ed['replica'].append(rd) + 53 dl.append(ed) + 54 return dl + 55 + 56 def _gen_cdata_d_from_list(ol): + 57 dl = [] + 58 for name in ol[0].cov_names: + 59 ed = {} + 60 ed['id'] = name + 61 ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') + 62 ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) + 63 ncov = ol[0].covobs[name].cov.shape[0] + 64 ed['grad'] = [] + 65 for i in range(ncov): + 66 ed['grad'].append([]) + 67 for o in ol: + 68 ed['grad'][-1].append(o.covobs[name].grad[i][0]) + 69 dl.append(ed) + 70 return dl + 71 + 72 def write_Obs_to_dict(o): + 73 d = {} + 74 d['type'] = 'Obs' + 75 d['layout'] = '1' + 76 if o.tag: + 77 d['tag'] = [o.tag] + 78 if o.reweighted: + 79 d['reweighted'] = o.reweighted + 80 d['value'] = [o.value] + 81 data = _gen_data_d_from_list([o]) + 82 if len(data) > 0: + 83 d['data'] = data + 84 cdata = _gen_cdata_d_from_list([o]) + 85 if len(cdata) > 0: + 86 d['cdata'] = cdata + 87 return d + 88 + 89 def write_List_to_dict(ol): + 90 _assert_equal_properties(ol) + 91 d = {} + 92 d['type'] = 'List' + 93 d['layout'] = '%d' % len(ol) + 94 taglist = [o.tag for o in ol] + 95 if np.any([tag is not None for tag in taglist]): + 96 d['tag'] = taglist + 97 if ol[0].reweighted: + 98 d['reweighted'] = ol[0].reweighted + 99 d['value'] = [o.value for o in ol] +100 data = _gen_data_d_from_list(ol) +101 if len(data) > 0: +102 d['data'] = data +103 cdata = _gen_cdata_d_from_list(ol) +104 if len(cdata) > 0: +105 d['cdata'] = cdata +106 return d +107 +108 def write_Array_to_dict(oa): +109 ol = np.ravel(oa) +110 _assert_equal_properties(ol) +111 d = {} +112 d['type'] = 'Array' +113 d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') +114 taglist = [o.tag for o in ol] +115 if np.any([tag is not None for tag in taglist]): +116 d['tag'] = taglist +117 if ol[0].reweighted: +118 d['reweighted'] = ol[0].reweighted +119 d['value'] = [o.value for o in ol] +120 data = _gen_data_d_from_list(ol) +121 if len(data) > 0: +122 d['data'] = data +123 cdata = _gen_cdata_d_from_list(ol) +124 if len(cdata) > 0: +125 d['cdata'] = cdata +126 return d +127 +128 def _nan_Obs_like(obs): +129 samples = [] +130 names = [] +131 idl = [] +132 for key, value in obs.idl.items(): +133 samples.append([np.nan] * len(value)) +134 names.append(key) +135 idl.append(value) +136 my_obs = Obs(samples, names, idl) +137 my_obs._covobs = obs._covobs +138 for name in obs._covobs: +139 my_obs.names.append(name) +140 my_obs.reweighted = obs.reweighted +141 my_obs.is_merged = obs.is_merged +142 return my_obs +143 +144 def write_Corr_to_dict(my_corr): +145 first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j)) +146 dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object) +147 dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0]) +148 content = [o if o is not None else dummy_array for o in my_corr.content] +149 dat = write_Array_to_dict(np.array(content, dtype=object)) +150 dat['type'] = 'Corr' +151 corr_meta_data = str(my_corr.tag) +152 if 'tag' in dat.keys(): +153 dat['tag'].append(corr_meta_data) +154 else: +155 dat['tag'] = [corr_meta_data] +156 taglist = dat['tag'] +157 dat['tag'] = {} # tag is now a dictionary, that contains the previous taglist in the key "tag" +158 dat['tag']['tag'] = taglist +159 if my_corr.prange is not None: +160 dat['tag']['prange'] = my_corr.prange +161 return dat +162 +163 if not isinstance(ol, list): +164 ol = [ol] +165 +166 d = {} +167 d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__) +168 d['version'] = '1.0' +169 d['who'] = getpass.getuser() +170 d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z') +171 d['host'] = socket.gethostname() + ', ' + platform.platform() +172 +173 if description: +174 d['description'] = description +175 +176 d['obsdata'] = [] +177 for io in ol: +178 if isinstance(io, Obs): +179 d['obsdata'].append(write_Obs_to_dict(io)) +180 elif isinstance(io, list): +181 d['obsdata'].append(write_List_to_dict(io)) +182 elif isinstance(io, np.ndarray): +183 d['obsdata'].append(write_Array_to_dict(io)) +184 elif isinstance(io, Corr): +185 d['obsdata'].append(write_Corr_to_dict(io)) +186 else: +187 raise Exception("Unkown datatype.") +188 +189 if indent: +190 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY) +191 else: +192 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT) +193 +194 +195def dump_to_json(ol, fname, description='', indent=1, gz=True): +196 """Export a list of Obs or structures containing Obs to a .json(.gz) file +197 +198 Parameters +199 ---------- +200 ol : list +201 List of objects that will be exported. At the moment, these objects can be +202 either of: Obs, list, numpy.ndarray, Corr. +203 All Obs inside a structure have to be defined on the same set of configurations. +204 fname : str +205 Filename of the output file. +206 description : str +207 Optional string that describes the contents of the json file. +208 indent : int +209 Specify the indentation level of the json file. None or 0 is permissible and +210 saves disk space. +211 gz : bool +212 If True, the output is a gzipped json. If False, the output is a json file. +213 """ 214 -215 if not fname.endswith('.json') and not fname.endswith('.gz'): -216 fname += '.json' -217 -218 if gz: -219 if not fname.endswith('.gz'): -220 fname += '.gz' -221 -222 fp = gzip.open(fname, 'wb') -223 fp.write(jsonstring.encode('utf-8')) -224 else: -225 fp = open(fname, 'w', encoding='utf-8') -226 fp.write(jsonstring) -227 fp.close() -228 -229 -230def _parse_json_dict(json_dict, verbose=True, full_output=False): -231 """Reconstruct a list of Obs or structures containing Obs from a dict that -232 was built out of a json string. -233 -234 The following structures are supported: Obs, list, numpy.ndarray, Corr -235 If the list contains only one element, it is unpacked from the list. -236 -237 Parameters -238 ---------- -239 json_string : str -240 json string containing the data. -241 verbose : bool -242 Print additional information that was written to the file. -243 full_output : bool -244 If True, a dict containing auxiliary information and the data is returned. -245 If False, only the data is returned. -246 """ -247 -248 def _gen_obsd_from_datad(d): -249 retd = {} -250 if d: -251 retd['names'] = [] -252 retd['idl'] = [] -253 retd['deltas'] = [] -254 retd['is_merged'] = {} -255 for ens in d: -256 for rep in ens['replica']: -257 rep_name = rep['name'] -258 if len(rep_name) > len(ens["id"]): -259 if rep_name[len(ens["id"])] != "|": -260 tmp_list = list(rep_name) -261 tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):] -262 rep_name = ''.join(tmp_list) -263 retd['names'].append(rep_name) -264 retd['idl'].append([di[0] for di in rep['deltas']]) -265 retd['deltas'].append(np.array([di[1:] for di in rep['deltas']])) -266 retd['is_merged'][rep_name] = rep.get('is_merged', False) -267 return retd -268 -269 def _gen_covobsd_from_cdatad(d): -270 retd = {} -271 for ens in d: -272 retl = [] -273 name = ens['id'] -274 layouts = ens.get('layout', '1').strip() -275 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] -276 cov = np.reshape(ens['cov'], layout) -277 grad = ens['grad'] -278 nobs = len(grad[0]) -279 for i in range(nobs): -280 retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]}) -281 retd[name] = retl -282 return retd -283 -284 def get_Obs_from_dict(o): -285 layouts = o.get('layout', '1').strip() -286 if layouts != '1': -287 raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning) -288 -289 values = o['value'] -290 od = _gen_obsd_from_datad(o.get('data', {})) -291 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) -292 -293 if od: -294 ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl']) -295 ret.is_merged = od['is_merged'] -296 else: -297 ret = Obs([], [], means=[]) -298 ret._value = values[0] -299 for name in cd: -300 co = cd[name][0] -301 ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) -302 ret.names.append(co['name']) -303 -304 ret.reweighted = o.get('reweighted', False) -305 ret.tag = o.get('tag', [None])[0] -306 return ret -307 -308 def get_List_from_dict(o): -309 layouts = o.get('layout', '1').strip() -310 layout = int(layouts) -311 values = o['value'] -312 od = _gen_obsd_from_datad(o.get('data', {})) -313 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) -314 -315 ret = [] -316 taglist = o.get('tag', layout * [None]) -317 for i in range(layout): -318 if od: -319 ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl'])) -320 ret[-1].is_merged = od['is_merged'] -321 else: -322 ret.append(Obs([], [], means=[])) -323 ret[-1]._value = values[i] -324 print('Created Obs with means= ', values[i]) -325 for name in cd: -326 co = cd[name][i] -327 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) -328 ret[-1].names.append(co['name']) -329 -330 ret[-1].reweighted = o.get('reweighted', False) -331 ret[-1].tag = taglist[i] -332 return ret -333 -334 def get_Array_from_dict(o): -335 layouts = o.get('layout', '1').strip() -336 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] -337 N = np.prod(layout) -338 values = o['value'] -339 od = _gen_obsd_from_datad(o.get('data', {})) -340 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) -341 -342 ret = [] -343 taglist = o.get('tag', N * [None]) -344 for i in range(N): -345 if od: -346 ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl'])) -347 ret[-1].is_merged = od['is_merged'] -348 else: -349 ret.append(Obs([], [], means=[])) -350 ret[-1]._value = values[i] -351 for name in cd: -352 co = cd[name][i] -353 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) -354 ret[-1].names.append(co['name']) -355 ret[-1].reweighted = o.get('reweighted', False) -356 ret[-1].tag = taglist[i] -357 return np.reshape(ret, layout) -358 -359 def get_Corr_from_dict(o): -360 if isinstance(o.get('tag'), list): # supports the old way -361 taglist = o.get('tag') # This had to be modified to get the taglist from the dictionary -362 temp_prange = None -363 elif isinstance(o.get('tag'), dict): -364 tagdic = o.get('tag') -365 taglist = tagdic['tag'] -366 if 'prange' in tagdic: -367 temp_prange = tagdic['prange'] -368 else: -369 temp_prange = None -370 else: -371 raise Exception("The tag is not a list or dict") -372 -373 corr_tag = taglist[-1] -374 tmp_o = o -375 tmp_o['tag'] = taglist[:-1] -376 if len(tmp_o['tag']) == 0: -377 del tmp_o['tag'] -378 dat = get_Array_from_dict(tmp_o) -379 my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)]) -380 if corr_tag != 'None': -381 my_corr.tag = corr_tag -382 -383 my_corr.prange = temp_prange -384 return my_corr -385 -386 prog = json_dict.get('program', '') -387 version = json_dict.get('version', '') -388 who = json_dict.get('who', '') -389 date = json_dict.get('date', '') -390 host = json_dict.get('host', '') -391 if prog and verbose: -392 print('Data has been written using %s.' % (prog)) -393 if version and verbose: -394 print('Format version %s' % (version)) -395 if np.any([who, date, host] and verbose): -396 print('Written by %s on %s on host %s' % (who, date, host)) -397 description = json_dict.get('description', '') -398 if description and verbose: -399 print() -400 print('Description: ', description) -401 obsdata = json_dict['obsdata'] -402 ol = [] -403 for io in obsdata: -404 if io['type'] == 'Obs': -405 ol.append(get_Obs_from_dict(io)) -406 elif io['type'] == 'List': -407 ol.append(get_List_from_dict(io)) -408 elif io['type'] == 'Array': -409 ol.append(get_Array_from_dict(io)) -410 elif io['type'] == 'Corr': -411 ol.append(get_Corr_from_dict(io)) -412 else: -413 raise Exception("Unkown datatype.") -414 -415 if full_output: -416 retd = {} -417 retd['program'] = prog -418 retd['version'] = version -419 retd['who'] = who -420 retd['date'] = date -421 retd['host'] = host -422 retd['description'] = description -423 retd['obsdata'] = ol -424 -425 return retd -426 else: -427 if len(obsdata) == 1: -428 ol = ol[0] -429 -430 return ol +215 jsonstring = create_json_string(ol, description, indent) +216 +217 if not fname.endswith('.json') and not fname.endswith('.gz'): +218 fname += '.json' +219 +220 if gz: +221 if not fname.endswith('.gz'): +222 fname += '.gz' +223 +224 fp = gzip.open(fname, 'wb') +225 fp.write(jsonstring.encode('utf-8')) +226 else: +227 fp = open(fname, 'w', encoding='utf-8') +228 fp.write(jsonstring) +229 fp.close() +230 +231 +232def _parse_json_dict(json_dict, verbose=True, full_output=False): +233 """Reconstruct a list of Obs or structures containing Obs from a dict that +234 was built out of a json string. +235 +236 The following structures are supported: Obs, list, numpy.ndarray, Corr +237 If the list contains only one element, it is unpacked from the list. +238 +239 Parameters +240 ---------- +241 json_string : str +242 json string containing the data. +243 verbose : bool +244 Print additional information that was written to the file. +245 full_output : bool +246 If True, a dict containing auxiliary information and the data is returned. +247 If False, only the data is returned. +248 """ +249 +250 def _gen_obsd_from_datad(d): +251 retd = {} +252 if d: +253 retd['names'] = [] +254 retd['idl'] = [] +255 retd['deltas'] = [] +256 retd['is_merged'] = {} +257 for ens in d: +258 for rep in ens['replica']: +259 rep_name = rep['name'] +260 if len(rep_name) > len(ens["id"]): +261 if rep_name[len(ens["id"])] != "|": +262 tmp_list = list(rep_name) +263 tmp_list = tmp_list[:len(ens["id"])] + ["|"] + tmp_list[len(ens["id"]):] +264 rep_name = ''.join(tmp_list) +265 retd['names'].append(rep_name) +266 retd['idl'].append([di[0] for di in rep['deltas']]) +267 retd['deltas'].append(np.array([di[1:] for di in rep['deltas']])) +268 retd['is_merged'][rep_name] = rep.get('is_merged', False) +269 return retd +270 +271 def _gen_covobsd_from_cdatad(d): +272 retd = {} +273 for ens in d: +274 retl = [] +275 name = ens['id'] +276 layouts = ens.get('layout', '1').strip() +277 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] +278 cov = np.reshape(ens['cov'], layout) +279 grad = ens['grad'] +280 nobs = len(grad[0]) +281 for i in range(nobs): +282 retl.append({'name': name, 'cov': cov, 'grad': [g[i] for g in grad]}) +283 retd[name] = retl +284 return retd +285 +286 def get_Obs_from_dict(o): +287 layouts = o.get('layout', '1').strip() +288 if layouts != '1': +289 raise Exception("layout is %s has to be 1 for type Obs." % (layouts), RuntimeWarning) +290 +291 values = o['value'] +292 od = _gen_obsd_from_datad(o.get('data', {})) +293 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) +294 +295 if od: +296 ret = Obs([[ddi[0] + values[0] for ddi in di] for di in od['deltas']], od['names'], idl=od['idl']) +297 ret.is_merged = od['is_merged'] +298 else: +299 ret = Obs([], [], means=[]) +300 ret._value = values[0] +301 for name in cd: +302 co = cd[name][0] +303 ret._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) +304 ret.names.append(co['name']) +305 +306 ret.reweighted = o.get('reweighted', False) +307 ret.tag = o.get('tag', [None])[0] +308 return ret +309 +310 def get_List_from_dict(o): +311 layouts = o.get('layout', '1').strip() +312 layout = int(layouts) +313 values = o['value'] +314 od = _gen_obsd_from_datad(o.get('data', {})) +315 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) +316 +317 ret = [] +318 taglist = o.get('tag', layout * [None]) +319 for i in range(layout): +320 if od: +321 ret.append(Obs([list(di[:, i] + values[i]) for di in od['deltas']], od['names'], idl=od['idl'])) +322 ret[-1].is_merged = od['is_merged'] +323 else: +324 ret.append(Obs([], [], means=[])) +325 ret[-1]._value = values[i] +326 print('Created Obs with means= ', values[i]) +327 for name in cd: +328 co = cd[name][i] +329 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) +330 ret[-1].names.append(co['name']) +331 +332 ret[-1].reweighted = o.get('reweighted', False) +333 ret[-1].tag = taglist[i] +334 return ret +335 +336 def get_Array_from_dict(o): +337 layouts = o.get('layout', '1').strip() +338 layout = [int(ls.strip()) for ls in layouts.split(',') if len(ls) > 0] +339 N = np.prod(layout) +340 values = o['value'] +341 od = _gen_obsd_from_datad(o.get('data', {})) +342 cd = _gen_covobsd_from_cdatad(o.get('cdata', {})) +343 +344 ret = [] +345 taglist = o.get('tag', N * [None]) +346 for i in range(N): +347 if od: +348 ret.append(Obs([di[:, i] + values[i] for di in od['deltas']], od['names'], idl=od['idl'])) +349 ret[-1].is_merged = od['is_merged'] +350 else: +351 ret.append(Obs([], [], means=[])) +352 ret[-1]._value = values[i] +353 for name in cd: +354 co = cd[name][i] +355 ret[-1]._covobs[name] = Covobs(None, co['cov'], co['name'], grad=co['grad']) +356 ret[-1].names.append(co['name']) +357 ret[-1].reweighted = o.get('reweighted', False) +358 ret[-1].tag = taglist[i] +359 return np.reshape(ret, layout) +360 +361 def get_Corr_from_dict(o): +362 if isinstance(o.get('tag'), list): # supports the old way +363 taglist = o.get('tag') # This had to be modified to get the taglist from the dictionary +364 temp_prange = None +365 elif isinstance(o.get('tag'), dict): +366 tagdic = o.get('tag') +367 taglist = tagdic['tag'] +368 if 'prange' in tagdic: +369 temp_prange = tagdic['prange'] +370 else: +371 temp_prange = None +372 else: +373 raise Exception("The tag is not a list or dict") +374 +375 corr_tag = taglist[-1] +376 tmp_o = o +377 tmp_o['tag'] = taglist[:-1] +378 if len(tmp_o['tag']) == 0: +379 del tmp_o['tag'] +380 dat = get_Array_from_dict(tmp_o) +381 my_corr = Corr([None if np.isnan(o.ravel()[0].value) else o for o in list(dat)]) +382 if corr_tag != 'None': +383 my_corr.tag = corr_tag +384 +385 my_corr.prange = temp_prange +386 return my_corr +387 +388 prog = json_dict.get('program', '') +389 version = json_dict.get('version', '') +390 who = json_dict.get('who', '') +391 date = json_dict.get('date', '') +392 host = json_dict.get('host', '') +393 if prog and verbose: +394 print('Data has been written using %s.' % (prog)) +395 if version and verbose: +396 print('Format version %s' % (version)) +397 if np.any([who, date, host] and verbose): +398 print('Written by %s on %s on host %s' % (who, date, host)) +399 description = json_dict.get('description', '') +400 if description and verbose: +401 print() +402 print('Description: ', description) +403 obsdata = json_dict['obsdata'] +404 ol = [] +405 for io in obsdata: +406 if io['type'] == 'Obs': +407 ol.append(get_Obs_from_dict(io)) +408 elif io['type'] == 'List': +409 ol.append(get_List_from_dict(io)) +410 elif io['type'] == 'Array': +411 ol.append(get_Array_from_dict(io)) +412 elif io['type'] == 'Corr': +413 ol.append(get_Corr_from_dict(io)) +414 else: +415 raise Exception("Unkown datatype.") +416 +417 if full_output: +418 retd = {} +419 retd['program'] = prog +420 retd['version'] = version +421 retd['who'] = who +422 retd['date'] = date +423 retd['host'] = host +424 retd['description'] = description +425 retd['obsdata'] = ol +426 +427 return retd +428 else: +429 if len(obsdata) == 1: +430 ol = ol[0] 431 -432 -433def import_json_string(json_string, verbose=True, full_output=False): -434 """Reconstruct a list of Obs or structures containing Obs from a json string. -435 -436 The following structures are supported: Obs, list, numpy.ndarray, Corr -437 If the list contains only one element, it is unpacked from the list. -438 -439 Parameters -440 ---------- -441 json_string : str -442 json string containing the data. -443 verbose : bool -444 Print additional information that was written to the file. -445 full_output : bool -446 If True, a dict containing auxiliary information and the data is returned. -447 If False, only the data is returned. -448 """ -449 -450 return _parse_json_dict(json.loads(json_string), verbose, full_output) +432 return ol +433 +434 +435def import_json_string(json_string, verbose=True, full_output=False): +436 """Reconstruct a list of Obs or structures containing Obs from a json string. +437 +438 The following structures are supported: Obs, list, numpy.ndarray, Corr +439 If the list contains only one element, it is unpacked from the list. +440 +441 Parameters +442 ---------- +443 json_string : str +444 json string containing the data. +445 verbose : bool +446 Print additional information that was written to the file. +447 full_output : bool +448 If True, a dict containing auxiliary information and the data is returned. +449 If False, only the data is returned. +450 """ 451 -452 -453def load_json(fname, verbose=True, gz=True, full_output=False): -454 """Import a list of Obs or structures containing Obs from a .json(.gz) file. -455 -456 The following structures are supported: Obs, list, numpy.ndarray, Corr -457 If the list contains only one element, it is unpacked from the list. -458 -459 Parameters -460 ---------- -461 fname : str -462 Filename of the input file. -463 verbose : bool -464 Print additional information that was written to the file. -465 gz : bool -466 If True, assumes that data is gzipped. If False, assumes JSON file. -467 full_output : bool -468 If True, a dict containing auxiliary information and the data is returned. -469 If False, only the data is returned. -470 """ -471 if not fname.endswith('.json') and not fname.endswith('.gz'): -472 fname += '.json' -473 if gz: -474 if not fname.endswith('.gz'): -475 fname += '.gz' -476 with gzip.open(fname, 'r') as fin: -477 d = json.load(fin) -478 else: -479 if fname.endswith('.gz'): -480 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) -481 with open(fname, 'r', encoding='utf-8') as fin: -482 d = json.loads(fin.read()) -483 -484 return _parse_json_dict(d, verbose, full_output) +452 return _parse_json_dict(json.loads(json_string), verbose, full_output) +453 +454 +455def load_json(fname, verbose=True, gz=True, full_output=False): +456 """Import a list of Obs or structures containing Obs from a .json(.gz) file. +457 +458 The following structures are supported: Obs, list, numpy.ndarray, Corr +459 If the list contains only one element, it is unpacked from the list. +460 +461 Parameters +462 ---------- +463 fname : str +464 Filename of the input file. +465 verbose : bool +466 Print additional information that was written to the file. +467 gz : bool +468 If True, assumes that data is gzipped. If False, assumes JSON file. +469 full_output : bool +470 If True, a dict containing auxiliary information and the data is returned. +471 If False, only the data is returned. +472 """ +473 if not fname.endswith('.json') and not fname.endswith('.gz'): +474 fname += '.json' +475 if gz: +476 if not fname.endswith('.gz'): +477 fname += '.gz' +478 with gzip.open(fname, 'r') as fin: +479 d = json.load(fin) +480 else: +481 if fname.endswith('.gz'): +482 warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning) +483 with open(fname, 'r', encoding='utf-8') as fin: +484 d = json.loads(fin.read()) 485 -486 -487def _ol_from_dict(ind, reps='DICTOBS'): -488 """Convert a dictionary of Obs objects to a list and a dictionary that contains -489 placeholders instead of the Obs objects. -490 -491 Parameters -492 ---------- -493 ind : dict -494 Dict of JSON valid structures and objects that will be exported. -495 At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr. -496 All Obs inside a structure have to be defined on the same set of configurations. -497 reps : str -498 Specify the structure of the placeholder in exported dict to be reps[0-9]+. -499 """ -500 -501 obstypes = (Obs, Corr, np.ndarray) +486 return _parse_json_dict(d, verbose, full_output) +487 +488 +489def _ol_from_dict(ind, reps='DICTOBS'): +490 """Convert a dictionary of Obs objects to a list and a dictionary that contains +491 placeholders instead of the Obs objects. +492 +493 Parameters +494 ---------- +495 ind : dict +496 Dict of JSON valid structures and objects that will be exported. +497 At the moment, these object can be either of: Obs, list, numpy.ndarray, Corr. +498 All Obs inside a structure have to be defined on the same set of configurations. +499 reps : str +500 Specify the structure of the placeholder in exported dict to be reps[0-9]+. +501 """ 502 -503 if not reps.isalnum(): -504 raise Exception('Placeholder string has to be alphanumeric!') -505 ol = [] -506 counter = 0 -507 -508 def dict_replace_obs(d): -509 nonlocal ol -510 nonlocal counter -511 x = {} -512 for k, v in d.items(): -513 if isinstance(v, dict): -514 v = dict_replace_obs(v) -515 elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]): -516 v = obslist_replace_obs(v) -517 elif isinstance(v, list): -518 v = list_replace_obs(v) -519 elif isinstance(v, obstypes): -520 ol.append(v) -521 v = reps + '%d' % (counter) -522 counter += 1 -523 elif isinstance(v, str): -524 if bool(re.match(r'%s[0-9]+' % (reps), v)): -525 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps)) -526 x[k] = v -527 return x -528 -529 def list_replace_obs(li): -530 nonlocal ol -531 nonlocal counter -532 x = [] -533 for e in li: -534 if isinstance(e, list): -535 e = list_replace_obs(e) -536 elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]): -537 e = obslist_replace_obs(e) -538 elif isinstance(e, dict): -539 e = dict_replace_obs(e) -540 elif isinstance(e, obstypes): -541 ol.append(e) -542 e = reps + '%d' % (counter) -543 counter += 1 -544 elif isinstance(e, str): -545 if bool(re.match(r'%s[0-9]+' % (reps), e)): -546 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps)) -547 x.append(e) -548 return x -549 -550 def obslist_replace_obs(li): -551 nonlocal ol -552 nonlocal counter -553 il = [] -554 for e in li: -555 il.append(e) -556 -557 ol.append(il) -558 x = reps + '%d' % (counter) -559 counter += 1 -560 return x -561 -562 nd = dict_replace_obs(ind) +503 obstypes = (Obs, Corr, np.ndarray) +504 +505 if not reps.isalnum(): +506 raise Exception('Placeholder string has to be alphanumeric!') +507 ol = [] +508 counter = 0 +509 +510 def dict_replace_obs(d): +511 nonlocal ol +512 nonlocal counter +513 x = {} +514 for k, v in d.items(): +515 if isinstance(v, dict): +516 v = dict_replace_obs(v) +517 elif isinstance(v, list) and all([isinstance(o, Obs) for o in v]): +518 v = obslist_replace_obs(v) +519 elif isinstance(v, list): +520 v = list_replace_obs(v) +521 elif isinstance(v, obstypes): +522 ol.append(v) +523 v = reps + '%d' % (counter) +524 counter += 1 +525 elif isinstance(v, str): +526 if bool(re.match(r'%s[0-9]+' % (reps), v)): +527 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (v, reps)) +528 x[k] = v +529 return x +530 +531 def list_replace_obs(li): +532 nonlocal ol +533 nonlocal counter +534 x = [] +535 for e in li: +536 if isinstance(e, list): +537 e = list_replace_obs(e) +538 elif isinstance(e, list) and all([isinstance(o, Obs) for o in e]): +539 e = obslist_replace_obs(e) +540 elif isinstance(e, dict): +541 e = dict_replace_obs(e) +542 elif isinstance(e, obstypes): +543 ol.append(e) +544 e = reps + '%d' % (counter) +545 counter += 1 +546 elif isinstance(e, str): +547 if bool(re.match(r'%s[0-9]+' % (reps), e)): +548 raise Exception('Dict contains string %s that matches the placeholder! %s Cannot be savely exported.' % (e, reps)) +549 x.append(e) +550 return x +551 +552 def obslist_replace_obs(li): +553 nonlocal ol +554 nonlocal counter +555 il = [] +556 for e in li: +557 il.append(e) +558 +559 ol.append(il) +560 x = reps + '%d' % (counter) +561 counter += 1 +562 return x 563 -564 return ol, nd +564 nd = dict_replace_obs(ind) 565 -566 -567def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): -568 """Export a dict of Obs or structures containing Obs to a .json(.gz) file -569 -570 Parameters -571 ---------- -572 od : dict -573 Dict of JSON valid structures and objects that will be exported. -574 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. -575 All Obs inside a structure have to be defined on the same set of configurations. -576 fname : str -577 Filename of the output file. -578 description : str -579 Optional string that describes the contents of the json file. -580 indent : int -581 Specify the indentation level of the json file. None or 0 is permissible and -582 saves disk space. -583 reps : str -584 Specify the structure of the placeholder in exported dict to be reps[0-9]+. -585 gz : bool -586 If True, the output is a gzipped json. If False, the output is a json file. -587 """ -588 -589 if not isinstance(od, dict): -590 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') -591 -592 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' -593 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' -594 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' -595 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') -596 -597 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} -598 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) -599 -600 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz) +566 return ol, nd +567 +568 +569def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True): +570 """Export a dict of Obs or structures containing Obs to a .json(.gz) file +571 +572 Parameters +573 ---------- +574 od : dict +575 Dict of JSON valid structures and objects that will be exported. +576 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. +577 All Obs inside a structure have to be defined on the same set of configurations. +578 fname : str +579 Filename of the output file. +580 description : str +581 Optional string that describes the contents of the json file. +582 indent : int +583 Specify the indentation level of the json file. None or 0 is permissible and +584 saves disk space. +585 reps : str +586 Specify the structure of the placeholder in exported dict to be reps[0-9]+. +587 gz : bool +588 If True, the output is a gzipped json. If False, the output is a json file. +589 """ +590 +591 if not isinstance(od, dict): +592 raise Exception('od has to be a dictionary. Did you want to use dump_to_json?') +593 +594 infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. ' +595 'OBSDICT contains the dictionary, where Obs or other structures have been replaced by ' +596 '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. ' +597 'This file may be parsed to a dict with the pyerrors routine load_json_dict.') +598 +599 desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description} +600 ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps) 601 -602 -603def _od_from_list_and_dict(ol, ind, reps='DICTOBS'): -604 """Parse a list of Obs or structures containing Obs and an accompanying -605 dict, where the structures have been replaced by placeholders to a -606 dict that contains the structures. -607 -608 The following structures are supported: Obs, list, numpy.ndarray, Corr +602 dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz) +603 +604 +605def _od_from_list_and_dict(ol, ind, reps='DICTOBS'): +606 """Parse a list of Obs or structures containing Obs and an accompanying +607 dict, where the structures have been replaced by placeholders to a +608 dict that contains the structures. 609 -610 Parameters -611 ---------- -612 ol : list -613 List of objects - -614 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. -615 All Obs inside a structure have to be defined on the same set of configurations. -616 ind : dict -617 Dict that defines the structure of the resulting dict and contains placeholders -618 reps : str -619 Specify the structure of the placeholder in imported dict to be reps[0-9]+. -620 """ -621 if not reps.isalnum(): -622 raise Exception('Placeholder string has to be alphanumeric!') -623 -624 counter = 0 +610 The following structures are supported: Obs, list, numpy.ndarray, Corr +611 +612 Parameters +613 ---------- +614 ol : list +615 List of objects - +616 At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr. +617 All Obs inside a structure have to be defined on the same set of configurations. +618 ind : dict +619 Dict that defines the structure of the resulting dict and contains placeholders +620 reps : str +621 Specify the structure of the placeholder in imported dict to be reps[0-9]+. +622 """ +623 if not reps.isalnum(): +624 raise Exception('Placeholder string has to be alphanumeric!') 625 -626 def dict_replace_string(d): -627 nonlocal counter -628 nonlocal ol -629 x = {} -630 for k, v in d.items(): -631 if isinstance(v, dict): -632 v = dict_replace_string(v) -633 elif isinstance(v, list): -634 v = list_replace_string(v) -635 elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)): -636 index = int(v[len(reps):]) -637 v = ol[index] -638 counter += 1 -639 x[k] = v -640 return x -641 -642 def list_replace_string(li): -643 nonlocal counter -644 nonlocal ol -645 x = [] -646 for e in li: -647 if isinstance(e, list): -648 e = list_replace_string(e) -649 elif isinstance(e, dict): -650 e = dict_replace_string(e) -651 elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)): -652 index = int(e[len(reps):]) -653 e = ol[index] -654 counter += 1 -655 x.append(e) -656 return x -657 -658 nd = dict_replace_string(ind) +626 counter = 0 +627 +628 def dict_replace_string(d): +629 nonlocal counter +630 nonlocal ol +631 x = {} +632 for k, v in d.items(): +633 if isinstance(v, dict): +634 v = dict_replace_string(v) +635 elif isinstance(v, list): +636 v = list_replace_string(v) +637 elif isinstance(v, str) and bool(re.match(r'%s[0-9]+' % (reps), v)): +638 index = int(v[len(reps):]) +639 v = ol[index] +640 counter += 1 +641 x[k] = v +642 return x +643 +644 def list_replace_string(li): +645 nonlocal counter +646 nonlocal ol +647 x = [] +648 for e in li: +649 if isinstance(e, list): +650 e = list_replace_string(e) +651 elif isinstance(e, dict): +652 e = dict_replace_string(e) +653 elif isinstance(e, str) and bool(re.match(r'%s[0-9]+' % (reps), e)): +654 index = int(e[len(reps):]) +655 e = ol[index] +656 counter += 1 +657 x.append(e) +658 return x 659 -660 if counter == 0: -661 raise Exception('No placeholder has been replaced! Check if reps is set correctly.') -662 -663 return nd +660 nd = dict_replace_string(ind) +661 +662 if counter == 0: +663 raise Exception('No placeholder has been replaced! Check if reps is set correctly.') 664 -665 -666def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): -667 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. -668 -669 The following structures are supported: Obs, list, numpy.ndarray, Corr +665 return nd +666 +667 +668def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'): +669 """Import a dict of Obs or structures containing Obs from a .json(.gz) file. 670 -671 Parameters -672 ---------- -673 fname : str -674 Filename of the input file. -675 verbose : bool -676 Print additional information that was written to the file. -677 gz : bool -678 If True, assumes that data is gzipped. If False, assumes JSON file. -679 full_output : bool -680 If True, a dict containing auxiliary information and the data is returned. -681 If False, only the data is returned. -682 reps : str -683 Specify the structure of the placeholder in imported dict to be reps[0-9]+. -684 """ -685 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) -686 description = indata['description']['description'] -687 indict = indata['description']['OBSDICT'] -688 ol = indata['obsdata'] -689 od = _od_from_list_and_dict(ol, indict, reps=reps) -690 -691 if full_output: -692 indata['description'] = description -693 indata['obsdata'] = od -694 return indata -695 else: -696 return od +671 The following structures are supported: Obs, list, numpy.ndarray, Corr +672 +673 Parameters +674 ---------- +675 fname : str +676 Filename of the input file. +677 verbose : bool +678 Print additional information that was written to the file. +679 gz : bool +680 If True, assumes that data is gzipped. If False, assumes JSON file. +681 full_output : bool +682 If True, a dict containing auxiliary information and the data is returned. +683 If False, only the data is returned. +684 reps : str +685 Specify the structure of the placeholder in imported dict to be reps[0-9]+. +686 """ +687 indata = load_json(fname, verbose=verbose, gz=gz, full_output=True) +688 description = indata['description']['description'] +689 indict = indata['description']['OBSDICT'] +690 ol = indata['obsdata'] +691 od = _od_from_list_and_dict(ol, indict, reps=reps) +692 +693 if full_output: +694 indata['description'] = description +695 indata['obsdata'] = od +696 return indata +697 else: +698 return od @@ -816,161 +818,163 @@ 34 35 def _gen_data_d_from_list(ol): 36 dl = [] - 37 for name in ol[0].mc_names: - 38 ed = {} - 39 ed['id'] = name - 40 ed['replica'] = [] - 41 for r_name in ol[0].e_content[name]: - 42 rd = {} - 43 rd['name'] = r_name - 44 if ol[0].is_merged.get(r_name, False): - 45 rd['is_merged'] = True - 46 rd['deltas'] = [] - 47 for i in range(len(ol[0].idl[r_name])): - 48 rd['deltas'].append([ol[0].idl[r_name][i]]) - 49 for o in ol: - 50 rd['deltas'][-1].append(o.deltas[r_name][i]) - 51 ed['replica'].append(rd) - 52 dl.append(ed) - 53 return dl - 54 - 55 def _gen_cdata_d_from_list(ol): - 56 dl = [] - 57 for name in ol[0].cov_names: - 58 ed = {} - 59 ed['id'] = name - 60 ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') - 61 ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) - 62 ncov = ol[0].covobs[name].cov.shape[0] - 63 ed['grad'] = [] - 64 for i in range(ncov): - 65 ed['grad'].append([]) - 66 for o in ol: - 67 ed['grad'][-1].append(o.covobs[name].grad[i][0]) - 68 dl.append(ed) - 69 return dl - 70 - 71 def write_Obs_to_dict(o): - 72 d = {} - 73 d['type'] = 'Obs' - 74 d['layout'] = '1' - 75 if o.tag: - 76 d['tag'] = [o.tag] - 77 if o.reweighted: - 78 d['reweighted'] = o.reweighted - 79 d['value'] = [o.value] - 80 data = _gen_data_d_from_list([o]) - 81 if len(data) > 0: - 82 d['data'] = data - 83 cdata = _gen_cdata_d_from_list([o]) - 84 if len(cdata) > 0: - 85 d['cdata'] = cdata - 86 return d - 87 - 88 def write_List_to_dict(ol): - 89 _assert_equal_properties(ol) - 90 d = {} - 91 d['type'] = 'List' - 92 d['layout'] = '%d' % len(ol) - 93 taglist = [o.tag for o in ol] - 94 if np.any([tag is not None for tag in taglist]): - 95 d['tag'] = taglist - 96 if ol[0].reweighted: - 97 d['reweighted'] = ol[0].reweighted - 98 d['value'] = [o.value for o in ol] - 99 data = _gen_data_d_from_list(ol) -100 if len(data) > 0: -101 d['data'] = data -102 cdata = _gen_cdata_d_from_list(ol) -103 if len(cdata) > 0: -104 d['cdata'] = cdata -105 return d -106 -107 def write_Array_to_dict(oa): -108 ol = np.ravel(oa) -109 _assert_equal_properties(ol) -110 d = {} -111 d['type'] = 'Array' -112 d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') -113 taglist = [o.tag for o in ol] -114 if np.any([tag is not None for tag in taglist]): -115 d['tag'] = taglist -116 if ol[0].reweighted: -117 d['reweighted'] = ol[0].reweighted -118 d['value'] = [o.value for o in ol] -119 data = _gen_data_d_from_list(ol) -120 if len(data) > 0: -121 d['data'] = data -122 cdata = _gen_cdata_d_from_list(ol) -123 if len(cdata) > 0: -124 d['cdata'] = cdata -125 return d -126 -127 def _nan_Obs_like(obs): -128 samples = [] -129 names = [] -130 idl = [] -131 for key, value in obs.idl.items(): -132 samples.append([np.nan] * len(value)) -133 names.append(key) -134 idl.append(value) -135 my_obs = Obs(samples, names, idl) -136 my_obs._covobs = obs._covobs -137 for name in obs._covobs: -138 my_obs.names.append(name) -139 my_obs.reweighted = obs.reweighted -140 my_obs.is_merged = obs.is_merged -141 return my_obs -142 -143 def write_Corr_to_dict(my_corr): -144 first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j)) -145 dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object) -146 dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0]) -147 content = [o if o is not None else dummy_array for o in my_corr.content] -148 dat = write_Array_to_dict(np.array(content, dtype=object)) -149 dat['type'] = 'Corr' -150 corr_meta_data = str(my_corr.tag) -151 if 'tag' in dat.keys(): -152 dat['tag'].append(corr_meta_data) -153 else: -154 dat['tag'] = [corr_meta_data] -155 taglist = dat['tag'] -156 dat['tag'] = {} # tag is now a dictionary, that contains the previous taglist in the key "tag" -157 dat['tag']['tag'] = taglist -158 if my_corr.prange is not None: -159 dat['tag']['prange'] = my_corr.prange -160 return dat -161 -162 if not isinstance(ol, list): -163 ol = [ol] -164 -165 d = {} -166 d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__) -167 d['version'] = '1.0' -168 d['who'] = getpass.getuser() -169 d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z') -170 d['host'] = socket.gethostname() + ', ' + platform.platform() -171 -172 if description: -173 d['description'] = description -174 -175 d['obsdata'] = [] -176 for io in ol: -177 if isinstance(io, Obs): -178 d['obsdata'].append(write_Obs_to_dict(io)) -179 elif isinstance(io, list): -180 d['obsdata'].append(write_List_to_dict(io)) -181 elif isinstance(io, np.ndarray): -182 d['obsdata'].append(write_Array_to_dict(io)) -183 elif isinstance(io, Corr): -184 d['obsdata'].append(write_Corr_to_dict(io)) -185 else: -186 raise Exception("Unkown datatype.") -187 -188 if indent: -189 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY) -190 else: -191 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT) + 37 No = len(ol) + 38 for name in ol[0].mc_names: + 39 ed = {} + 40 ed['id'] = name + 41 ed['replica'] = [] + 42 for r_name in ol[0].e_content[name]: + 43 rd = {} + 44 rd['name'] = r_name + 45 if ol[0].is_merged.get(r_name, False): + 46 rd['is_merged'] = True + 47 rd['deltas'] = [] + 48 offsets = [o.r_values[r_name] - o.value for o in ol] + 49 deltas = np.column_stack([ol[oi].deltas[r_name] + offsets[oi] for oi in range(No)]) + 50 for i in range(len(ol[0].idl[r_name])): + 51 rd['deltas'].append([ol[0].idl[r_name][i]]) + 52 rd['deltas'][-1] += deltas[i].tolist() + 53 ed['replica'].append(rd) + 54 dl.append(ed) + 55 return dl + 56 + 57 def _gen_cdata_d_from_list(ol): + 58 dl = [] + 59 for name in ol[0].cov_names: + 60 ed = {} + 61 ed['id'] = name + 62 ed['layout'] = str(ol[0].covobs[name].cov.shape).lstrip('(').rstrip(')').rstrip(',') + 63 ed['cov'] = list(np.ravel(ol[0].covobs[name].cov)) + 64 ncov = ol[0].covobs[name].cov.shape[0] + 65 ed['grad'] = [] + 66 for i in range(ncov): + 67 ed['grad'].append([]) + 68 for o in ol: + 69 ed['grad'][-1].append(o.covobs[name].grad[i][0]) + 70 dl.append(ed) + 71 return dl + 72 + 73 def write_Obs_to_dict(o): + 74 d = {} + 75 d['type'] = 'Obs' + 76 d['layout'] = '1' + 77 if o.tag: + 78 d['tag'] = [o.tag] + 79 if o.reweighted: + 80 d['reweighted'] = o.reweighted + 81 d['value'] = [o.value] + 82 data = _gen_data_d_from_list([o]) + 83 if len(data) > 0: + 84 d['data'] = data + 85 cdata = _gen_cdata_d_from_list([o]) + 86 if len(cdata) > 0: + 87 d['cdata'] = cdata + 88 return d + 89 + 90 def write_List_to_dict(ol): + 91 _assert_equal_properties(ol) + 92 d = {} + 93 d['type'] = 'List' + 94 d['layout'] = '%d' % len(ol) + 95 taglist = [o.tag for o in ol] + 96 if np.any([tag is not None for tag in taglist]): + 97 d['tag'] = taglist + 98 if ol[0].reweighted: + 99 d['reweighted'] = ol[0].reweighted +100 d['value'] = [o.value for o in ol] +101 data = _gen_data_d_from_list(ol) +102 if len(data) > 0: +103 d['data'] = data +104 cdata = _gen_cdata_d_from_list(ol) +105 if len(cdata) > 0: +106 d['cdata'] = cdata +107 return d +108 +109 def write_Array_to_dict(oa): +110 ol = np.ravel(oa) +111 _assert_equal_properties(ol) +112 d = {} +113 d['type'] = 'Array' +114 d['layout'] = str(oa.shape).lstrip('(').rstrip(')').rstrip(',') +115 taglist = [o.tag for o in ol] +116 if np.any([tag is not None for tag in taglist]): +117 d['tag'] = taglist +118 if ol[0].reweighted: +119 d['reweighted'] = ol[0].reweighted +120 d['value'] = [o.value for o in ol] +121 data = _gen_data_d_from_list(ol) +122 if len(data) > 0: +123 d['data'] = data +124 cdata = _gen_cdata_d_from_list(ol) +125 if len(cdata) > 0: +126 d['cdata'] = cdata +127 return d +128 +129 def _nan_Obs_like(obs): +130 samples = [] +131 names = [] +132 idl = [] +133 for key, value in obs.idl.items(): +134 samples.append([np.nan] * len(value)) +135 names.append(key) +136 idl.append(value) +137 my_obs = Obs(samples, names, idl) +138 my_obs._covobs = obs._covobs +139 for name in obs._covobs: +140 my_obs.names.append(name) +141 my_obs.reweighted = obs.reweighted +142 my_obs.is_merged = obs.is_merged +143 return my_obs +144 +145 def write_Corr_to_dict(my_corr): +146 first_not_none = next(i for i, j in enumerate(my_corr.content) if np.all(j)) +147 dummy_array = np.empty((my_corr.N, my_corr.N), dtype=object) +148 dummy_array[:] = _nan_Obs_like(my_corr.content[first_not_none].ravel()[0]) +149 content = [o if o is not None else dummy_array for o in my_corr.content] +150 dat = write_Array_to_dict(np.array(content, dtype=object)) +151 dat['type'] = 'Corr' +152 corr_meta_data = str(my_corr.tag) +153 if 'tag' in dat.keys(): +154 dat['tag'].append(corr_meta_data) +155 else: +156 dat['tag'] = [corr_meta_data] +157 taglist = dat['tag'] +158 dat['tag'] = {} # tag is now a dictionary, that contains the previous taglist in the key "tag" +159 dat['tag']['tag'] = taglist +160 if my_corr.prange is not None: +161 dat['tag']['prange'] = my_corr.prange +162 return dat +163 +164 if not isinstance(ol, list): +165 ol = [ol] +166 +167 d = {} +168 d['program'] = 'pyerrors %s' % (pyerrorsversion.__version__) +169 d['version'] = '1.0' +170 d['who'] = getpass.getuser() +171 d['date'] = datetime.datetime.now().astimezone().strftime('%Y-%m-%d %H:%M:%S %z') +172 d['host'] = socket.gethostname() + ', ' + platform.platform() +173 +174 if description: +175 d['description'] = description +176 +177 d['obsdata'] = [] +178 for io in ol: +179 if isinstance(io, Obs): +180 d['obsdata'].append(write_Obs_to_dict(io)) +181 elif isinstance(io, list): +182 d['obsdata'].append(write_List_to_dict(io)) +183 elif isinstance(io, np.ndarray): +184 d['obsdata'].append(write_Array_to_dict(io)) +185 elif isinstance(io, Corr): +186 d['obsdata'].append(write_Corr_to_dict(io)) +187 else: +188 raise Exception("Unkown datatype.") +189 +190 if indent: +191 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_SINGLE_LINE_ARRAY) +192 else: +193 return json.dumps(d, indent=indent, ensure_ascii=False, write_mode=json.WM_COMPACT) @@ -1005,41 +1009,41 @@ saves disk space. - 
194def dump_to_json(ol, fname, description='', indent=1, gz=True):
-195    """Export a list of Obs or structures containing Obs to a .json(.gz) file
-196
-197    Parameters
-198    ----------
-199    ol : list
-200        List of objects that will be exported. At the moment, these objects can be
-201        either of: Obs, list, numpy.ndarray, Corr.
-202        All Obs inside a structure have to be defined on the same set of configurations.
-203    fname : str
-204        Filename of the output file.
-205    description : str
-206        Optional string that describes the contents of the json file.
-207    indent : int
-208        Specify the indentation level of the json file. None or 0 is permissible and
-209        saves disk space.
-210    gz : bool
-211        If True, the output is a gzipped json. If False, the output is a json file.
-212    """
-213
-214    jsonstring = create_json_string(ol, description, indent)
+            
196def dump_to_json(ol, fname, description='', indent=1, gz=True):
+197    """Export a list of Obs or structures containing Obs to a .json(.gz) file
+198
+199    Parameters
+200    ----------
+201    ol : list
+202        List of objects that will be exported. At the moment, these objects can be
+203        either of: Obs, list, numpy.ndarray, Corr.
+204        All Obs inside a structure have to be defined on the same set of configurations.
+205    fname : str
+206        Filename of the output file.
+207    description : str
+208        Optional string that describes the contents of the json file.
+209    indent : int
+210        Specify the indentation level of the json file. None or 0 is permissible and
+211        saves disk space.
+212    gz : bool
+213        If True, the output is a gzipped json. If False, the output is a json file.
+214    """
 215
-216    if not fname.endswith('.json') and not fname.endswith('.gz'):
-217        fname += '.json'
-218
-219    if gz:
-220        if not fname.endswith('.gz'):
-221            fname += '.gz'
-222
-223        fp = gzip.open(fname, 'wb')
-224        fp.write(jsonstring.encode('utf-8'))
-225    else:
-226        fp = open(fname, 'w', encoding='utf-8')
-227        fp.write(jsonstring)
-228    fp.close()
+216    jsonstring = create_json_string(ol, description, indent)
+217
+218    if not fname.endswith('.json') and not fname.endswith('.gz'):
+219        fname += '.json'
+220
+221    if gz:
+222        if not fname.endswith('.gz'):
+223            fname += '.gz'
+224
+225        fp = gzip.open(fname, 'wb')
+226        fp.write(jsonstring.encode('utf-8'))
+227    else:
+228        fp = open(fname, 'w', encoding='utf-8')
+229        fp.write(jsonstring)
+230    fp.close()
 

 
 
@@ -1077,24 +1081,24 @@ If True, the output is a gzipped json. If False, the output is a json file.
- 
434def import_json_string(json_string, verbose=True, full_output=False):
-435    """Reconstruct a list of Obs or structures containing Obs from a json string.
-436
-437    The following structures are supported: Obs, list, numpy.ndarray, Corr
-438    If the list contains only one element, it is unpacked from the list.
-439
-440    Parameters
-441    ----------
-442    json_string : str
-443        json string containing the data.
-444    verbose : bool
-445        Print additional information that was written to the file.
-446    full_output : bool
-447        If True, a dict containing auxiliary information and the data is returned.
-448        If False, only the data is returned.
-449    """
-450
-451    return _parse_json_dict(json.loads(json_string), verbose, full_output)
+            
436def import_json_string(json_string, verbose=True, full_output=False):
+437    """Reconstruct a list of Obs or structures containing Obs from a json string.
+438
+439    The following structures are supported: Obs, list, numpy.ndarray, Corr
+440    If the list contains only one element, it is unpacked from the list.
+441
+442    Parameters
+443    ----------
+444    json_string : str
+445        json string containing the data.
+446    verbose : bool
+447        Print additional information that was written to the file.
+448    full_output : bool
+449        If True, a dict containing auxiliary information and the data is returned.
+450        If False, only the data is returned.
+451    """
+452
+453    return _parse_json_dict(json.loads(json_string), verbose, full_output)
 

 
 
@@ -1129,38 +1133,38 @@ If False, only the data is returned.
- 
454def load_json(fname, verbose=True, gz=True, full_output=False):
-455    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
-456
-457    The following structures are supported: Obs, list, numpy.ndarray, Corr
-458    If the list contains only one element, it is unpacked from the list.
-459
-460    Parameters
-461    ----------
-462    fname : str
-463        Filename of the input file.
-464    verbose : bool
-465        Print additional information that was written to the file.
-466    gz : bool
-467        If True, assumes that data is gzipped. If False, assumes JSON file.
-468    full_output : bool
-469        If True, a dict containing auxiliary information and the data is returned.
-470        If False, only the data is returned.
-471    """
-472    if not fname.endswith('.json') and not fname.endswith('.gz'):
-473        fname += '.json'
-474    if gz:
-475        if not fname.endswith('.gz'):
-476            fname += '.gz'
-477        with gzip.open(fname, 'r') as fin:
-478            d = json.load(fin)
-479    else:
-480        if fname.endswith('.gz'):
-481            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
-482        with open(fname, 'r', encoding='utf-8') as fin:
-483            d = json.loads(fin.read())
-484
-485    return _parse_json_dict(d, verbose, full_output)
+            
456def load_json(fname, verbose=True, gz=True, full_output=False):
+457    """Import a list of Obs or structures containing Obs from a .json(.gz) file.
+458
+459    The following structures are supported: Obs, list, numpy.ndarray, Corr
+460    If the list contains only one element, it is unpacked from the list.
+461
+462    Parameters
+463    ----------
+464    fname : str
+465        Filename of the input file.
+466    verbose : bool
+467        Print additional information that was written to the file.
+468    gz : bool
+469        If True, assumes that data is gzipped. If False, assumes JSON file.
+470    full_output : bool
+471        If True, a dict containing auxiliary information and the data is returned.
+472        If False, only the data is returned.
+473    """
+474    if not fname.endswith('.json') and not fname.endswith('.gz'):
+475        fname += '.json'
+476    if gz:
+477        if not fname.endswith('.gz'):
+478            fname += '.gz'
+479        with gzip.open(fname, 'r') as fin:
+480            d = json.load(fin)
+481    else:
+482        if fname.endswith('.gz'):
+483            warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
+484        with open(fname, 'r', encoding='utf-8') as fin:
+485            d = json.loads(fin.read())
+486
+487    return _parse_json_dict(d, verbose, full_output)
 

 
 
@@ -1197,40 +1201,40 @@ If False, only the data is returned.
- 
568def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
-569    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
-570
-571    Parameters
-572    ----------
-573    od : dict
-574        Dict of JSON valid structures and objects that will be exported.
-575        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
-576        All Obs inside a structure have to be defined on the same set of configurations.
-577    fname : str
-578        Filename of the output file.
-579    description : str
-580        Optional string that describes the contents of the json file.
-581    indent : int
-582        Specify the indentation level of the json file. None or 0 is permissible and
-583        saves disk space.
-584    reps : str
-585        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
-586    gz : bool
-587        If True, the output is a gzipped json. If False, the output is a json file.
-588    """
-589
-590    if not isinstance(od, dict):
-591        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
-592
-593    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
-594                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
-595                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
-596                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
-597
-598    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
-599    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
-600
-601    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
+            
570def dump_dict_to_json(od, fname, description='', indent=1, reps='DICTOBS', gz=True):
+571    """Export a dict of Obs or structures containing Obs to a .json(.gz) file
+572
+573    Parameters
+574    ----------
+575    od : dict
+576        Dict of JSON valid structures and objects that will be exported.
+577        At the moment, these objects can be either of: Obs, list, numpy.ndarray, Corr.
+578        All Obs inside a structure have to be defined on the same set of configurations.
+579    fname : str
+580        Filename of the output file.
+581    description : str
+582        Optional string that describes the contents of the json file.
+583    indent : int
+584        Specify the indentation level of the json file. None or 0 is permissible and
+585        saves disk space.
+586    reps : str
+587        Specify the structure of the placeholder in exported dict to be reps[0-9]+.
+588    gz : bool
+589        If True, the output is a gzipped json. If False, the output is a json file.
+590    """
+591
+592    if not isinstance(od, dict):
+593        raise Exception('od has to be a dictionary. Did you want to use dump_to_json?')
+594
+595    infostring = ('This JSON file contains a python dictionary that has been parsed to a list of structures. '
+596                  'OBSDICT contains the dictionary, where Obs or other structures have been replaced by '
+597                  '' + reps + '[0-9]+. The field description contains the additional description of this JSON file. '
+598                  'This file may be parsed to a dict with the pyerrors routine load_json_dict.')
+599
+600    desc_dict = {'INFO': infostring, 'OBSDICT': {}, 'description': description}
+601    ol, desc_dict['OBSDICT'] = _ol_from_dict(od, reps=reps)
+602
+603    dump_to_json(ol, fname, description=desc_dict, indent=indent, gz=gz)
 

 
 
@@ -1270,37 +1274,37 @@ If True, the output is a gzipped json. If False, the output is a json file.
- 
667def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
-668    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
-669
-670    The following structures are supported: Obs, list, numpy.ndarray, Corr
+            
669def load_json_dict(fname, verbose=True, gz=True, full_output=False, reps='DICTOBS'):
+670    """Import a dict of Obs or structures containing Obs from a .json(.gz) file.
 671
-672    Parameters
-673    ----------
-674    fname : str
-675        Filename of the input file.
-676    verbose : bool
-677        Print additional information that was written to the file.
-678    gz : bool
-679        If True, assumes that data is gzipped. If False, assumes JSON file.
-680    full_output : bool
-681        If True, a dict containing auxiliary information and the data is returned.
-682        If False, only the data is returned.
-683    reps : str
-684        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
-685    """
-686    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
-687    description = indata['description']['description']
-688    indict = indata['description']['OBSDICT']
-689    ol = indata['obsdata']
-690    od = _od_from_list_and_dict(ol, indict, reps=reps)
-691
-692    if full_output:
-693        indata['description'] = description
-694        indata['obsdata'] = od
-695        return indata
-696    else:
-697        return od
+672    The following structures are supported: Obs, list, numpy.ndarray, Corr
+673
+674    Parameters
+675    ----------
+676    fname : str
+677        Filename of the input file.
+678    verbose : bool
+679        Print additional information that was written to the file.
+680    gz : bool
+681        If True, assumes that data is gzipped. If False, assumes JSON file.
+682    full_output : bool
+683        If True, a dict containing auxiliary information and the data is returned.
+684        If False, only the data is returned.
+685    reps : str
+686        Specify the structure of the placeholder in imported dict to be reps[0-9]+.
+687    """
+688    indata = load_json(fname, verbose=verbose, gz=gz, full_output=True)
+689    description = indata['description']['description']
+690    indict = indata['description']['OBSDICT']
+691    ol = indata['obsdata']
+692    od = _od_from_list_and_dict(ol, indict, reps=reps)
+693
+694    if full_output:
+695        indata['description'] = description
+696        indata['obsdata'] = od
+697        return indata
+698    else:
+699        return od