Slightly better Typechecking when exporting to SQL (#174)

* corret type clause

* add tests, changes in create_json_string

* create json-string now gives back None

* revert changes

* fix panda sql export

* add SQL test

* fixed None type export for csv and sql.gz

* move None parsing to json io

* alter regex

* revert changes

* only replace None with empty str when necessary

* fixed deserialze_df for python 3.7

* add more tesets

* fix case where gz was ignored

* hand voer gz explicitly

* replace nan  by None in non-Obs columns

* moved warning to csv export, mroe tests

* only values able to be nan are put in np.isnan()

* added python float for warning
This commit is contained in:
Justus Kuhlmann 2023-05-18 18:11:52 +02:00 committed by GitHub
parent b75aa741a9
commit a5b6f69160
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 198 additions and 24 deletions

View file

@ -479,7 +479,6 @@ def import_json_string(json_string, verbose=True, full_output=False):
result : dict
if full_output=True
"""
return _parse_json_dict(json.loads(json_string), verbose, full_output)

View file

@ -5,6 +5,7 @@ import pandas as pd
from ..obs import Obs
from ..correlators import Corr
from .json import create_json_string, import_json_string
import numpy as np
def to_sql(df, table_name, db, if_exists='fail', gz=True, **kwargs):
@ -76,6 +77,13 @@ def dump_df(df, fname, gz=True):
-------
None
"""
for column in df:
serialize = _need_to_serialize(df[column])
if not serialize:
if all(isinstance(entry, (int, np.integer, float, np.floating)) for entry in df[column]):
if any([np.isnan(entry) for entry in df[column]]):
warnings.warn("nan value in column " + column + " will be replaced by None", UserWarning)
out = _serialize_df(df, gz=False)
if not fname.endswith('.csv'):
@ -114,11 +122,11 @@ def load_df(fname, auto_gamma=False, gz=True):
if not fname.endswith('.gz'):
fname += '.gz'
with gzip.open(fname) as f:
re_import = pd.read_csv(f)
re_import = pd.read_csv(f, keep_default_na=False)
else:
if fname.endswith('.gz'):
warnings.warn("Trying to read from %s without unzipping!" % fname, UserWarning)
re_import = pd.read_csv(fname)
re_import = pd.read_csv(fname, keep_default_na=False)
return _deserialize_df(re_import, auto_gamma=auto_gamma)
@ -135,17 +143,12 @@ def _serialize_df(df, gz=False):
"""
out = df.copy()
for column in out:
serialize = False
if isinstance(out[column][0], (Obs, Corr)):
serialize = True
elif isinstance(out[column][0], list):
if all(isinstance(o, Obs) for o in out[column][0]):
serialize = True
serialize = _need_to_serialize(out[column])
if serialize is True:
out[column] = out[column].transform(lambda x: create_json_string(x, indent=0))
out[column] = out[column].transform(lambda x: create_json_string(x, indent=0) if x is not None else None)
if gz is True:
out[column] = out[column].transform(lambda x: gzip.compress(x.encode('utf-8')))
out[column] = out[column].transform(lambda x: gzip.compress((x if x is not None else '').encode('utf-8')))
return out
@ -168,12 +171,29 @@ def _deserialize_df(df, auto_gamma=False):
if isinstance(df[column][0], bytes):
if df[column][0].startswith(b"\x1f\x8b\x08\x00"):
df[column] = df[column].transform(lambda x: gzip.decompress(x).decode('utf-8'))
if isinstance(df[column][0], str):
if '"program":' in df[column][0][:20]:
df[column] = df[column].transform(lambda x: import_json_string(x, verbose=False))
df = df.replace({r'^$': None}, regex=True)
i = 0
while df[column][i] is None:
i += 1
if isinstance(df[column][i], str):
if '"program":' in df[column][i][:20]:
df[column] = df[column].transform(lambda x: import_json_string(x, verbose=False) if x is not None else None)
if auto_gamma is True:
if isinstance(df[column][0], list):
df[column].apply(lambda x: [o.gm() for o in x])
if isinstance(df[column][i], list):
df[column].apply(lambda x: [o.gm() if o is not None else x for o in x])
else:
df[column].apply(lambda x: x.gamma_method())
df[column].apply(lambda x: x.gm() if x is not None else x)
return df
def _need_to_serialize(col):
serialize = False
i = 0
while col[i] is None:
i += 1
if isinstance(col[i], (Obs, Corr)):
serialize = True
elif isinstance(col[i], list):
if all(isinstance(o, Obs) for o in col[i]):
serialize = True
return serialize

View file

@ -2,12 +2,14 @@ import numpy as np
import pandas as pd
import pyerrors as pe
import pytest
import warnings
def test_df_export_import(tmp_path):
my_dict = {"int": 1,
"float": -0.01,
"Obs1": pe.pseudo_Obs(87, 21, "test_ensemble"),
"Obs2": pe.pseudo_Obs(-87, 21, "test_ensemble2")}
"float": -0.01,
"Obs1": pe.pseudo_Obs(87, 21, "test_ensemble"),
"Obs2": pe.pseudo_Obs(-87, 21, "test_ensemble2")}
for gz in [True, False]:
my_df = pd.DataFrame([my_dict] * 10)
@ -18,13 +20,166 @@ def test_df_export_import(tmp_path):
pe.input.pandas.load_df((tmp_path / 'df_output.csv').as_posix(), gz=gz)
def test_null_first_line_df_export_import(tmp_path):
my_dict = {"int": 1,
"float": -0.01,
"Obs1": pe.pseudo_Obs(87, 21, "test_ensemble"),
"Obs2": pe.pseudo_Obs(-87, 21, "test_ensemble2")}
my_df = pd.DataFrame([my_dict] * 4)
my_df.loc[0, "Obs1"] = None
my_df.loc[2, "Obs1"] = None
for gz in [True, False]:
pe.input.pandas.dump_df(my_df, (tmp_path / 'df_output').as_posix(), gz=gz)
reconstructed_df = pe.input.pandas.load_df((tmp_path / 'df_output').as_posix(), auto_gamma=True, gz=gz)
assert reconstructed_df.loc[0, "Obs1"] is None
assert reconstructed_df.loc[2, "Obs1"] is None
assert np.all(reconstructed_df.loc[1] == my_df.loc[1])
assert np.all(reconstructed_df.loc[3] == my_df.loc[3])
def test_nan_df_export_import(tmp_path):
my_dict = {"int": 1,
"float": -0.01,
"Obs1": pe.pseudo_Obs(87, 21, "test_ensemble"),
"Obs2": pe.pseudo_Obs(-87, 21, "test_ensemble2")}
my_df = pd.DataFrame([my_dict] * 4)
my_df.loc[1, "int"] = np.nan
for gz in [True, False]:
pe.input.pandas.dump_df(my_df, (tmp_path / 'df_output').as_posix(), gz=gz)
reconstructed_df = pe.input.pandas.load_df((tmp_path / 'df_output').as_posix(), auto_gamma=True, gz=gz)
with pytest.warns(UserWarning, match="nan value in column int will be replaced by None"):
warnings.warn("nan value in column int will be replaced by None", UserWarning)
assert reconstructed_df.loc[1, "int"] is None
assert np.all(reconstructed_df.loc[:, "float"] == my_df.loc[:, "float"])
assert np.all(reconstructed_df.loc[:, "Obs1"] == my_df.loc[:, "Obs1"])
assert np.all(reconstructed_df.loc[:, "Obs2"] == my_df.loc[:, "Obs2"])
def test_null_second_line_df_export_import(tmp_path):
my_dict = {"int": 1,
"float": -0.01,
"Obs1": pe.pseudo_Obs(87, 21, "test_ensemble"),
"Obs2": pe.pseudo_Obs(-87, 21, "test_ensemble2")}
my_df = pd.DataFrame([my_dict] * 4)
my_df.loc[1, "Obs1"] = None
for gz in [True, False]:
pe.input.pandas.dump_df(my_df, (tmp_path / 'df_output').as_posix(), gz=gz)
reconstructed_df = pe.input.pandas.load_df((tmp_path / 'df_output').as_posix(), auto_gamma=True, gz=gz)
assert reconstructed_df.loc[1, "Obs1"] is None
assert np.all(reconstructed_df.loc[0] == my_df.loc[0])
assert np.all(reconstructed_df.loc[2:] == my_df.loc[2:])
def test_null_first_line_df_gzsql_export_import(tmp_path):
my_dict = {"int": 1,
"float": -0.01,
"Obs1": pe.pseudo_Obs(87, 21, "test_ensemble"),
"Obs2": pe.pseudo_Obs(-87, 21, "test_ensemble2")}
my_df = pd.DataFrame([my_dict] * 4)
my_df.loc[0, "Obs1"] = None
my_df.loc[2, "Obs1"] = None
gz = True
pe.input.pandas.to_sql(my_df, 'test', (tmp_path / 'test.db').as_posix(), gz=gz)
reconstructed_df = pe.input.pandas.read_sql('SELECT * FROM test', (tmp_path / 'test.db').as_posix(), auto_gamma=True)
assert reconstructed_df.loc[0, "Obs1"] is None
assert reconstructed_df.loc[2, "Obs1"] is None
assert np.all(reconstructed_df.loc[1] == my_df.loc[1])
assert np.all(reconstructed_df.loc[3] == my_df.loc[3])
def test_null_second_line_df_gzsql_export_import(tmp_path):
my_dict = {"int": 1,
"float": -0.01,
"Obs1": pe.pseudo_Obs(87, 21, "test_ensemble"),
"Obs2": pe.pseudo_Obs(-87, 21, "test_ensemble2")}
my_df = pd.DataFrame([my_dict] * 4)
my_df.loc[1, "Obs1"] = None
gz = True
pe.input.pandas.to_sql(my_df, 'test', (tmp_path / 'test.db').as_posix(), gz=gz)
reconstructed_df = pe.input.pandas.read_sql('SELECT * FROM test', (tmp_path / 'test.db').as_posix(), auto_gamma=True)
assert reconstructed_df.loc[1, "Obs1"] is None
assert np.all(reconstructed_df.loc[0] == my_df.loc[0])
assert np.all(reconstructed_df.loc[2:] == my_df.loc[2:])
def test_null_first_line_df_sql_export_import(tmp_path):
my_dict = {"int": 1,
"float": -0.01,
"Obs1": pe.pseudo_Obs(87, 21, "test_ensemble"),
"Obs2": pe.pseudo_Obs(-87, 21, "test_ensemble2")}
my_df = pd.DataFrame([my_dict] * 4)
my_df.loc[0, "Obs1"] = None
my_df.loc[2, "Obs1"] = None
gz = False
pe.input.pandas.to_sql(my_df, 'test', (tmp_path / 'test.db').as_posix(), gz=gz)
reconstructed_df = pe.input.pandas.read_sql('SELECT * FROM test', (tmp_path / 'test.db').as_posix(), auto_gamma=True)
assert reconstructed_df.loc[0, "Obs1"] is None
assert reconstructed_df.loc[2, "Obs1"] is None
assert np.all(reconstructed_df.loc[1] == my_df.loc[1])
assert np.all(reconstructed_df.loc[3] == my_df.loc[3])
def test_nan_sql_export_import(tmp_path):
my_dict = {"int": 1,
"float": -0.01,
"Obs1": pe.pseudo_Obs(87, 21, "test_ensemble"),
"Obs2": pe.pseudo_Obs(-87, 21, "test_ensemble2")}
my_df = pd.DataFrame([my_dict] * 4)
my_df.loc[1, "int"] = np.nan
gz = False
pe.input.pandas.to_sql(my_df, 'test', (tmp_path / 'test.db').as_posix(), gz=gz)
reconstructed_df = pe.input.pandas.read_sql('SELECT * FROM test', (tmp_path / 'test.db').as_posix(), auto_gamma=True)
with pytest.warns(UserWarning, match="nan value in column int will be replaced by None"):
warnings.warn("nan value in column int will be replaced by None", UserWarning)
assert np.isnan(reconstructed_df.loc[1, "int"])
assert np.all(reconstructed_df.loc[:, "float"] == my_df.loc[:, "float"])
assert np.all(reconstructed_df.loc[:, "Obs1"] == my_df.loc[:, "Obs1"])
assert np.all(reconstructed_df.loc[:, "Obs2"] == my_df.loc[:, "Obs2"])
def test_nan_gzsql_export_import(tmp_path):
my_dict = {"int": 1,
"float": -0.01,
"Obs1": pe.pseudo_Obs(87, 21, "test_ensemble"),
"Obs2": pe.pseudo_Obs(-87, 21, "test_ensemble2")}
my_df = pd.DataFrame([my_dict] * 4)
my_df.loc[1, "int"] = np.nan
gz = True
pe.input.pandas.to_sql(my_df, 'test', (tmp_path / 'test.db').as_posix(), gz=gz)
reconstructed_df = pe.input.pandas.read_sql('SELECT * FROM test', (tmp_path / 'test.db').as_posix(), auto_gamma=True)
assert np.isnan(reconstructed_df.loc[1, "int"])
assert np.all(reconstructed_df.loc[:, "float"] == my_df.loc[:, "float"])
assert np.all(reconstructed_df.loc[:, "Obs1"] == my_df.loc[:, "Obs1"])
assert np.all(reconstructed_df.loc[:, "Obs2"] == my_df.loc[:, "Obs2"])
def test_null_second_line_df_sql_export_import(tmp_path):
my_dict = {"int": 1,
"float": -0.01,
"Obs1": pe.pseudo_Obs(87, 21, "test_ensemble"),
"Obs2": pe.pseudo_Obs(-87, 21, "test_ensemble2")}
my_df = pd.DataFrame([my_dict] * 4)
my_df.loc[1, "Obs1"] = None
gz = False
pe.input.pandas.to_sql(my_df, 'test', (tmp_path / 'test.db').as_posix(), gz=gz)
reconstructed_df = pe.input.pandas.read_sql('SELECT * FROM test', (tmp_path / 'test.db').as_posix(), auto_gamma=True)
assert reconstructed_df.loc[1, "Obs1"] is None
assert np.all(reconstructed_df.loc[0] == my_df.loc[0])
assert np.all(reconstructed_df.loc[2:] == my_df.loc[2:])
def test_df_Corr(tmp_path):
my_corr = pe.Corr([pe.pseudo_Obs(-0.48, 0.04, "test"), pe.pseudo_Obs(-0.154, 0.03, "test")])
my_dict = {"int": 1,
"float": -0.01,
"Corr": my_corr}
"float": -0.01,
"Corr": my_corr}
my_df = pd.DataFrame([my_dict] * 5)
pe.input.pandas.dump_df(my_df, (tmp_path / 'df_output').as_posix())
@ -76,8 +231,8 @@ def test_sql_if_exists_fail(tmp_path):
def test_Obs_list_sql(tmp_path):
my_dict = {"int": 1,
"Obs1": pe.pseudo_Obs(17, 11, "test_sql_if_exists_failnsemble"),
"Obs_list": [[pe.pseudo_Obs(0.0, 0.1, "test_ensemble2"), pe.pseudo_Obs(3.2, 1.1, "test_ensemble2")]]}
"Obs1": pe.pseudo_Obs(17, 11, "test_sql_if_exists_failnsemble"),
"Obs_list": [[pe.pseudo_Obs(0.0, 0.1, "test_ensemble2"), pe.pseudo_Obs(3.2, 1.1, "test_ensemble2")]]}
pe_df = pd.DataFrame(my_dict)
my_db = (tmp_path / "test_db.sqlite").as_posix()
pe.input.pandas.to_sql(pe_df, "My_table", my_db)