Commit 114a785c authored by Arnaud Campeas's avatar Arnaud Campeas
Browse files

tsio: add type checking during insert to produce human readable error

add numerical normalisation (integer to float) when reading Json

close #15
parent 47cd6a83f8bd
......@@ -640,8 +640,8 @@ def test_multi_index(engine):
assert_df("""
ts_multi_simple
a b c
2015-01-11 12:30:00 2015-01-01 2015-01-11 12:00:00 0
2015-01-02 2015-01-11 12:00:00 1
2015-01-11 12:30:00 2015-01-01 2015-01-11 12:00:00 0.0
2015-01-02 2015-01-11 12:00:00 1.0
""", pd.DataFrame(ts))
diff = tso.insert(engine, ts_multi, 'ts_multi_simple', 'test')
......@@ -656,8 +656,8 @@ a b c
assert_df("""
ts_multi_simple
a b c
2015-01-11 12:30:00 2015-01-01 2015-01-11 12:00:00 0
2015-01-02 2015-01-11 12:00:00 2
2015-01-11 12:30:00 2015-01-01 2015-01-11 12:00:00 0.0
2015-01-02 2015-01-11 12:00:00 2.0
""", pd.DataFrame(ts))
# bigger ts
......@@ -689,14 +689,14 @@ a b c
assert_df("""
ts_multi
a b c
2015-01-01 2015-01-11 12:30:00 2015-01-11 12:00:00 0
2015-01-21 12:30:00 2015-01-21 12:00:00 4
2015-01-02 2015-01-11 12:30:00 2015-01-11 12:00:00 1
2015-01-21 12:30:00 2015-01-21 12:00:00 5
2015-01-03 2015-01-11 12:30:00 2015-01-11 12:00:00 2
2015-01-21 12:30:00 2015-01-21 12:00:00 6
2015-01-04 2015-01-11 12:30:00 2015-01-11 12:00:00 3
2015-01-21 12:30:00 2015-01-21 12:00:00 7
2015-01-01 2015-01-11 12:30:00 2015-01-11 12:00:00 0.0
2015-01-21 12:30:00 2015-01-21 12:00:00 4.0
2015-01-02 2015-01-11 12:30:00 2015-01-11 12:00:00 1.0
2015-01-21 12:30:00 2015-01-21 12:00:00 5.0
2015-01-03 2015-01-11 12:30:00 2015-01-11 12:00:00 2.0
2015-01-21 12:30:00 2015-01-21 12:00:00 6.0
2015-01-04 2015-01-11 12:30:00 2015-01-11 12:00:00 3.0
2015-01-21 12:30:00 2015-01-21 12:00:00 7.0
""", pd.DataFrame(ts.sort_index()))
# Note: the columnns are returned according to the alphabetic order
......@@ -738,18 +738,18 @@ a b c
assert_df("""
ts_multi
a b c
2015-01-01 2015-01-11 12:30:00 2015-01-11 12:00:00 0
2015-01-21 12:30:00 2015-01-21 12:00:00 4
2015-01-31 12:30:00 2015-01-31 12:00:00 4
2015-01-02 2015-01-11 12:30:00 2015-01-11 12:00:00 1
2015-01-21 12:30:00 2015-01-21 12:00:00 4
2015-01-31 12:30:00 2015-01-31 12:00:00 4
2015-01-03 2015-01-11 12:30:00 2015-01-11 12:00:00 2
2015-01-21 12:30:00 2015-01-21 12:00:00 4
2015-01-31 12:30:00 2015-01-31 12:00:00 4
2015-01-04 2015-01-11 12:30:00 2015-01-11 12:00:00 3
2015-01-21 12:30:00 2015-01-21 12:00:00 4
2015-01-31 12:30:00 2015-01-31 12:00:00 4
2015-01-01 2015-01-11 12:30:00 2015-01-11 12:00:00 0.0
2015-01-21 12:30:00 2015-01-21 12:00:00 4.0
2015-01-31 12:30:00 2015-01-31 12:00:00 4.0
2015-01-02 2015-01-11 12:30:00 2015-01-11 12:00:00 1.0
2015-01-21 12:30:00 2015-01-21 12:00:00 4.0
2015-01-31 12:30:00 2015-01-31 12:00:00 4.0
2015-01-03 2015-01-11 12:30:00 2015-01-11 12:00:00 2.0
2015-01-21 12:30:00 2015-01-21 12:00:00 4.0
2015-01-31 12:30:00 2015-01-31 12:00:00 4.0
2015-01-04 2015-01-11 12:30:00 2015-01-11 12:00:00 3.0
2015-01-21 12:30:00 2015-01-21 12:00:00 4.0
2015-01-31 12:30:00 2015-01-31 12:00:00 4.0
""", pd.DataFrame(ts.sort_index()))
# the result ts have now 3 values for each point in 'a'
......@@ -781,3 +781,33 @@ def test_add_na(engine):
result = tso.get(engine, 'ts_add_na')
assert len(result) == 5
def test_dtype_mismatch(engine):
tso = TimeSerie()
tso.insert(engine,
genserie(datetime(2015, 1, 1), 'D', 11).astype('str'),
'error1',
'test')
with pytest.raises(Exception) as excinfo:
tso.insert(engine,
genserie(datetime(2015, 1, 1), 'D', 11),
'error1',
'test')
assert 'Type error when inserting error1, new type is float64, type in base is object' == str(excinfo.value)
tso.insert(engine,
genserie(datetime(2015, 1, 1), 'D', 11),
'error2',
'test')
with pytest.raises(Exception) as excinfo:
tso.insert(engine,
genserie(datetime(2015, 1, 1), 'D', 11).astype('str'),
'error2',
'test')
assert 'Type error when inserting error2, new type is object, type in base is float64' == str(excinfo.value)
......@@ -31,6 +31,11 @@ def tojson(ts):
# multi index case
return ts.to_frame().reset_index().to_json(date_format='iso')
def num2float(pdobj):
# get a Series or a Dataframe column
if str(pdobj.dtype).startswith('int'):
return pdobj.astype('float64')
return pdobj
def fromjson(jsonb, tsname):
return _fromjson(jsonb, tsname).fillna(value=np.nan)
......@@ -42,6 +47,7 @@ def _fromjson(jsonb, tsname):
result = pd.read_json(jsonb, typ='series', dtype=False)
if isinstance(result.index, pd.DatetimeIndex):
result = num2float(result)
return result
# multi index case
......@@ -50,8 +56,7 @@ def _fromjson(jsonb, tsname):
result = pd.read_json(jsonb, typ='frame',
convert_dates=columns)
result.set_index(sorted(columns), inplace=True)
return result.iloc[:, 0] # get a Series object
return num2float(result.iloc[:, 0]) # get a Series object
class TimeSerie(object):
......@@ -94,8 +99,7 @@ class TimeSerie(object):
assert isinstance(newts, pd.Series)
assert not newts.index.duplicated().any()
if str(newts.dtype).startswith('int'):
newts = newts.astype('float64')
newts = num2float(newts)
if not len(newts):
return
......@@ -336,8 +340,19 @@ class TimeSerie(object):
).values(snapshot=None)
)
def _validate_type(self, oldts, newts, name):
if (oldts is None or newts.isnull().all()):
return
old_type = oldts.dtype
new_type = newts.dtype
if new_type != old_type:
m = 'Type error when inserting {}, new type is {}, type in base is {}'.format(
name, new_type, old_type)
raise Exception(m)
def _compute_diff_and_newsnapshot(self, cn, table, newts, **extra_scalars):
snapshot = self._build_snapshot_upto(cn, table)
self._validate_type(snapshot, newts, table.name)
diff = self._compute_diff(snapshot, newts)
if len(diff) == 0:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment