Commit 3789debe authored by Aurélien Campéas's avatar Aurélien Campéas
Browse files

tsio: handle series of any type

parent aacad8844341
......@@ -25,12 +25,12 @@ def test_changeset(engine):
tso = TimeSerie()
index = pd.date_range(start=datetime(2017, 1, 1), freq='D', periods=3)
data = [1,2,3]
data = [1., 2., 3.]
with engine.connect() as cnx:
with tso.newchangeset(cnx, 'babar'):
tso.insert(cnx, pd.Series(data, index=index), 'ts_values')
tso.insert(cnx, pd.Series([5,6,7], index=index), 'ts_othervalues')
tso.insert(cnx, pd.Series(['a', 'b', 'c'], index=index), 'ts_othervalues')
g = tso.get_group(engine, 'ts_values')
g2 = tso.get_group(engine, 'ts_othervalues')
......@@ -44,7 +44,7 @@ def test_changeset(engine):
with tso.newchangeset(cnx, 'celeste'):
tso.insert(cnx, pd.Series(data, index=index), 'ts_values')
# below should be a noop
tso.insert(cnx, pd.Series([5,6,7], index=index), 'ts_othervalues')
tso.insert(cnx, pd.Series(['a', 'b', 'c'], index=index), 'ts_othervalues')
g = tso.get_group(engine, 'ts_values')
assert ['ts_values'] == list(g.keys())
......@@ -56,9 +56,9 @@ def test_changeset(engine):
""".strip() == tso.get(engine, 'ts_values').to_string().strip()
assert """
2017-01-01 5.0
2017-01-02 6.0
2017-01-03 7.0
2017-01-01 a
2017-01-02 b
2017-01-03 c
""".strip() == tso.get(engine, 'ts_othervalues').to_string().strip()
tso.delete_last_changeset_for(engine, 'ts_values')
......@@ -70,9 +70,9 @@ def test_changeset(engine):
""".strip() == tso.get(engine, 'ts_values').to_string().strip()
assert """
2017-01-01 5.0
2017-01-02 6.0
2017-01-03 7.0
2017-01-01 a
2017-01-02 b
2017-01-03 c
""".strip() == tso.get(engine, 'ts_othervalues').to_string().strip()
tso.delete_last_changeset_for(engine, 'ts_values')
......@@ -91,32 +91,32 @@ def test_differential(engine):
tso.insert(engine, ts_begin, 'ts_test', 'test')
assert """
2010-01-01 0.0
2010-01-02 1.0
2010-01-03 2.0
2010-01-04 3.0
2010-01-05 4.0
2010-01-06 5.0
2010-01-07 6.0
2010-01-08 7.0
2010-01-09 8.0
2010-01-10 9.0
2010-01-01 0
2010-01-02 1
2010-01-03 2
2010-01-04 3
2010-01-05 4
2010-01-06 5
2010-01-07 6
2010-01-08 7
2010-01-09 8
2010-01-10 9
""".strip() == tso.get(engine, 'ts_test').to_string().strip()
# we should detect the emission of a message
tso.insert(engine, ts_begin, 'ts_test', 'babar')
assert """
2010-01-01 0.0
2010-01-02 1.0
2010-01-03 2.0
2010-01-04 3.0
2010-01-05 4.0
2010-01-06 5.0
2010-01-07 6.0
2010-01-08 7.0
2010-01-09 8.0
2010-01-10 9.0
2010-01-01 0
2010-01-02 1
2010-01-03 2
2010-01-04 3
2010-01-05 4
2010-01-06 5
2010-01-07 6
2010-01-08 7
2010-01-09 8
2010-01-10 9
""".strip() == tso.get(engine, 'ts_test').to_string().strip()
ts_slight_variation = ts_begin.copy()
......@@ -125,16 +125,16 @@ def test_differential(engine):
tso.insert(engine, ts_slight_variation, 'ts_test', 'celeste')
assert """
2010-01-01 0.0
2010-01-02 1.0
2010-01-03 2.0
2010-01-04 0.0
2010-01-05 4.0
2010-01-06 5.0
2010-01-07 0.0
2010-01-08 7.0
2010-01-09 8.0
2010-01-10 9.0
2010-01-01 0
2010-01-02 1
2010-01-03 2
2010-01-04 0
2010-01-05 4
2010-01-06 5
2010-01-07 0
2010-01-08 7
2010-01-09 8
2010-01-10 9
""".strip() == tso.get(engine, 'ts_test').to_string().strip()
ts_longer = pd.Series(range(15))
......@@ -173,11 +173,11 @@ def test_differential(engine):
# -1 represents bogus upstream data
assert """
2010-01-01 2.0
2010-01-02 2.0
2010-01-03 2.0
2010-01-04 -1.0
2010-01-05 2.0
2010-01-01 2
2010-01-02 2
2010-01-03 2
2010-01-04 -1
2010-01-05 2
""".strip() == tso.get(engine, 'ts_mixte').to_string().strip()
# refresh all the period + 1 extra data point
......@@ -274,7 +274,7 @@ def test_bad_import(engine):
tso.insert(engine, ts, 'SND_SC', 'test')
result = tso.get(engine, 'SND_SC')
assert result.dtype == 'float64'
assert result.dtype == 'int64'
# insertion of empty ts
ts = pd.Series(name='truc', dtype='object')
......@@ -342,30 +342,30 @@ def test_revision_date(engine):
ts = tso.get(engine, 'ts_through_time')
assert """
2010-01-04 3.0
2010-01-05 3.0
2010-01-06 3.0
2010-01-07 3.0
2010-01-04 3
2010-01-05 3
2010-01-06 3
2010-01-07 3
""".strip() == ts.to_string().strip()
ts = tso.get(engine, 'ts_through_time',
revision_date=datetime(2015, 1, 2, 18, 43, 23) )
assert """
2010-01-04 2.0
2010-01-05 2.0
2010-01-06 2.0
2010-01-07 2.0
2010-01-04 2
2010-01-05 2
2010-01-06 2
2010-01-07 2
""".strip() == ts.to_string().strip()
ts = tso.get(engine, 'ts_through_time',
revision_date=datetime(2015, 1, 1, 18, 43, 23))
assert """
2010-01-04 1.0
2010-01-05 1.0
2010-01-06 1.0
2010-01-07 1.0
2010-01-04 1
2010-01-05 1
2010-01-06 1
2010-01-07 1
""".strip() == ts.to_string().strip()
ts = tso.get(engine, 'ts_through_time',
......
......@@ -63,14 +63,10 @@ class TimeSerie(object):
assert isinstance(newts, pd.Series)
newts = newts[~newts.isnull()] # wipe the the NaNs
if len(newts):
assert newts.index.dtype.name == 'datetime64[ns]'
else:
if not len(newts):
return
newts = newts.astype('float64')
newts.name = name
table = self._get_ts_table(cnx, name)
if table is None:
......@@ -85,7 +81,6 @@ class TimeSerie(object):
}
# callback for extenders
self._complete_insertion_value(value, extra_scalars)
cnx.execute(table.insert().values(value))
self._finalize_insertion(cnx, csid, name)
print('Fisrt insertion of %s by %s' % (name, author))
......@@ -313,23 +308,28 @@ class TimeSerie(object):
# diff handling
def _compute_diff(self, ts1, ts2):
"""Compute the difference between ts1 and ts2 (like in ts2 - ts1).
def _compute_diff(self, fromts, tots):
"""Compute the difference between fromts and tots (like in tots - fromts).
Deletions are not handled. New lines in ts2 and lines that
changed in ts2 relatively to ts2 will appear in the diff.
Deletions are not handled. New lines in tots and lines that
changed in tots relatively to tots will appear in the diff.
"""
if ts1 is None:
return ts2
mask_overlap = ts2.index.isin(ts1.index)
ts_bef_overlap = ts1[ts2.index[mask_overlap]]
ts_overlap = ts2[mask_overlap]
mask_equal = np.isclose(ts_bef_overlap, ts_overlap, atol=PRECISION)
ts_diff_overlap = ts2[mask_overlap][~mask_equal]
ts_diff_new = ts2[~mask_overlap]
ts_result = pd.concat([ts_diff_overlap, ts_diff_new])
return ts_result
if fromts is None:
return tots
mask_overlap = tots.index.isin(fromts.index)
fromts_overlap = fromts[tots.index[mask_overlap]]
tots_overlap = tots[mask_overlap]
if fromts.dtype == 'float64':
mask_equal = np.isclose(fromts_overlap, tots_overlap, atol=PRECISION)
else:
mask_equal = fromts_overlap == tots_overlap
diff_overlap = tots[mask_overlap][~mask_equal]
diff_new = tots[~mask_overlap]
return pd.concat([diff_overlap, diff_new])
def _apply_diff(self, base_ts, new_ts):
"""Produce a new ts using base_ts as a base and taking any
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment