Commit 1a685fa0 authored by Aurélien Campéas's avatar Aurélien Campéas
Browse files

tsio: just don't store full nans on initial insertion

Next step might be to also don't store those rightmost and leftmost
nans on update when they overstretch the current horizon.
parent 1d3bacdc6ce7
......@@ -516,13 +516,14 @@ def test_point_deletion(engine, tsh):
erased = tsh.get(engine, 'ts_string_del')
assert len(erased) == 0
# first insertion with only nan
def test_nan_first(engine, tsh):
# first insertion with only nan
ts_begin = genserie(datetime(2010, 1, 1), 'D', 10, [np.nan])
tsh.insert(engine, ts_begin, 'ts_null', 'test')
assert tsh.insert(engine, ts_begin, 'ts_null', 'test') is None
assert len(tsh.get(engine, 'ts_null')) == 0
def test_more_point_deletion(engine, tsh):
ts_repushed = genserie(datetime(2010, 1, 1), 'D', 11)
ts_repushed[0:3] = np.nan
......@@ -606,7 +607,16 @@ def test_deletion_over_horizon(engine, tsh):
tsh.insert(engine, ts, name, 'Celeste')
ival = tsh.interval(engine, name)
assert ival.left == datetime(2018, 1, 1)
assert ival.right == datetime(2018, 1, 5)
assert ival.right == datetime(2018, 1, 3)
ts = pd.Series(
[np.nan, np.nan, np.nan],
index=pd.date_range(datetime(2017, 12, 30), freq='D', periods=3)
)
tsh.insert(engine, ts, name, 'Arthur')
ival = tsh.interval(engine, name)
assert ival.left == datetime(2018, 1, 1)
assert ival.right == datetime(2018, 1, 3)
def test_get_history(engine, tsh):
......@@ -861,18 +871,7 @@ def test_add_na(engine, tsh):
ts_nan[[True] * len(ts_nan)] = np.nan
diff = tsh.insert(engine, ts_nan, 'ts_add_na', 'test')
assert len(diff) == 5
result = tsh.get(engine, 'ts_add_na')
assert len(result) == 0
result = tsh.get(engine, 'ts_add_na', _keep_nans=True)
assert_df("""
2010-01-01 NaN
2010-01-02 NaN
2010-01-03 NaN
2010-01-04 NaN
2010-01-05 NaN
""", result)
assert diff is None
# in case of insertion in existing data
ts_begin = genserie(datetime(2010, 1, 1), 'D', 5)
......@@ -1408,9 +1407,7 @@ def test_null_serie(engine, tsh):
def test_na(engine, tsh):
# all na
ts = genserie(datetime(2010, 1, 10), 'D', 10, [np.nan], name='truc')
tsh.insert(engine, ts, 'test_nan', 'test')
assert len(tsh.get(engine, 'test_nan')) == 0
assert len(tsh.get(engine, 'test_nan', _keep_nans=True)) == 10
assert tsh.insert(engine, ts, 'test_nan', 'test') is None
# mix na
ts = pd.Series([np.nan] * 5 + [3] * 5,
......
......@@ -399,7 +399,7 @@ class TimeSerie(SeriesServices):
log.sort(key=lambda rev: rev['rev'])
return log
def interval(self, cn, seriename):
def interval(self, cn, seriename, notz=False):
tablename = self._serie_to_tablename(cn, seriename)
if tablename is None:
raise ValueError(f'no such serie: {seriename}')
......@@ -408,7 +408,7 @@ class TimeSerie(SeriesServices):
f'order by cset desc limit 1')
res = cn.execute(sql).fetchone()
start, end = res.start, res.end
if self.metadata(cn, seriename).get('tzaware'):
if self.metadata(cn, seriename).get('tzaware') and not notz:
start, end = pd.Timestamp(start, tz='UTC'), pd.Timestamp(end, tz='UTC')
return pd.Interval(left=start, right=end, closed='both')
......@@ -422,6 +422,11 @@ class TimeSerie(SeriesServices):
# initial insertion
if len(newts) == 0:
return None
start, end = start_end(newts)
if start is None:
assert end is None
# this is just full of nans
return None
# at creation time we take an exclusive lock to avoid
# a deadlock on created tables against the changeset-series fk
cn.execute(
......@@ -459,9 +464,9 @@ class TimeSerie(SeriesServices):
csid = self._newchangeset(cn, author, insertion_date, metadata)
tsstart, tsend = start_end(newts)
ival = self.interval(cn, seriename)
start = min(tsstart, ival.left.replace(tzinfo=None))
end = max(tsend, ival.right.replace(tzinfo=None))
ival = self.interval(cn, seriename, notz=True)
start = min(tsstart or ival.left, ival.left)
end = max(tsend or ival.right, ival.right)
head = snapshot.update(diff)
value = {
'cset': csid,
......
......@@ -15,6 +15,9 @@ def tzaware_serie(ts):
def start_end(ts):
ts = ts.dropna()
if not len(ts):
return None, None
start = ts.index.min()
end = ts.index.max()
if start.tzinfo is not None:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment