Commit 837dd449 authored by Aurélien Campéas's avatar Aurélien Campéas
Browse files

delete: remove the log entries cleanup phase from the delete operation

Having a slightly cluttered changeset log is a much less serious issue
than having slow and/or failing delete operations.

We now mark the changeset entries to tell them from normal entries.

Proper handling of the changeset entries deletion will be addressed in
#54.

closes #51
parent 66322fd79100
......@@ -16,25 +16,6 @@ from tshistory.testutil import (
)
@pytest.mark.perf
def test_delete(engine, ptsh):
for year in range(1900, 2100):
for month in range(1, 13):
idate = utcdt(year, month, 1)
ts = pd.Series(np.random.random(30),
index=pd.date_range(start=utcdt(year, month, 1),
freq='D', periods=30))
ptsh.insert(engine, ts, 'delete_perf', 'Babar')
ptsh.insert(engine, ts, 'delete_perf_a', 'Babar')
ptsh.insert(engine, ts, 'delete_perf_b', 'Babar')
ptsh.insert(engine, ts, 'delete_perf_c', 'Babar')
t0 = time()
print(len(ptsh.get(engine, 'delete_perf')))
print(len(ptsh.insertion_dates(engine, 'delete_perf')))
ptsh.delete(engine, 'delete_perf')
print(time() - t0)
@pytest.mark.perf
def test_hourly_forecast(engine, tracker, ptsh):
tsh = ptsh
......
......@@ -1086,8 +1086,8 @@ def test_serie_deletion(engine, tsh):
seriecount2, csetcount2, csetseriecount2 = assert_structures(engine, tsh)
assert csetcount - csetcount2 == 2
assert csetseriecount - csetseriecount2 == 2
assert csetcount - csetcount2 == 0
assert csetseriecount - csetseriecount2 == 0
assert seriecount - seriecount2 == 1
assert tsh.metadata(engine, 'deleteme') is None
......@@ -1108,6 +1108,12 @@ def test_serie_deletion(engine, tsh):
'value_dtype': '<f8'
}
sql = f"""select count(*) from {tsh.namespace}.changeset
where (metadata::jsonb ->> 'tshistory.info')
like 'belonged to deleted series `deleteme`' """
count = engine.execute(sql).scalar()
assert count == 2
def test_strip(engine, tsh):
for i in range(1, 5):
......@@ -1185,7 +1191,9 @@ insertion_date value_date
meta = l['meta']
stripinfo = meta.get('tshistory.info')
if stripinfo:
assert stripinfo.startswith('got stripped from')
assert (stripinfo.startswith('got stripped from') or
# see how this test is silly ?
stripinfo.startswith('belonged to'))
def test_long_name(engine, tsh):
......
......@@ -257,7 +257,6 @@ def check(db_uri, series=None, namespace='tsh'):
)
@tsh.command(name='shell')
@click.argument('db-uri')
@click.option('--namespace', default='tsh')
......@@ -268,6 +267,25 @@ def shell(db_uri, namespace='tsh'):
import pdb; pdb.set_trace()
@tsh.command(name='migrate-0.6-to-0.7')
@click.argument('db-uri')
@click.option('--namespace', default='tsh')
def migrate_dot_6_to_dot_7(db_uri, namespace='tsh'):
e = create_engine(find_dburi(db_uri))
tsh = TimeSerie(namespace)
with e.begin() as cn:
# drop not null
cn.execute(f'alter table "{namespace}".changeset_series '
'alter column serie drop not null')
# alter foreign key on delete: delete -> set null
cn.execute(f'alter table "{namespace}".changeset_series '
'drop constraint "changeset_series_serie_fkey"')
cn.execute(f'alter table "{namespace}".changeset_series '
'add constraint "changeset_series_serie_fkey" '
f'foreign key (serie) references "{namespace}".registry (id) '
'on delete set null')
for ep in iter_entry_points('tshistory.subcommands'):
tsh.add_command(ep.load())
......
......@@ -21,7 +21,7 @@ create index on "{ns}".changeset(insertion_date) ;
create table "{ns}".changeset_series (
cset integer references "{ns}".changeset(id) on delete set null,
serie integer not null references "{ns}".registry(id) on delete cascade
serie integer references "{ns}".registry(id) on delete set null
);
create index on "{ns}".changeset_series(cset);
......
......@@ -350,13 +350,27 @@ class timeseries(SeriesServices):
@tx
def delete(self, cn, seriename):
if not self.exists(cn, seriename):
tablename = self._serie_to_tablename(cn, seriename)
if tablename is None:
print('not deleting unknown series', seriename, self.namespace)
return
# changeset will keep ghost entries
# we cleanup changeset series, then registry
# then we drop the two remaining tables
# cn *must* be a transaction scope
# whose cleanup is costly
# we will mark them as from a deleted series
# update changeset.metadata
msg = f'belonged to deleted series `{seriename}`'
csetsql = f'select cset from "{self.namespace}.timeserie"."{tablename}"'
for csid, in cn.execute(csetsql):
metadata = self.changeset_metadata(cn, csid) or {}
metadata['tshistory.info'] = msg
cn.execute(
f'update "{self.namespace}".changeset '
'set metadata = %(metadata)s '
'where id = %(csid)s',
csid=csid,
metadata=json.dumps(metadata)
)
rid, tablename = cn.execute(
'select id, table_name from "{}".registry '
'where seriename = %(seriename)s'.format(self.namespace),
......@@ -369,15 +383,6 @@ class timeseries(SeriesServices):
cn.execute(
'drop table "{}.snapshot"."{}" cascade'.format(self.namespace, tablename)
)
# cleanup changesets table
cn.execute('with csets as ('
' select cset from "{ns}".changeset_series '
' where serie = %(rid)s'
') '
'delete from "{ns}".changeset as cset using csets '
'where cset.id = csets.cset'.format(ns=self.namespace),
rid=rid
)
cn.execute('delete from "{}".registry '
'where id = %(rid)s'.format(self.namespace),
rid=rid)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment