Commit c0f0a180 authored by Aurélien Campéas's avatar Aurélien Campéas
Browse files

storage: handle serie names of arbitrary length

The postgres table name limit should not creep up.
parent 610487f69848
......@@ -914,6 +914,6 @@ def test_prepend(engine, tsh):
def test_long_name(engine, tsh):
serie = genserie(datetime(2010, 1, 1), 'D', 40)
import sqlalchemy
with pytest.raises(sqlalchemy.exc.IdentifierError):
tsh.insert(engine, serie, 'a' * 64, 'babar')
name = 'a' * 64
tsh.insert(engine, serie, name, 'babar')
assert tsh.get(engine, name) is not None
......@@ -24,7 +24,8 @@ class Snapshot(SeriesServices):
def __init__(self, cn, tsh, seriename):
self.cn = cn
self.tsh = tsh
self.name = seriename
self.seriename = seriename
self.name = self._tablename(seriename)
@property
def namespace(self):
......@@ -65,7 +66,7 @@ class Snapshot(SeriesServices):
body = b'{' + b','.join(self._deserialize(chunk) for chunk in chunks) + b'}'
return self.tsh._ensure_tz_consistency(
self.cn,
fromjson(body.decode('utf-8'), self.name)
fromjson(body.decode('utf-8'), self.seriename)
)
# /serialisation
......@@ -101,7 +102,7 @@ class Snapshot(SeriesServices):
def update(self, diff):
# get last chunkhead for cset
tstable = self.tsh._get_ts_table(self.cn, self.name)
tstable = self.tsh._get_ts_table(self.cn, self.seriename)
headsql = select(
[tstable.c.snapshot]
).order_by(desc(tstable.c.id)
......@@ -176,7 +177,7 @@ class Snapshot(SeriesServices):
def find(self, qfilter=(),
from_value_date=None, to_value_date=None):
cset = self.tsh.schema.changeset
table = self.tsh._get_ts_table(self.cn, self.name)
table = self.tsh._get_ts_table(self.cn, self.seriename)
sql = select([table.c.cset, table.c.snapshot]
).order_by(desc(table.c.id)
).limit(1
......
from datetime import datetime
from contextlib import contextmanager
import logging
import hashlib
import pandas as pd
......@@ -357,11 +358,12 @@ class TimeSerie(SeriesServices):
# serie table handling
def _ts_table_name(self, seriename):
# namespace.seriename
seriename = self._tablename(seriename)
return '{}.timeserie.{}'.format(self.namespace, seriename)
def _table_definition_for(self, seriename):
tablename = self._ts_table_name(seriename)
seriename = self._tablename(seriename)
table = TABLES.get(tablename)
if table is None:
TABLES[tablename] = table = Table(
......
import math
import zlib
import hashlib
import numpy as np
import pandas as pd
......@@ -105,6 +106,12 @@ class SeriesServices(object):
# serialization
def _tablename(self, name):
# postgresql table names are limited to 63 chars.
if len(name) > 63:
return hashlib.sha1(name.encode('utf-8')).hexdigest()
return name
def _serialize(self, ts):
if ts is None:
return None
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment