Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
pony
tshistory
Commits
8fcee4ea2e61
Commit
51f74e22
authored
Aug 21, 2019
by
Aurélien Campéas
Browse files
insertion_date: make it official and check it is always increasing
parent
da5ad6f2a35e
Changes
4
Show whitespace changes
Inline
Side-by-side
test/test_cli.py
View file @
8fcee4ea
...
...
@@ -32,10 +32,10 @@ def test_log(engine, cli, tsh):
def
test_history
(
engine
,
cli
,
tsh
):
serie
=
genserie
(
datetime
(
2020
,
1
,
1
),
'D'
,
3
)
tsh
.
insert
(
engine
,
serie
,
'some_history'
,
'Babar'
,
_
insertion_date
=
utcdt
(
2019
,
1
,
1
))
insertion_date
=
utcdt
(
2019
,
1
,
1
))
serie
=
genserie
(
datetime
(
2020
,
1
,
2
),
'D'
,
3
)
tsh
.
insert
(
engine
,
serie
,
'some_history'
,
'Babar'
,
_
insertion_date
=
utcdt
(
2019
,
1
,
2
))
insertion_date
=
utcdt
(
2019
,
1
,
2
))
r
=
cli
(
'history'
,
engine
.
url
,
'some_history'
,
...
...
test/test_snapshot.py
View file @
8fcee4ea
...
...
@@ -223,7 +223,7 @@ def test_append(engine, tsh):
freq
=
'D'
,
periods
=
10
)):
ts
=
genserie
(
dt
,
'D'
,
1
,
[
x
],
name
=
'daily'
)
tsh
.
insert
(
engine
,
ts
,
'append'
,
'aurelien.campeas@pythonian.fr'
,
_
insertion_date
=
dt
)
insertion_date
=
dt
)
sql
=
'select id, parent, chunk from "{}.snapshot".append order by id'
.
format
(
tsh
.
namespace
...
...
test/test_tsio.py
View file @
8fcee4ea
...
...
@@ -2,6 +2,7 @@ from datetime import datetime, timedelta
from
pathlib
import
Path
import
pytz
from
dateutil
import
parser
import
pytest
import
numpy
as
np
...
...
@@ -69,7 +70,7 @@ Freq: H
"""
,
ts
)
tsh
.
insert
(
engine
,
ts
,
'tztest'
,
'Babar'
,
_
insertion_date
=
utcdt
(
2018
,
1
,
1
))
insertion_date
=
utcdt
(
2018
,
1
,
1
))
back
=
tsh
.
get
(
engine
,
'tztest'
)
# though un localized we understand it's been normalized to utc
...
...
@@ -91,7 +92,7 @@ Freq: H
'H'
,
4
,
tz
=
'UTC'
)
ts
.
index
=
ts
.
index
.
tz_convert
(
'Europe/Paris'
)
tsh
.
insert
(
engine
,
ts
,
'tztest'
,
'Celeste'
,
_
insertion_date
=
utcdt
(
2018
,
1
,
3
))
insertion_date
=
utcdt
(
2018
,
1
,
3
))
ts
=
tsh
.
get
(
engine
,
'tztest'
)
assert_df
(
"""
...
...
@@ -382,7 +383,7 @@ def test_changeset_metadata(engine, tsh):
serie
=
genserie
(
datetime
(
2010
,
1
,
1
),
'D'
,
1
,
initval
=
[
1
])
tsh
.
insert
(
engine
,
serie
,
'ts-cs-metadata'
,
'babar'
,
{
'foo'
:
'A'
,
'bar'
:
42
},
_
insertion_date
=
utcdt
(
2019
,
1
,
1
)
insertion_date
=
utcdt
(
2019
,
1
,
1
)
)
log
=
tsh
.
log
(
engine
,
'ts-cs-metadata'
)
...
...
@@ -400,31 +401,31 @@ def test_revision_date(engine, tsh):
for
i
in
range
(
1
,
5
):
with
engine
.
begin
()
as
cn
:
tsh
.
insert
(
cn
,
genserie
(
datetime
(
2017
,
1
,
i
),
'D'
,
3
,
[
i
]),
'revdate'
,
'test'
,
_
insertion_date
=
utcdt
(
2016
,
1
,
i
))
'test'
,
insertion_date
=
utcdt
(
2016
,
1
,
i
))
# end of prologue, now some real meat
idate0
=
pd
.
Timestamp
(
'2015-1-1 00:00:00'
,
tz
=
'UTC'
)
ts
=
genserie
(
datetime
(
2010
,
1
,
4
),
'D'
,
4
,
[
0
],
name
=
'truc'
)
tsh
.
insert
(
engine
,
ts
,
'ts_through_time'
,
'test'
,
_
insertion_date
=
idate0
)
'test'
,
insertion_date
=
idate0
)
assert
idate0
==
tsh
.
latest_insertion_date
(
engine
,
'ts_through_time'
)
idate1
=
pd
.
Timestamp
(
'2015-1-1 15:45:23'
,
tz
=
'UTC'
)
ts
=
genserie
(
datetime
(
2010
,
1
,
4
),
'D'
,
4
,
[
1
],
name
=
'truc'
)
tsh
.
insert
(
engine
,
ts
,
'ts_through_time'
,
'test'
,
_
insertion_date
=
idate1
)
'test'
,
insertion_date
=
idate1
)
assert
idate1
==
tsh
.
latest_insertion_date
(
engine
,
'ts_through_time'
)
idate2
=
pd
.
Timestamp
(
'2015-1-2 15:43:23'
,
tz
=
'UTC'
)
ts
=
genserie
(
datetime
(
2010
,
1
,
4
),
'D'
,
4
,
[
2
],
name
=
'truc'
)
tsh
.
insert
(
engine
,
ts
,
'ts_through_time'
,
'test'
,
_
insertion_date
=
idate2
)
'test'
,
insertion_date
=
idate2
)
assert
idate2
==
tsh
.
latest_insertion_date
(
engine
,
'ts_through_time'
)
idate3
=
pd
.
Timestamp
(
'2015-1-3'
,
tz
=
'UTC'
)
ts
=
genserie
(
datetime
(
2010
,
1
,
4
),
'D'
,
4
,
[
3
],
name
=
'truc'
)
tsh
.
insert
(
engine
,
ts
,
'ts_through_time'
,
'test'
,
_
insertion_date
=
idate3
)
'test'
,
insertion_date
=
idate3
)
assert
idate3
==
tsh
.
latest_insertion_date
(
engine
,
'ts_through_time'
)
ts
=
tsh
.
get
(
engine
,
'ts_through_time'
)
...
...
@@ -653,7 +654,7 @@ def test_deletion_over_horizon(engine, tsh):
name
=
'delete_over_hz'
tsh
.
insert
(
engine
,
ts
,
name
,
'Babar'
,
_
insertion_date
=
idate
)
insertion_date
=
idate
)
ts
=
pd
.
Series
(
[
np
.
nan
,
np
.
nan
,
np
.
nan
],
...
...
@@ -661,7 +662,7 @@ def test_deletion_over_horizon(engine, tsh):
)
tsh
.
insert
(
engine
,
ts
,
name
,
'Celeste'
,
_
insertion_date
=
idate
.
replace
(
day
=
2
))
insertion_date
=
idate
.
replace
(
day
=
2
))
ival
=
tsh
.
interval
(
engine
,
name
)
assert
ival
.
left
==
datetime
(
2018
,
1
,
1
)
assert
ival
.
right
==
datetime
(
2018
,
1
,
2
)
...
...
@@ -671,7 +672,7 @@ def test_deletion_over_horizon(engine, tsh):
index
=
pd
.
date_range
(
datetime
(
2017
,
12
,
30
),
freq
=
'D'
,
periods
=
3
)
)
tsh
.
insert
(
engine
,
ts
,
name
,
'Arthur'
,
_
insertion_date
=
idate
.
replace
(
day
=
3
))
insertion_date
=
idate
.
replace
(
day
=
3
))
ival
=
tsh
.
interval
(
engine
,
name
)
assert
ival
.
left
==
datetime
(
2018
,
1
,
2
)
assert
ival
.
right
==
datetime
(
2018
,
1
,
2
)
...
...
@@ -682,7 +683,7 @@ def test_history(engine, tsh):
with
engine
.
begin
()
as
cn
:
tsh
.
insert
(
cn
,
genserie
(
datetime
(
2017
,
1
,
1
),
'D'
,
numserie
),
'smallserie'
,
'aurelien.campeas@pythonian.fr'
,
_
insertion_date
=
utcdt
(
2017
,
2
,
numserie
))
insertion_date
=
utcdt
(
2017
,
2
,
numserie
))
ts
=
tsh
.
get
(
engine
,
'smallserie'
)
assert_df
(
"""
...
...
@@ -733,7 +734,7 @@ insertion_date value_date
with
engine
.
begin
()
as
cn
:
idate
=
idate
.
replace
(
tzinfo
=
pytz
.
timezone
(
'UTC'
))
tsh
.
insert
(
cn
,
histts
[
idate
],
'smallserie2'
,
'aurelien.campeas@pythonian.f'
,
_
insertion_date
=
idate
)
'aurelien.campeas@pythonian.f'
,
insertion_date
=
idate
)
# this is perfectly round-tripable
assert
(
tsh
.
get
(
engine
,
'smallserie2'
)
==
ts
).
all
()
...
...
@@ -847,7 +848,7 @@ def test_delta_na(engine, tsh):
for
idx
,
idate
in
enumerate
(
ldates
):
ts
=
pd
.
Series
([
idx
]
*
3
,
index
=
ldates
)
tsh
.
insert
(
engine
,
ts
,
'without_na'
,
'arnaud'
,
_
insertion_date
=
idate
)
insertion_date
=
idate
)
assert_df
(
"""
2015-01-20 00:00:00+00:00 2.0
...
...
@@ -887,7 +888,7 @@ insertion_date value_date
if
idx
==
2
:
serie
[
-
1
]
=
np
.
nan
tsh
.
insert
(
engine
,
serie
,
'with_na'
,
'arnaud'
,
_
insertion_date
=
idate
)
insertion_date
=
idate
)
# the value at 2015-01-22 is hidden by the inserted nan
assert_df
(
"""
...
...
@@ -922,7 +923,8 @@ def test_nr_gethistory(engine, tsh):
index
=
pd
.
date_range
(
start
=
datetime
(
2016
,
12
,
29
),
end
=
datetime
(
2017
,
1
,
1
),
freq
=
'D'
))
tsh
.
insert
(
engine
,
s0
,
'foo'
,
'zogzog'
)
tsh
.
insert
(
engine
,
s0
,
'foo'
,
'zogzog'
,
insertion_date
=
utcdt
(
2015
,
12
,
31
))
s1
=
pd
.
Series
([
1
,
0
,
0
,
1
],
index
=
pd
.
date_range
(
start
=
datetime
(
2017
,
1
,
1
),
...
...
@@ -933,7 +935,7 @@ def test_nr_gethistory(engine, tsh):
with
engine
.
begin
()
as
cn
:
tsh
.
insert
(
cn
,
s1
*
i
,
'foo'
,
'aurelien.campeas@pythonian.f'
,
_
insertion_date
=
idate
+
timedelta
(
days
=
i
))
insertion_date
=
idate
+
timedelta
(
days
=
i
))
df
=
tsh
.
history
(
engine
,
'foo'
,
datetime
(
2016
,
1
,
3
),
...
...
@@ -1080,7 +1082,7 @@ def test_strip(engine, tsh):
for
i
in
range
(
1
,
5
):
pubdate
=
utcdt
(
2017
,
1
,
i
)
ts
=
genserie
(
datetime
(
2017
,
1
,
10
),
'H'
,
1
+
i
)
tsh
.
insert
(
engine
,
ts
,
'xserie'
,
'babar'
,
_
insertion_date
=
pubdate
)
tsh
.
insert
(
engine
,
ts
,
'xserie'
,
'babar'
,
insertion_date
=
pubdate
)
# also insert something completely unrelated
tsh
.
insert
(
engine
,
genserie
(
datetime
(
2018
,
1
,
1
),
'D'
,
1
+
i
),
'yserie'
,
'celeste'
)
...
...
@@ -1185,7 +1187,7 @@ def test_staircase(engine, tsh):
freq
=
'H'
):
ts
=
genserie
(
start
=
idate
,
freq
=
'H'
,
repeat
=
7
)
tsh
.
insert
(
engine
,
ts
,
'republication'
,
'test'
,
_
insertion_date
=
idate
)
insertion_date
=
idate
)
hist
=
tsh
.
history
(
engine
,
'republication'
)
assert_hist
(
"""
...
...
@@ -1255,7 +1257,7 @@ def test_staircase_2_tzaware(engine, tsh):
end
=
utcdt
(
2015
,
1
,
4
),
freq
=
'D'
)):
ts
=
genserie
(
start
=
idate
,
freq
=
'H'
,
repeat
=
7
)
tsh
.
insert
(
engine
,
ts
,
'repu2'
,
'test'
,
_
insertion_date
=
idate
)
tsh
.
insert
(
engine
,
ts
,
'repu2'
,
'test'
,
insertion_date
=
idate
)
deltas
=
tsh
.
staircase
(
engine
,
'repu2'
,
delta
=
timedelta
(
hours
=
3
))
assert_df
(
"""
...
...
@@ -1317,7 +1319,7 @@ def test_staircase_2_tznaive(engine, tsh):
end
=
utcdt
(
2015
,
1
,
4
),
freq
=
'D'
)):
ts
=
genserie
(
start
=
idate
.
replace
(
tzinfo
=
None
),
freq
=
'H'
,
repeat
=
7
)
tsh
.
insert
(
engine
,
ts
,
'repu-tz-naive'
,
'test'
,
_
insertion_date
=
idate
)
tsh
.
insert
(
engine
,
ts
,
'repu-tz-naive'
,
'test'
,
insertion_date
=
idate
)
deltas
=
tsh
.
staircase
(
engine
,
'repu-tz-naive'
,
delta
=
timedelta
(
hours
=
3
))
assert_df
(
"""
...
...
@@ -1553,7 +1555,7 @@ def test_insert_errors(engine, tsh):
tsh
.
insert
(
engine
,
ts
,
'error'
,
42
)
with
pytest
.
raises
(
AssertionError
):
tsh
.
insert
(
engine
,
ts
,
'error'
,
'Babar'
,
_
insertion_date
=
'2010-1-1'
)
tsh
.
insert
(
engine
,
ts
,
'error'
,
'Babar'
,
insertion_date
=
'2010-1-1'
)
with
pytest
.
raises
(
AssertionError
):
tsh
.
insert
(
engine
,
ts
,
'error'
,
'Babar'
,
metadata
=
42
)
...
...
tshistory/tsio.py
View file @
8fcee4ea
...
...
@@ -47,7 +47,7 @@ class timeseries(SeriesServices):
@
tx
def
insert
(
self
,
cn
,
newts
,
name
,
author
,
metadata
=
None
,
_
insertion_date
=
None
):
insertion_date
=
None
):
"""Create a new revision of a given time series
newts: pandas.Series with date index
...
...
@@ -59,7 +59,7 @@ class timeseries(SeriesServices):
return
newts
=
self
.
_guard_insert
(
newts
,
name
,
author
,
metadata
,
_
insertion_date
insertion_date
)
assert
(
'<M8[ns]'
==
newts
.
index
.
dtype
or
...
...
@@ -72,10 +72,10 @@ class timeseries(SeriesServices):
if
tablename
is
None
:
seriesmeta
=
self
.
_series_initial_meta
(
cn
,
name
,
newts
)
return
self
.
_create
(
cn
,
newts
,
name
,
author
,
seriesmeta
,
metadata
,
_
insertion_date
)
metadata
,
insertion_date
)
return
self
.
_update
(
cn
,
tablename
,
newts
,
name
,
author
,
metadata
,
_
insertion_date
)
metadata
,
insertion_date
)
def
list_series
(
self
,
cn
):
"""Return the mapping of all series to their type"""
...
...
@@ -273,9 +273,11 @@ class timeseries(SeriesServices):
q
=
select
(
'max(insertion_date)'
).
table
(
f
'"
{
self
.
namespace
}
.revision"."
{
tablename
}
"'
)
return
pd
.
Timestamp
(
idate
=
pd
.
Timestamp
(
q
.
do
(
cn
).
scalar
()
).
astimezone
(
'UTC'
)
)
if
not
pd
.
isnull
(
idate
):
return
idate
.
astimezone
(
'UTC'
)
@
tx
def
insertion_dates
(
self
,
cn
,
name
,
...
...
@@ -469,12 +471,6 @@ class timeseries(SeriesServices):
cn
.
execute
(
f
'select pg_advisory_xact_lock(
{
self
.
create_lock_id
}
)'
)
if
insertion_date
is
not
None
:
assert
insertion_date
.
tzinfo
is
not
None
idate
=
pd
.
Timestamp
(
insertion_date
)
else
:
idate
=
pd
.
Timestamp
(
datetime
.
utcnow
(),
tz
=
'UTC'
)
if
metadata
:
metadata
=
json
.
dumps
(
metadata
)
...
...
@@ -541,6 +537,9 @@ class timeseries(SeriesServices):
idate
=
pd
.
Timestamp
(
insertion_date
)
else
:
idate
=
pd
.
Timestamp
(
datetime
.
utcnow
(),
tz
=
'UTC'
)
latest_idate
=
self
.
latest_insertion_date
(
cn
,
name
)
if
latest_idate
:
assert
idate
>
latest_idate
if
metadata
:
metadata
=
json
.
dumps
(
metadata
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment