test_tsio.py 21.7 KB
Newer Older
1
2
3
4
5
6
7
# coding: utf-8
from pathlib import Path
from datetime import datetime
from dateutil import parser

import pandas as pd
import numpy as np
8
import pytest
9
from mock import patch
10

11
from tshistory.tsio import TimeSerie
12
13
14
15

DATADIR = Path(__file__).parent / 'data'


16
17
18
19
20
21
22
def assert_group_equals(g1, g2):
    for (n1, s1), (n2, s2) in zip(sorted(g1.items()),
                                  sorted(g2.items())):
        assert n1 == n2
        assert s1.equals(s2)


23
24
25
26
def assert_df(expected, df):
    assert expected.strip() == df.to_string().strip()


27
28
29
30
31
32
33
34
35
36
37
38
def genserie(start, freq, repeat, initval=None, tz=None, name=None):
    if initval is None:
        values = range(repeat)
    else:
        values = initval * repeat
    return pd.Series(values,
                     name=name,
                     index=pd.date_range(start=start,
                                         freq=freq,
                                         periods=repeat,
                                         tz=tz))

39
def test_changeset(engine):
40
41
42
43
    # instantiate one time serie handler object
    tso = TimeSerie()

    index = pd.date_range(start=datetime(2017, 1, 1), freq='D', periods=3)
44
    data = [1., 2., 3.]
45

46
47
    with patch('tshistory.tsio.datetime') as mock_date:
        mock_date.now.return_value = datetime(2020, 1, 1)
48
49
50
51
        with engine.connect() as cn:
            with tso.newchangeset(cn, 'babar'):
                tso.insert(cn, pd.Series(data, index=index), 'ts_values')
                tso.insert(cn, pd.Series(['a', 'b', 'c'], index=index), 'ts_othervalues')
52

53
54
55
        g = tso.get_group(engine, 'ts_values')
        g2 = tso.get_group(engine, 'ts_othervalues')
        assert_group_equals(g, g2)
56

57
58
        with pytest.raises(AssertionError):
            tso.insert(engine, pd.Series([2,3,4], index=index), 'ts_values')
59

60
        with engine.connect() as cn:
61
            data.append(data.pop(0))
62
63
            with tso.newchangeset(cn, 'celeste'):
                tso.insert(cn, pd.Series(data, index=index), 'ts_values')
64
                # below should be a noop
65
                tso.insert(cn, pd.Series(['a', 'b', 'c'], index=index), 'ts_othervalues')
66

67
68
69
    g = tso.get_group(engine, 'ts_values')
    assert ['ts_values'] == list(g.keys())

70
    assert_df("""
71
72
73
2017-01-01    2.0
2017-01-02    3.0
2017-01-03    1.0
74
""", tso.get(engine, 'ts_values'))
75

76
    assert_df("""
77
78
79
2017-01-01    a
2017-01-02    b
2017-01-03    c
80
""", tso.get(engine, 'ts_othervalues'))
81

82
83
84
85
86
87
88
89
90
91
92
93
    log = tso.log(engine)
    assert [
        {'author': 'babar',
         'rev': 1,
         'date': datetime(2020, 1, 1, 0, 0),
         'names': ['ts_values', 'ts_othervalues']},
        {'author': 'celeste',
         'rev': 2,
         'date': datetime(2020, 1, 1, 0, 0),
         'names': ['ts_values']}
    ] == log

94
95
96
    log = tso.log(engine, names=['ts_othervalues'])
    assert len(log) == 1
    assert log[0]['rev'] == 1
97
    assert log[0]['names'] == ['ts_values', 'ts_othervalues']
98

99
100
101
102
103
104
    log = tso.log(engine, fromrev=2)
    assert len(log) == 1

    log = tso.log(engine, torev=1)
    assert len(log) == 1

105
106
107
108
109
110
111
    info = tso.info(engine)
    assert {
        'changeset count': 2,
        'serie names': ['ts_othervalues', 'ts_values'],
        'series count': 2
    } == info

112

113
114
def test_tstamp_roundtrip(engine):
    tso = TimeSerie()
115
116
    ts = genserie(datetime(2017, 10, 28, 23),
                  'H', 4, tz='UTC')
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
    ts.index = ts.index.tz_convert('Europe/Paris')

    assert_df("""
2017-10-29 01:00:00+02:00    0
2017-10-29 02:00:00+02:00    1
2017-10-29 02:00:00+01:00    2
2017-10-29 03:00:00+01:00    3
Freq: H
    """, ts)

    tso.insert(engine, ts, 'tztest', 'Babar')
    back = tso.get(engine, 'tztest')

    # though un localized we understand it's been normalized to utc
    assert_df("""
2017-10-28 23:00:00    0.0
2017-10-29 00:00:00    1.0
2017-10-29 01:00:00    2.0
2017-10-29 02:00:00    3.0
""", back)

    back.index = back.index.tz_localize('UTC')
    assert (ts.index == back.index).all()


142
def test_differential(engine):
143
144
    # instantiate one time serie handler object
    tso = TimeSerie()
145

146
    ts_begin = genserie(datetime(2010, 1, 1), 'D', 10)
147
    tso.insert(engine, ts_begin, 'ts_test', 'test')
148

149
    assert_df("""
150
151
152
153
154
155
156
157
158
159
2010-01-01    0.0
2010-01-02    1.0
2010-01-03    2.0
2010-01-04    3.0
2010-01-05    4.0
2010-01-06    5.0
2010-01-07    6.0
2010-01-08    7.0
2010-01-09    8.0
2010-01-10    9.0
160
""", tso.get(engine, 'ts_test'))
161
162

    # we should detect the emission of a message
163
    tso.insert(engine, ts_begin, 'ts_test', 'babar')
164

165
    assert_df("""
166
167
168
169
170
171
172
173
174
175
2010-01-01    0.0
2010-01-02    1.0
2010-01-03    2.0
2010-01-04    3.0
2010-01-05    4.0
2010-01-06    5.0
2010-01-07    6.0
2010-01-08    7.0
2010-01-09    8.0
2010-01-10    9.0
176
""", tso.get(engine, 'ts_test'))
177
178
179
180

    ts_slight_variation = ts_begin.copy()
    ts_slight_variation.iloc[3] = 0
    ts_slight_variation.iloc[6] = 0
181
    tso.insert(engine, ts_slight_variation, 'ts_test', 'celeste')
182

183
    assert_df("""
184
185
186
187
188
189
190
191
192
193
2010-01-01    0.0
2010-01-02    1.0
2010-01-03    2.0
2010-01-04    0.0
2010-01-05    4.0
2010-01-06    5.0
2010-01-07    0.0
2010-01-08    7.0
2010-01-09    8.0
2010-01-10    9.0
194
""", tso.get(engine, 'ts_test'))
195

196
    ts_longer = genserie(datetime(2010, 1, 3), 'D', 15)
197
198
199
200
    ts_longer.iloc[1] = 2.48
    ts_longer.iloc[3] = 3.14
    ts_longer.iloc[5] = ts_begin.iloc[7]

201
    tso.insert(engine, ts_longer, 'ts_test', 'test')
202

203
    assert_df("""
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
2010-01-01     0.00
2010-01-02     1.00
2010-01-03     0.00
2010-01-04     2.48
2010-01-05     2.00
2010-01-06     3.14
2010-01-07     4.00
2010-01-08     7.00
2010-01-09     6.00
2010-01-10     7.00
2010-01-11     8.00
2010-01-12     9.00
2010-01-13    10.00
2010-01-14    11.00
2010-01-15    12.00
2010-01-16    13.00
2010-01-17    14.00
221
""", tso.get(engine, 'ts_test'))
222
223

    # start testing manual overrides
224
    ts_begin = genserie(datetime(2010, 1, 1), 'D', 5, initval=[2])
225
    ts_begin.loc['2010-01-04'] = -1
226
    tso.insert(engine, ts_begin, 'ts_mixte', 'test')
227
228

    # -1 represents bogus upstream data
229
    assert_df("""
230
231
232
233
234
2010-01-01    2.0
2010-01-02    2.0
2010-01-03    2.0
2010-01-04   -1.0
2010-01-05    2.0
235
""", tso.get(engine, 'ts_mixte'))
236
237

    # refresh all the period + 1 extra data point
238
    ts_more = genserie(datetime(2010, 1, 2), 'D', 5, [2])
239
    ts_more.loc['2010-01-04'] = -1
240
    tso.insert(engine, ts_more, 'ts_mixte', 'test')
241

242
    assert_df("""
243
244
245
246
247
248
2010-01-01    2.0
2010-01-02    2.0
2010-01-03    2.0
2010-01-04   -1.0
2010-01-05    2.0
2010-01-06    2.0
249
""", tso.get(engine, 'ts_mixte'))
250
251

    # just append an extra data point
252
253
    # with no intersection with the previous ts
    ts_one_more = genserie(datetime(2010, 1, 7), 'D', 1, [3])
254
    tso.insert(engine, ts_one_more, 'ts_mixte', 'test')
255

256
    assert_df("""
257
258
259
260
261
262
263
2010-01-01    2.0
2010-01-02    2.0
2010-01-03    2.0
2010-01-04   -1.0
2010-01-05    2.0
2010-01-06    2.0
2010-01-07    3.0
264
""", tso.get(engine, 'ts_mixte'))
265

266
    hist = pd.read_sql('select id, parent from timeserie.ts_test order by id',
267
                        engine)
268
    assert_df("""
269
270
271
272
   id  parent
0   1     NaN
1   2     1.0
2   3     2.0
273
""", hist)
274

275
    hist = pd.read_sql('select id, parent from timeserie.ts_mixte order by id',
276
                        engine)
277
    assert_df("""
278
279
280
281
   id  parent
0   1     NaN
1   2     1.0
2   3     2.0
282
""", hist)
283

284
    allts = pd.read_sql("select name, table_name from registry "
285
                        "where name in ('ts_test', 'ts_mixte')",
286
287
                        engine)

288
    assert_df("""
289
290
291
       name          table_name
0   ts_test   timeserie.ts_test
1  ts_mixte  timeserie.ts_mixte
292
""", allts)
293

294
    assert_df("""
295
296
297
298
299
300
301
2010-01-01    2.0
2010-01-02    2.0
2010-01-03    2.0
2010-01-04   -1.0
2010-01-05    2.0
2010-01-06    2.0
2010-01-07    3.0
302
303
""", tso.get(engine, 'ts_mixte',
             revision_date=datetime.now()))
304
305
306


def test_bad_import(engine):
307
308
309
    # instantiate one time serie handler object
    tso = TimeSerie()

310
311
312
313
314
    # the data were parsed as date by pd.read_json()
    df_result = pd.read_csv(DATADIR / 'test_data.csv')
    df_result['Gas Day'] = df_result['Gas Day'].apply(parser.parse, dayfirst=True, yearfirst=False)
    df_result.set_index('Gas Day', inplace=True)
    ts = df_result['SC']
315
316
317

    tso.insert(engine, ts, 'SND_SC', 'test')
    result = tso.get(engine, 'SND_SC')
318
    assert result.dtype == 'float64'
319
320
321

    # insertion of empty ts
    ts = pd.Series(name='truc', dtype='object')
322
323
    tso.insert(engine, ts, 'empty_ts', 'test')
    assert tso.get(engine, 'empty_ts') is None
324
325
326

    # nan in ts
    # all na
327
    ts = genserie(datetime(2010, 1, 10), 'D', 10, [np.nan], name='truc')
328
329
    tso.insert(engine, ts, 'test_nan', 'test')
    assert tso.get(engine, 'test_nan') is None
330
331
332
333
334

    # mixe na
    ts = pd.Series([np.nan] * 5 + [3] * 5,
                   index=pd.date_range(start=datetime(2010, 1, 10),
                                       freq='D', periods=10), name='truc')
335
336
    tso.insert(engine, ts, 'test_nan', 'test')
    result = tso.get(engine, 'test_nan')
337

338
339
    tso.insert(engine, ts, 'test_nan', 'test')
    result = tso.get(engine, 'test_nan')
340
    assert_df("""
341
342
343
344
345
2010-01-15    3.0
2010-01-16    3.0
2010-01-17    3.0
2010-01-18    3.0
2010-01-19    3.0
346
""", result)
347
348
349

    # get_ts with name not in database

350
    tso.get(engine, 'inexisting_name', 'test')
351
352
353


def test_revision_date(engine):
354
355
356
    # instantiate one time serie handler object
    tso = TimeSerie()

357
358
    idate1 = datetime(2015, 1, 1, 15, 43, 23)
    with tso.newchangeset(engine, 'test', _insertion_date=idate1):
359

360
        ts = genserie(datetime(2010, 1, 4), 'D', 4, [1], name='truc')
361
362
        tso.insert(engine, ts, 'ts_through_time')
        assert idate1 == tso.latest_insertion_date(engine, 'ts_through_time')
363

364
365
    idate2 = datetime(2015, 1, 2, 15, 43, 23)
    with tso.newchangeset(engine, 'test', _insertion_date=idate2):
366

367
        ts = genserie(datetime(2010, 1, 4), 'D', 4, [2], name='truc')
368
369
        tso.insert(engine, ts, 'ts_through_time')
        assert idate2 == tso.latest_insertion_date(engine, 'ts_through_time')
370

371
372
    idate3 = datetime(2015, 1, 3, 15, 43, 23)
    with tso.newchangeset(engine, 'test', _insertion_date=idate3):
373

374
        ts = genserie(datetime(2010, 1, 4), 'D', 4, [3], name='truc')
375
376
        tso.insert(engine, ts, 'ts_through_time')
        assert idate3 == tso.latest_insertion_date(engine, 'ts_through_time')
377

378
    ts = tso.get(engine, 'ts_through_time')
379

380
    assert_df("""
381
382
383
384
2010-01-04    3.0
2010-01-05    3.0
2010-01-06    3.0
2010-01-07    3.0
385
""", ts)
386

387
388
    ts = tso.get(engine, 'ts_through_time',
                 revision_date=datetime(2015, 1, 2, 18, 43, 23) )
389

390
    assert_df("""
391
392
393
394
2010-01-04    2.0
2010-01-05    2.0
2010-01-06    2.0
2010-01-07    2.0
395
""", ts)
396

397
398
    ts = tso.get(engine, 'ts_through_time',
                 revision_date=datetime(2015, 1, 1, 18, 43, 23))
399

400
    assert_df("""
401
402
403
404
2010-01-04    1.0
2010-01-05    1.0
2010-01-06    1.0
2010-01-07    1.0
405
""", ts)
406

407
408
    ts = tso.get(engine, 'ts_through_time',
                 revision_date=datetime(2014, 1, 1, 18, 43, 23))
409
410
411

    assert ts is None

412
413
414

def test_snapshots(engine):
    tso = TimeSerie()
415
    tso._snapshot_interval = 4
416

417
    with engine.connect() as cn:
418
        for tscount in range(1, 11):
419
            ts = genserie(datetime(2015, 1, 1), 'D', tscount, [1])
420
            diff = tso.insert(cn, ts, 'growing', 'babar')
421
422
423
424
            assert diff.index[0] == diff.index[-1] == ts.index[-1]

    diff = tso.insert(engine, ts, 'growing', 'babar')
    assert diff is None
425

426
427
    df = pd.read_sql("select id from timeserie.growing where snapshot is not null",
                     engine)
428
    assert_df("""
429
430
   id
0   1
431
432
433
1   4
2   8
3  10
434
""", df)
435
436

    ts = tso.get(engine, 'growing')
437
    assert_df("""
438
439
440
441
442
443
444
445
446
447
2015-01-01    1.0
2015-01-02    1.0
2015-01-03    1.0
2015-01-04    1.0
2015-01-05    1.0
2015-01-06    1.0
2015-01-07    1.0
2015-01-08    1.0
2015-01-09    1.0
2015-01-10    1.0
448
""", ts)
449

450
    df = pd.read_sql("select id, diff, snapshot from timeserie.growing order by id", engine)
451
452
453
    for attr in ('diff', 'snapshot'):
        df[attr] = df[attr].apply(lambda x: 0 if x is None else len(x))

454
    assert_df("""
455
   id  diff  snapshot
Arnaud Campeas's avatar
Arnaud Campeas committed
456
457
458
459
460
461
462
463
464
465
0   1     0        32
1   2    32         0
2   3    32         0
3   4    32       125
4   5    32         0
5   6    32         0
6   7    32         0
7   8    32       249
8   9    32         0
9  10    32       311
466
""", df)
467
468
469
470
471

    table = tso._get_ts_table(engine, 'growing')
    snapid, snap = tso._find_snapshot(engine, table, ())
    assert snapid == 10
    assert (ts == snap).all()
472
473
474
475
476


def test_deletion(engine):
    tso = TimeSerie()

477
478
    ts_begin = genserie(datetime(2010, 1, 1), 'D', 11)
    ts_begin.iloc[-1] = np.nan
479
480
    tso.insert(engine, ts_begin, 'ts_del', 'test')

481
    ts = tso._build_snapshot_upto(engine, tso._get_ts_table(engine, 'ts_del'))
482
    assert ts.iloc[-1] == 9.0
483

484
    ts_begin.iloc[0] = np.nan
485
    ts_begin.iloc[3] = np.nan
486
487
488

    tso.insert(engine, ts_begin, 'ts_del', 'test')

489
    assert_df("""
490
491
492
493
494
495
496
2010-01-02    1.0
2010-01-03    2.0
2010-01-05    4.0
2010-01-06    5.0
2010-01-07    6.0
2010-01-08    7.0
2010-01-09    8.0
497
498
2010-01-10    9.0
""", tso.get(engine, 'ts_del'))
499

500
501
502
503
504
    ts2 = tso.get(engine, 'ts_del',
                 # force snapshot reconstruction feature
                 revision_date=datetime(2038, 1, 1))
    assert (tso.get(engine, 'ts_del') == ts2).all()

505
506
507
508
509
    ts_begin.iloc[0] = 42
    ts_begin.iloc[3] = 23

    tso.insert(engine, ts_begin, 'ts_del', 'test')

510
    assert_df("""
511
512
513
514
515
516
517
518
519
2010-01-01    42.0
2010-01-02     1.0
2010-01-03     2.0
2010-01-04    23.0
2010-01-05     4.0
2010-01-06     5.0
2010-01-07     6.0
2010-01-08     7.0
2010-01-09     8.0
520
521
2010-01-10     9.0
""", tso.get(engine, 'ts_del'))
522
523
524

    # now with string!

525
    ts_string = genserie(datetime(2010, 1, 1), 'D', 10, ['machin'])
526
527
528
529
530
531
    tso.insert(engine, ts_string, 'ts_string_del', 'test')

    ts_string[4] = None
    ts_string[5] = None

    tso.insert(engine, ts_string, 'ts_string_del', 'test')
532
    assert_df("""
533
534
535
536
537
538
539
540
2010-01-01    machin
2010-01-02    machin
2010-01-03    machin
2010-01-04    machin
2010-01-07    machin
2010-01-08    machin
2010-01-09    machin
2010-01-10    machin
541
""", tso.get(engine, 'ts_string_del'))
542
543
544
545
546

    ts_string[4] = 'truc'
    ts_string[6] = 'truc'

    tso.insert(engine, ts_string, 'ts_string_del', 'test')
547
    assert_df("""
548
549
550
551
552
553
554
555
2010-01-01    machin
2010-01-02    machin
2010-01-03    machin
2010-01-04    machin
2010-01-05      truc
2010-01-07      truc
2010-01-08    machin
2010-01-09    machin
556
557
2010-01-10    machin
""", tso.get(engine, 'ts_string_del'))
558

559
560
561
562
563
564
    ts_string[ts_string.index] = np.nan
    tso.insert(engine, ts_string, 'ts_string_del', 'test')

    erased = tso.get(engine, 'ts_string_del')
    assert len(erased) == 0

565
566
    # first insertion with only nan

567
    ts_begin = genserie(datetime(2010, 1, 1), 'D', 10, [np.nan])
568
569
570
    tso.insert(engine, ts_begin, 'ts_null', 'test')

    assert tso.get(engine, 'ts_null') is None
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709


def test_multi_index(engine):
    tso = TimeSerie()

    appdate_0 = pd.DatetimeIndex(start=datetime(2015, 1, 1),
                                 end=datetime(2015, 1, 2),
                                 freq='D').values
    pubdate_0 = [pd.datetime(2015, 1, 11, 12, 0, 0)] * 2
    insertion_date_0 = [pd.datetime(2015, 1, 11, 12, 30, 0)] * 2

    multi = [
        appdate_0,
        np.array(pubdate_0),
        np.array(insertion_date_0)
    ]

    ts_multi = pd.Series(range(2), index=multi)
    ts_multi.index.rename(['b', 'c', 'a'], inplace=True)

    tso.insert(engine, ts_multi, 'ts_multi_simple', 'test')

    ts = tso.get(engine, 'ts_multi_simple')
    assert_df("""
                                                    ts_multi_simple
a                   b          c                                   
2015-01-11 12:30:00 2015-01-01 2015-01-11 12:00:00                0
                    2015-01-02 2015-01-11 12:00:00                1
""", pd.DataFrame(ts))

    diff = tso.insert(engine, ts_multi, 'ts_multi_simple', 'test')
    assert diff is None

    ts_multi_2 = pd.Series([0, 2], index=multi)
    ts_multi_2.index.rename(['b', 'c', 'a'], inplace=True)

    tso.insert(engine, ts_multi_2, 'ts_multi_simple', 'test')
    ts = tso.get(engine, 'ts_multi_simple')

    assert_df("""
                                                    ts_multi_simple
a                   b          c                                   
2015-01-11 12:30:00 2015-01-01 2015-01-11 12:00:00                0
                    2015-01-02 2015-01-11 12:00:00                2
""", pd.DataFrame(ts))

    # bigger ts
    appdate_0 = pd.DatetimeIndex(start=datetime(2015, 1, 1),
                                 end=datetime(2015, 1, 4),
                                 freq='D').values
    pubdate_0 = [pd.datetime(2015, 1, 11, 12, 0, 0)] * 4
    insertion_date_0 = [pd.datetime(2015, 1, 11, 12, 30, 0)] * 4

    appdate_1 = pd.DatetimeIndex(start=datetime(2015, 1, 1),
                                 end=datetime(2015, 1, 4),
                                 freq='D').values

    pubdate_1 = [pd.datetime(2015, 1, 21, 12, 0, 0)] * 4
    insertion_date_1 = [pd.datetime(2015, 1, 21, 12, 30, 0)] * 4

    multi = [
        np.concatenate([appdate_0, appdate_1]),
        np.array(pubdate_0 + pubdate_1),
        np.array(insertion_date_0 + insertion_date_1)
    ]

    ts_multi = pd.Series(range(8), index=multi)
    ts_multi.index.rename(['a', 'c', 'b'], inplace=True)

    tso.insert(engine, ts_multi, 'ts_multi', 'test')
    ts = tso.get(engine, 'ts_multi')

    assert_df("""
                                                    ts_multi
a          b                   c                            
2015-01-01 2015-01-11 12:30:00 2015-01-11 12:00:00         0
           2015-01-21 12:30:00 2015-01-21 12:00:00         4
2015-01-02 2015-01-11 12:30:00 2015-01-11 12:00:00         1
           2015-01-21 12:30:00 2015-01-21 12:00:00         5
2015-01-03 2015-01-11 12:30:00 2015-01-11 12:00:00         2
           2015-01-21 12:30:00 2015-01-21 12:00:00         6
2015-01-04 2015-01-11 12:30:00 2015-01-11 12:00:00         3
           2015-01-21 12:30:00 2015-01-21 12:00:00         7
    """, pd.DataFrame(ts.sort_index()))
    # Note: the columnns are returned according to the alphabetic order

    appdate_2 = pd.DatetimeIndex(start=datetime(2015, 1, 1),
                                 end=datetime(2015, 1, 4),
                                 freq='D').values
    pubdate_2 = [pd.datetime(2015, 1, 31, 12, 0, 0)] * 4
    insertion_date_2 = [pd.datetime(2015, 1, 31, 12, 30, 0)] * 4

    multi_2 = [
        np.concatenate([appdate_1, appdate_2]),
        np.array(pubdate_1 + pubdate_2),
        np.array(insertion_date_1 + insertion_date_2)
    ]

    ts_multi_2 = pd.Series([4] * 8, index=multi_2)
    ts_multi_2.index.rename(['a', 'c', 'b'], inplace=True)

    # A second ts is inserted with some index in common with the first
    # one: appdate_1, pubdate_1,and insertion_date_1. The value is set
    # at 4, which matches the previous value of the "2015-01-01" point.

    diff = tso.insert(engine, ts_multi_2, 'ts_multi', 'test')
    assert_df("""
                                                    ts_multi
a          b                   c                            
2015-01-01 2015-01-31 12:30:00 2015-01-31 12:00:00       4.0
2015-01-02 2015-01-21 12:30:00 2015-01-21 12:00:00       4.0
           2015-01-31 12:30:00 2015-01-31 12:00:00       4.0
2015-01-03 2015-01-21 12:30:00 2015-01-21 12:00:00       4.0
           2015-01-31 12:30:00 2015-01-31 12:00:00       4.0
2015-01-04 2015-01-21 12:30:00 2015-01-21 12:00:00       4.0
           2015-01-31 12:30:00 2015-01-31 12:00:00       4.0
        """, pd.DataFrame(diff.sort_index()))
    # the differential skips a value for "2015-01-01"
    # which does not change from the previous ts

    ts = tso.get(engine, 'ts_multi')
    assert_df("""
                                                    ts_multi
a          b                   c                            
2015-01-01 2015-01-11 12:30:00 2015-01-11 12:00:00         0
           2015-01-21 12:30:00 2015-01-21 12:00:00         4
           2015-01-31 12:30:00 2015-01-31 12:00:00         4
2015-01-02 2015-01-11 12:30:00 2015-01-11 12:00:00         1
           2015-01-21 12:30:00 2015-01-21 12:00:00         4
           2015-01-31 12:30:00 2015-01-31 12:00:00         4
2015-01-03 2015-01-11 12:30:00 2015-01-11 12:00:00         2
           2015-01-21 12:30:00 2015-01-21 12:00:00         4
           2015-01-31 12:30:00 2015-01-31 12:00:00         4
2015-01-04 2015-01-11 12:30:00 2015-01-11 12:00:00         3
           2015-01-21 12:30:00 2015-01-21 12:00:00         4
           2015-01-31 12:30:00 2015-01-31 12:00:00         4
        """, pd.DataFrame(ts.sort_index()))

    # the result ts have now 3 values for each point in 'a'
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737


def test_add_na(engine):
    tso = TimeSerie()

    # a serie of NaNs won't be insert in base
    # in case of first insertion
    ts_nan = genserie(datetime(2010, 1, 1), 'D', 5)
    ts_nan[[True] * len(ts_nan)] = np.nan

    diff = tso.insert(engine, ts_nan, 'ts_add_na', 'test')
    assert diff is None
    result = tso.get(engine, 'ts_add_na')
    assert result is None

    # in case of insertion in existing data
    ts_begin = genserie(datetime(2010, 1, 1), 'D', 5)
    tso.insert(engine, ts_begin, 'ts_add_na', 'test')

    ts_nan = genserie(datetime(2010, 1, 6), 'D', 5)
    ts_nan[[True] * len(ts_nan)] = np.nan
    ts_nan = pd.concat([ts_begin, ts_nan])

    diff = tso.insert(engine, ts_nan, 'ts_add_na', 'test')
    assert diff is None

    result = tso.get(engine, 'ts_add_na')
    assert len(result) == 5