View difference between Paste ID: hJFR7HbA and cfpqUmef
SHOW: | | - or go back to the newest paste.
1
#!/usr/bin/env python2
2
# -*- coding: utf-8 -*-
3
from __future__ import division
4
import numpy as np
5
import pandas as pd       # pip install pandas
6
import rosbag_pandas      # pip install rosbag_pandas
7
8
# Fast Introduction to pandas:
9
# http://www.gregreda.com/2013/10/26/intro-to-pandas-data-structures/
10
#
11
# tl;dr: pandas extends numpy array with named columns and a time index
12
13
14
# Example usage:
15
# dataframe = rosbag_pandas.bag_to_dataframe('2014-08-20-17-37-47.bag')
16-
# ridf=reindex_df(dataframe, dataframe.uC_time__data)
16+
# ridf=reindex_dataframe(dataframe, dataframe.uC_time__data)
17
# print ridf
18
# plt.plot(ridf.index, ridf.sensors_imu_data_raw__angular_velocity_z)
19
20
21
22
def reindex_dataframe(dataframe, reference_series, idx_start_from_zero=True):
23
    '''Reindex the pandas dataframe to only contain the timestamps from reference_series. 
24
    The closest values (time-wise) for each column are used. 
25
    idx_start_from_zero specifies whether the reindexed timestamps shall start from zero.
26
    Only columns with float64 values are include in the reindexed dataframe. 
27
    '''
28
    if type(reference_series) is str:
29
        reference_series = dataframe[reference_series]
30
    
31
    idx = notnull_keys_sec(reference_series)
32
    n = idx.shape[0]
33
    data = dict()
34
    for col in dataframe.columns:
35
        col = df.columns[i]
36
        data[col] = np.empty(shape=(n))
37
        series = df[col]
38
        series = series[series.notnull()]
39
        values = series.values
40
        keys = datetime_index_to_sec(series.keys())
41
        values = get_closest_values(idx,keys,values)
42
        data[col] = values
43
44
    if idx_start_from_zero:
45
        idx -= idx[0]
46
    
47
    reindexed = pd.DataFrame(data,idx)
48
    
49
    return reindexed
50
51
52
def datetime_index_to_sec(dti):
53
    '''Converts pandas DatetimeIndex into seconds.
54
    This may only work for DatetimeIndex from rosbag_pandas. Untested on other.'''
55
    return dti.map(lambda x:x.value/1e9)
56
57
def notnull_values(series):
58
    '''Returns all not null values in the panda Series'''
59
    return series[series.notnull()].values
60
61
def notnull_keys(series):
62
    '''Returns all not null keys in the panda Series'''
63
    return series[series.notnull()].keys()
64
65-
        notnull_values(series))
65+
66
    '''Returns all not null keys in the panda Series as seconds'''
67
    return datetime_index_to_sec(notnull_keys(series))
68
69
def get_closest_values(desired_timestamps, timestamps, data):
70
    '''Returns the values in data whose timestamps are closest to the desired_timestamps.
71
    Assumes sorted desired_timestamps and uniform dt.'''
72
    assert len(desired_timestamps.shape)==1
73
    assert len(timestamps.shape)==1
74
    assert len(data.shape)==1
75
    assert timestamps.shape[0] == data.shape[0]
76
    
77
    n = timestamps.shape[0]
78
    dt = np.diff(timestamps).mean()
79
    start = timestamps[0]
80
81
    values = np.empty_like(desired_timestamps)
82
    idx = (desired_timestamps-start)/dt
83
    idx = np.round(idx)
84
    idx[idx<0]=0
85
    idx[idx>n-1]=n-1
86
    idx=idx.astype('int')
87
    values = data[idx]
88
89
    return values
90
91
def get_closest_values_from_df(reference_series, series):
92
    '''Returns the values in series whose timestamps are closest to the timestamps in the reference_series.
93
    Assumes uniform dt in reference_series.'''
94-
    return reindexed
94+
95
        notnull_keys_sec(reference_series),
96
        notnull_keys_sec(series),
97
        notnull_values(series))