SHOW:
|
|
- or go back to the newest paste.
1 | #!/usr/bin/env python2 | |
2 | # -*- coding: utf-8 -*- | |
3 | from __future__ import division | |
4 | import numpy as np | |
5 | import pandas as pd # pip install pandas | |
6 | import rosbag_pandas # pip install rosbag_pandas | |
7 | ||
8 | # Fast Introduction to pandas: | |
9 | # http://www.gregreda.com/2013/10/26/intro-to-pandas-data-structures/ | |
10 | # | |
11 | # tl;dr: pandas extends numpy array with named columns and a time index | |
12 | ||
13 | ||
14 | # Example usage: | |
15 | # dataframe = rosbag_pandas.bag_to_dataframe('2014-08-20-17-37-47.bag') | |
16 | - | # ridf=reindex_df(dataframe, dataframe.uC_time__data) |
16 | + | # ridf=reindex_dataframe(dataframe, dataframe.uC_time__data) |
17 | # print ridf | |
18 | # plt.plot(ridf.index, ridf.sensors_imu_data_raw__angular_velocity_z) | |
19 | ||
20 | ||
21 | ||
22 | def reindex_dataframe(dataframe, reference_series, idx_start_from_zero=True): | |
23 | '''Reindex the pandas dataframe to only contain the timestamps from reference_series. | |
24 | The closest values (time-wise) for each column are used. | |
25 | idx_start_from_zero specifies whether the reindexed timestamps shall start from zero. | |
26 | Only columns with float64 values are include in the reindexed dataframe. | |
27 | ''' | |
28 | if type(reference_series) is str: | |
29 | reference_series = dataframe[reference_series] | |
30 | ||
31 | idx = notnull_keys_sec(reference_series) | |
32 | n = idx.shape[0] | |
33 | data = dict() | |
34 | for col in dataframe.columns: | |
35 | col = df.columns[i] | |
36 | data[col] = np.empty(shape=(n)) | |
37 | series = df[col] | |
38 | series = series[series.notnull()] | |
39 | values = series.values | |
40 | keys = datetime_index_to_sec(series.keys()) | |
41 | values = get_closest_values(idx,keys,values) | |
42 | data[col] = values | |
43 | ||
44 | if idx_start_from_zero: | |
45 | idx -= idx[0] | |
46 | ||
47 | reindexed = pd.DataFrame(data,idx) | |
48 | ||
49 | return reindexed | |
50 | ||
51 | ||
52 | def datetime_index_to_sec(dti): | |
53 | '''Converts pandas DatetimeIndex into seconds. | |
54 | This may only work for DatetimeIndex from rosbag_pandas. Untested on other.''' | |
55 | return dti.map(lambda x:x.value/1e9) | |
56 | ||
57 | def notnull_values(series): | |
58 | '''Returns all not null values in the panda Series''' | |
59 | return series[series.notnull()].values | |
60 | ||
61 | def notnull_keys(series): | |
62 | '''Returns all not null keys in the panda Series''' | |
63 | return series[series.notnull()].keys() | |
64 | ||
65 | - | notnull_values(series)) |
65 | + | |
66 | '''Returns all not null keys in the panda Series as seconds''' | |
67 | return datetime_index_to_sec(notnull_keys(series)) | |
68 | ||
69 | def get_closest_values(desired_timestamps, timestamps, data): | |
70 | '''Returns the values in data whose timestamps are closest to the desired_timestamps. | |
71 | Assumes sorted desired_timestamps and uniform dt.''' | |
72 | assert len(desired_timestamps.shape)==1 | |
73 | assert len(timestamps.shape)==1 | |
74 | assert len(data.shape)==1 | |
75 | assert timestamps.shape[0] == data.shape[0] | |
76 | ||
77 | n = timestamps.shape[0] | |
78 | dt = np.diff(timestamps).mean() | |
79 | start = timestamps[0] | |
80 | ||
81 | values = np.empty_like(desired_timestamps) | |
82 | idx = (desired_timestamps-start)/dt | |
83 | idx = np.round(idx) | |
84 | idx[idx<0]=0 | |
85 | idx[idx>n-1]=n-1 | |
86 | idx=idx.astype('int') | |
87 | values = data[idx] | |
88 | ||
89 | return values | |
90 | ||
91 | def get_closest_values_from_df(reference_series, series): | |
92 | '''Returns the values in series whose timestamps are closest to the timestamps in the reference_series. | |
93 | Assumes uniform dt in reference_series.''' | |
94 | - | return reindexed |
94 | + | |
95 | notnull_keys_sec(reference_series), | |
96 | notnull_keys_sec(series), | |
97 | notnull_values(series)) |