Guest User

Keras LSTM time series model

a guest
Feb 25th, 2018
721
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. '''Example script showing how to use a stateful LSTM model
  2. and how its stateless counterpart performs.
  3. More documentation about the Keras LSTM model can be found at
  4. https://keras.io/layers/recurrent/#lstm
  5. The models are trained on an input/output pair, where
  6. the input is a generated uniformly distributed
  7. random sequence of length = "input_len",
  8. and the output is a moving average of the input with window length = "tsteps".
  9. Both "input_len" and "tsteps" are defined in the "editable parameters" section.
  10. A larger "tsteps" value means that the LSTM will need more memory
  11. to figure out the input-output relationship.
  12. This memory length is controlled by the "lahead" variable (more details below).
  13. The rest of the parameters are:
  14. - input_len: the length of the generated input sequence
  15. - lahead: the input sequence length that the LSTM
  16.  is trained on for each output point
  17. - batch_size, epochs: same parameters as in the model.fit(...) function
  18. When lahead > 1, the model input is preprocessed to a "rolling window view"
  19. of the data, with the window length = "lahead".
  20. This is similar to sklearn's "view_as_windows"
  21. with "window_shape" being a single number
  22. Ref: http://scikit-image.org/docs/0.10.x/api/skimage.util.html#view-as-windows
  23. When lahead < tsteps, only the stateful LSTM converges because its
  24. statefulness allows it to see beyond the capability that lahead
  25. gave it to fit the n-point average. The stateless LSTM does not have
  26. this capability, and hence is limited by its "lahead" parameter,
  27. which is not sufficient to see the n-point average.
  28. When lahead >= tsteps, both the stateful and stateless LSTM converge.
  29. '''
  30. from __future__ import print_function
  31. import numpy as np
  32. import matplotlib.pyplot as plt
  33. import pandas as pd
  34. from keras.models import Sequential
  35. from keras.layers import Dense, LSTM
  36.  
  37. # ----------------------------------------------------------
  38. # EDITABLE PARAMETERS
  39. # Read the documentation in the script head for more details
  40. # ----------------------------------------------------------
  41.  
  42. # length of input
  43. input_len = 2000
  44.  
  45. # The window length of the moving average used to generate
  46. # the output from the input in the input/output pair used
  47. # to train the LSTM
  48. # e.g. if tsteps=2 and input=[1, 2, 3, 4, 5],
  49. #      then output=[1.5, 2.5, 3.5, 4.5]
  50. tsteps = 1
  51.  
  52. # The input sequence length that the LSTM is trained on for each output point
  53. lahead = 1
  54.  
  55. # training parameters passed to "model.fit(...)"
  56. batch_size = 1
  57. epochs = 10
  58.  
  59. # ------------
  60. # MAIN PROGRAM
  61. # ------------
  62.  
  63. print("*" * 33)
  64. if lahead >= tsteps:
  65.     print("STATELESS LSTM WILL ALSO CONVERGE")
  66. else:
  67.     print("STATELESS LSTM WILL NOT CONVERGE")
  68. print("*" * 33)
  69.  
  70. def gen_uniform_amp():
  71.     """Generates uniform random data between
  72.    -amp and +amp
  73.    and of length xn
  74.    Arguments:
  75.        amp: maximum/minimum range of uniform data
  76.        xn: length of series
  77.    """
  78.     #data_input = np.random.uniform(-1 * amp, +1 * amp, xn)
  79.     data_input = np.fromfile('CSV4.CSV',dtype=float,count=input_len,sep='\n')
  80.     data_input = pd.DataFrame(data_input)
  81.     return data_input
  82.  
  83. # Since the output is a moving average of the input,
  84. # the first few points of output will be NaN
  85. # and will be dropped from the generated data
  86. # before training the LSTM.
  87. # Also, when lahead > 1,
  88. # the preprocessing step later of "rolling window view"
  89. # will also cause some points to be lost.
  90. # For aesthetic reasons,
  91. # in order to maintain generated data length = input_len after pre-processing,
  92. # add a few points to account for the values that will be lost.
  93. to_drop = max(tsteps - 1, lahead - 1)
  94. data_input = gen_uniform_amp()
  95.  
  96.  
  97. # set the target to be a N-point average of the input
  98. #expected_output = data_input.rolling(window=tsteps, center=False).mean()
  99. expected_output = data_input[1:]
  100. expected_output= expected_output.append([1])
  101.  
  102. # when lahead > 1, need to convert the input to "rolling window view"
  103. # https://docs.scipy.org/doc/numpy/reference/generated/numpy.repeat.html
  104. if lahead > 1:
  105.     data_input = np.repeat(data_input.values, repeats=lahead, axis=1)
  106.     data_input = pd.DataFrame(data_input)
  107.     for i, c in enumerate(data_input.columns):
  108.         data_input[c] = data_input[c].shift(i)
  109.  
  110. # drop the nan
  111. expected_output = expected_output[to_drop:]
  112. data_input = data_input[to_drop:]
  113.  
  114. print('Input shape:', data_input.shape)
  115. print('Output shape:', expected_output.shape)
  116. print('Input head: ')
  117. print(data_input.head())
  118. print('Output head: ')
  119. print(expected_output.head())
  120. print('Input tail: ')
  121. print(data_input.tail())
  122. print('Output tail: ')
  123. print(expected_output.tail())
  124.  
  125. print('Plotting input and expected output')
  126. plt.plot(data_input[0][:50], '.')
  127. plt.plot(expected_output[0][:50], '-')
  128. plt.legend(['Input', 'Expected output'])
  129. plt.title('Input')
  130. plt.show()
  131.  
  132.  
  133. def create_model(stateful):
  134.     model = Sequential()
  135.     model.add(LSTM(100,
  136.               input_shape=(lahead, 1),
  137.               batch_size=batch_size,
  138.               stateful=stateful))
  139.     model.add(Dense(1))
  140.     model.compile(loss='mse', optimizer='adam')
  141.     return model
  142.  
  143. print('Creating Stateful Model...')
  144. model_stateful = create_model(stateful=True)
  145.  
  146.  
  147. # split train/test data
  148. def split_data(x, y, ratio=0.05):
  149.     to_train = int(input_len * ratio)
  150.     # tweak to match with batch_size
  151.     to_train -= to_train % batch_size
  152.  
  153.     x_train = x[:to_train]
  154.     y_train = y[:to_train]
  155.     x_test = x[to_train:]
  156.     y_test = y[to_train:]
  157.  
  158.     # tweak to match with batch_size
  159.     to_drop = x.shape[0] % batch_size
  160.     if to_drop > 0:
  161.         x_test = x_test[:-1 * to_drop]
  162.         y_test = y_test[:-1 * to_drop]
  163.  
  164.     # some reshaping
  165.     reshape_3 = lambda x: x.values.reshape((x.shape[0], x.shape[1], 1))
  166.     x_train = reshape_3(x_train)
  167.     x_test = reshape_3(x_test)
  168.  
  169.     reshape_2 = lambda x: x.values.reshape((x.shape[0], 1))
  170.     y_train = reshape_2(y_train)
  171.     y_test = reshape_2(y_test)
  172.  
  173.     return (x_train, y_train), (x_test, y_test)
  174.  
  175.  
  176. (x_train, y_train), (x_test, y_test) = split_data(data_input, expected_output)
  177. print('x_train.shape: ', x_train.shape)
  178. print('y_train.shape: ', y_train.shape)
  179. print('x_test.shape: ', x_test.shape)
  180. print('y_test.shape: ', y_test.shape)
  181.  
  182. print('Training')
  183. for i in range(epochs):
  184.     print('Epoch', i + 1, '/', epochs)
  185.     # Note that the last state for sample i in a batch will
  186.     # be used as initial state for sample i in the next batch.
  187.     # Thus we are simultaneously training on batch_size series with
  188.     # lower resolution than the original series contained in data_input.
  189.     # Each of these series are offset by one step and can be
  190.     # extracted with data_input[i::batch_size].
  191.     model_stateful.fit(x_train,
  192.                        y_train,
  193.                        batch_size=batch_size,
  194.                        epochs=1,
  195.                        verbose=1,
  196.                        validation_data=(x_test, y_test),
  197.                        shuffle=False)
  198.     model_stateful.reset_states()
  199.  
  200. print('Predicting')
  201. predicted_stateful = model_stateful.predict(x_test, batch_size=batch_size)
  202.  
  203. print('Creating Stateless Model...')
  204. model_stateless = create_model(stateful=False)
  205.  
  206. print('Training')
  207. model_stateless.fit(x_train,
  208.                     y_train,
  209.                     batch_size=batch_size,
  210.                     epochs=epochs,
  211.                     verbose=1,
  212.                     validation_data=(x_test, y_test),
  213.                     shuffle=False)
  214.  
  215. print('Predicting')
  216. predicted_stateless = model_stateless.predict(x_test, batch_size=batch_size)
  217.  
  218. # ----------------------------
  219.  
  220. print('Plotting Resultz')
  221. print((y_test - predicted_stateless).flatten())
  222. print(y_test)
  223. plt.subplot(3, 1, 1)
  224. plt.plot(y_test)
  225. plt.title('Expected')
  226. plt.subplot(3, 1, 2)
  227. # drop the first "tsteps-1" because it is not possible to predict them
  228. # since the "previous" timesteps to use do not exist
  229. plt.plot((predicted_stateful).flatten()[tsteps - 1:])
  230. plt.title('Stateful: Predicted')
  231. plt.subplot(3, 1, 3)
  232. plt.plot((predicted_stateless).flatten())
  233. plt.title('Stateless: Predicted')
  234. plt.show()
RAW Paste Data