model = Sequential()

model.add(
    TimeDistributed(
        Conv2D(64, (3, 3), activation='relu'),
        input_shape=(config.N_FRAMES_IN_SEQUENCE, config.IMAGE_H, config.IMAGE_W, config.N_CHANNELS)
    )
)
model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(1, 1))))

model.add(TimeDistributed(Conv2D(128, (4, 4), activation='relu')))
model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2))))

model.add(TimeDistributed(Conv2D(256, (4, 4), activation='relu')))
model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2))))

model.add(TimeDistributed(Flatten()))
model.add(Dropout(0.5))

model.add(LSTM(256, return_sequences=False, dropout=0.5))

model.add(Dense(config.N_LANDMARKS * 2, activation='linear'))

optimizer = Adadelta()
model.compile(optimizer=optimizer, loss='mae')

------------------------------------------------------------
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
time_distributed_1 (TimeDist (None, 3, 126, 126, 64)   1792
_________________________________________________________________
time_distributed_2 (TimeDist (None, 3, 125, 125, 64)   0
_________________________________________________________________
time_distributed_3 (TimeDist (None, 3, 122, 122, 128)  131200
_________________________________________________________________
time_distributed_4 (TimeDist (None, 3, 61, 61, 128)    0
_________________________________________________________________
time_distributed_5 (TimeDist (None, 3, 58, 58, 256)    524544
_________________________________________________________________
time_distributed_6 (TimeDist (None, 3, 29, 29, 256)    0
_________________________________________________________________
time_distributed_7 (TimeDist (None, 3, 215296)         0
_________________________________________________________________
dropout_1 (Dropout)          (None, 3, 215296)         0
_________________________________________________________________
lstm_1 (LSTM)                (None, 256)               220726272
_________________________________________________________________
dense_1 (Dense)              (None, 136)               34952
=================================================================
Total params: 221,418,760
Trainable params: 221,418,760
Non-trainable params: 0
_________________________________________________________________

inputs = Input(shape=(config.N_FRAMES_IN_SEQUENCE, config.IMAGE_H, config.IMAGE_W, config.N_CHANNELS))

x = TimeDistributed(Conv2D(filters=32, kernel_size=(3, 3), padding='same', activation='relu'))(inputs)
x = TimeDistributed(MaxPooling2D(pool_size=(2, 2)))(x)
x = TimeDistributed(MaxPooling2D(pool_size=(2, 2)))(x)
x = TimeDistributed(MaxPooling2D(pool_size=(2, 2)))(x)
x = TimeDistributed(MaxPooling2D(pool_size=(2, 2)))(x)
x = TimeDistributed(MaxPooling2D(pool_size=(2, 2)))(x)
x = LSTM(config.N_LANDMARKS * 2)(x)

model = Model(inputs=inputs, outputs=x)
optimizer = Adadelta()
model.compile(optimizer=optimizer, loss='mae')

ValueError: Input 0 is incompatible with layer lstm_1: expected ndim=3, found ndim=5