{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hgaiser/.local/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# show images inline\n",
    "%matplotlib inline\n",
    "\n",
    "# automatically reload modules when they have changed\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# import keras\n",
    "import keras\n",
    "\n",
    "# import keras_retinanet\n",
    "from keras_retinanet.models.resnet import custom_objects\n",
    "from keras_retinanet.utils.image import read_image_bgr, preprocess_image, resize_image\n",
    "from keras_retinanet.utils.transform import random_transform_generator\n",
    "from keras_retinanet.preprocessing.coco import CocoGenerator\n",
    "\n",
    "# import miscellaneous modules\n",
    "import matplotlib.pyplot as plt\n",
    "import cv2\n",
    "import os\n",
    "import numpy as np\n",
    "import time\n",
    "\n",
    "# set tf backend to allow memory to grow, instead of claiming everything\n",
    "import tensorflow as tf\n",
    "\n",
    "def get_session():\n",
    "    config = tf.ConfigProto()\n",
    "    config.gpu_options.allow_growth = True\n",
    "    return tf.Session(config=config)\n",
    "\n",
    "# use this environment flag to change which GPU to use\n",
    "#os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
    "\n",
    "# set the modified tf session as backend in keras\n",
    "keras.backend.tensorflow_backend.set_session(get_session())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hgaiser/.local/lib/python3.6/site-packages/keras/models.py:274: UserWarning: Output \"non_maximum_suppression_1\" missing from loss dictionary. We assume this was done on purpose, and we will not be expecting any data to be passed to \"non_maximum_suppression_1\" during training.\n",
      "  sample_weight_mode=sample_weight_mode)\n"
     ]
    }
   ],
   "source": [
    "# adjust this to point to your downloaded/trained model\n",
    "model_path = os.path.join('snapshots', 'resnet50_coco_best_v1.2.2.h5')\n",
    "\n",
    "# load retinanet model\n",
    "model = keras.models.load_model(model_path, custom_objects=custom_objects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=9.34s)\n",
      "creating index...\n",
      "index created!\n"
     ]
    }
   ],
   "source": [
    "transform_generator = random_transform_generator(flip_x_chance=0.5)\n",
    "\n",
    "train_generator = CocoGenerator(\n",
    "    '/srv/datasets/COCO/',\n",
    "    'train2017',\n",
    "    transform_generator=transform_generator,\n",
    "    batch_size=8\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8, 600, 899, 3)\n"
     ]
    }
   ],
   "source": [
    "inputs, targets = next(train_generator)\n",
    "print(inputs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.84327817, 0.72799397, 0.115284175]\n"
     ]
    }
   ],
   "source": [
    "loss = model.test_on_batch(inputs, targets)\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8432783745229244\n",
      "0.7279941476881504\n",
      "0.11528422217816114\n"
     ]
    }
   ],
   "source": [
    "total_losses = []\n",
    "regression_losses = []\n",
    "classification_losses = []\n",
    "for i in range(8):\n",
    "    inputs_ = inputs[i:i+1, ...]\n",
    "    targets_ = [targets[0][i:i+1, ...], targets[1][i:i+1, ...]]\n",
    "\n",
    "    loss = model.test_on_batch(inputs_, targets_)\n",
    "    total_losses.append(loss[0])\n",
    "    regression_losses.append(loss[1])\n",
    "    classification_losses.append(loss[2])\n",
    "    \n",
    "print(sum(total_losses) / 8)\n",
    "print(sum(regression_losses) / 8)\n",
    "print(sum(classification_losses) / 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py:97: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "outputs = model.outputs\n",
    "weights = model.trainable_weights\n",
    "gradients = keras.backend.gradients(outputs, weights)\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "sess.run(tf.initialize_all_variables())\n",
    "evaluated_gradients = sess.run(gradients, feed_dict={model.input:inputs})\n",
    "\n",
    "evaluated_gradients_split = [np.zeros_like(x) for x in evaluated_gradients]\n",
    "\n",
    "for i in range(8):\n",
    "    inputs_ = inputs[i:i+1, ...]\n",
    "    targets_ = [targets[0][i:i+1, ...], targets[1][i:i+1, ...]]\n",
    "\n",
    "    for idx, g in enumerate(sess.run(gradients, feed_dict={model.input:inputs_})):\n",
    "        evaluated_gradients_split[idx] += g\n",
    "        \n",
    "for i in range(len(evaluated_gradients_split)):\n",
    "    evaluated_gradients_split[i] /= 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-127909.53    44277.484 -165517.14   172065.67  -288450.4  ]\n",
      "[-15919.898    5572.7393 -20385.074   21735.04   -36725.746 ]\n",
      "[-127359.19    44581.914 -163080.6    173880.31  -293805.97 ]\n"
     ]
    }
   ],
   "source": [
    "print(evaluated_gradients[0].flatten()[:5])\n",
    "print(evaluated_gradients_split[0].flatten()[:5])\n",
    "print(evaluated_gradients_split[0].flatten()[:5] * 8)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}