{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pylab inline\n",
    "\n",
    "\n",
    "lw=6\n",
    "\n",
    "colors = {'FW': '#66c2a5','AdaFW': '#fc8d62', 'AdaPFW' : '#8da0cb', 'PFW':'#e78ac3',\n",
    "          'AFW' : '#a6d854', 'AdaAFW' : '#ffd92f', 'LSFW': '#756bb1', 'MP': '#998ec3', 'AdaptiveMP' : '#f1a340'}\n",
    "\n",
    "plt.rcParams['figure.figsize'] = (3 * 10.0, 1 * 8.0)\n",
    "plt.rcParams['font.size'] = 35\n",
    "\n",
    "\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.font_manager as font_manager\n",
    "\n",
    "font_dirs = ['/home/fabian/Dropbox/fonts/Open_Sans/', ]\n",
    "font_files = font_manager.findSystemFonts(fontpaths=font_dirs)\n",
    "font_list = font_manager.createFontList(font_files)\n",
    "font_manager.fontManager.ttflist.extend(font_list)\n",
    "\n",
    "path = '/home/fabian/Dropbox/fonts/Open_Sans/OpenSans-Light.ttf'\n",
    "prop = font_manager.FontProperties(fname=path)\n",
    "\n",
    "matplotlib.rc('font', family='sans-serif') \n",
    "# matplotlib.rc('text', usetex='false') \n",
    "matplotlib.rcParams['font.family'] = 'Open Sans'\n",
    "matplotlib.rcParams['font.weight'] = 'light'\n",
    "matplotlib.rcParams.update({'font.size': 35})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.rcParams['figure.figsize'] = (2 * 10.0, 1 * 8.0)\n",
    "plt.rcParams['font.size'] = 25\n",
    "\n",
    "\n",
    "matplotlib.rcParams['xtick.direction'] = 'out'\n",
    "matplotlib.rcParams['ytick.direction'] = 'out'\n",
    "\n",
    "np.random.seed(0)\n",
    "n_samples = 100\n",
    "A = np.random.rand(n_samples, 2)\n",
    "A[:, 0] *= 4\n",
    "b = np.sign(np.random.randn(100))\n",
    "alpha = 2./n_samples\n",
    "\n",
    "from sklearn.linear_model import logistic\n",
    "from scipy import optimize\n",
    "\n",
    "def f_obj(x):\n",
    "    return logistic._logistic_loss(x, A, b, alpha)\n",
    "\n",
    "delta = 0.025\n",
    "x = np.arange(-3.0, 2.0, delta)\n",
    "y = np.arange(-3.0, 2.0, delta)\n",
    "X, Y = np.meshgrid(x, y)\n",
    "\n",
    "Z = np.zeros_like(X)\n",
    "for i in range(X.shape[0]):\n",
    "    for j in range(X.shape[1]):\n",
    "        xx = np.array((X[i, j], Y[i, j]))\n",
    "        Z[i, j] = f_obj(xx)\n",
    "\n",
    "sol = optimize.minimize(f_obj, (0, 0))\n",
    "plot_x = []\n",
    "plot_y = []\n",
    "plot_x2 = []\n",
    "plot_y2 = []\n",
    "xt = np.array((-3, -3))\n",
    "L = 0.25 * linalg.linalg.norm(A) ** 2 + alpha\n",
    "L_adaptive = 0.01 * L\n",
    "step_size = 1 / L\n",
    "xt2 = xt.copy()\n",
    "for i in range(25):\n",
    "    f, axarr = plt.subplots(1, 2, sharex=True)\n",
    "    CS = axarr[0].contour(X, Y, Z, 20)\n",
    "    #plt.clabel(CS, inline=1, fontsize=10)\n",
    "\n",
    "    plot_x.append(xt[0])\n",
    "    plot_y.append(xt[1])\n",
    "    axarr[0].plot(plot_x, plot_y, marker='^', markersize=10, lw=2)\n",
    "    f.suptitle('Theoretical (left) vs Adaptive (right) Step Size, iteration %s' % i)\n",
    "\n",
    "    xt = xt - step_size * logistic._logistic_loss_and_grad(xt, A, b, alpha)[1]\n",
    "    axarr[0].set_xticks(())\n",
    "    axarr[1].set_xticks(())\n",
    "    axarr[0].set_yticks(())\n",
    "    axarr[1].set_yticks(())\n",
    "\n",
    "    # axarr[0].scatter(*sol.x, s=100)\n",
    "    \n",
    "    \n",
    "    ## second plot\n",
    "    CS = axarr[1].contour(X, Y, Z, 20)\n",
    "    plot_x2.append(xt2[0])\n",
    "    plot_y2.append(xt2[1])\n",
    "    axarr[1].plot(plot_x2, plot_y2, marker='^', markersize=10, lw=2)    \n",
    "\n",
    "    L_adaptive *= 0.1\n",
    "    f_grad = logistic._logistic_loss_and_grad(xt2, A, b, 0)[1]\n",
    "    x_next = xt2 - (1 / L_adaptive) * f_grad\n",
    "\n",
    "    while True:\n",
    "        if f_obj(x_next) <= f_obj(xt2) + f_grad.dot(x_next - xt2) + (L_adaptive/2.) * (np.linalg.norm(x_next - xt2) **2):\n",
    "            xt2 = xt2 - (1 / L_adaptive) * f_grad\n",
    "            break\n",
    "        else:\n",
    "            L_adaptive *= 1.1\n",
    "            x_next = xt2 - (1 / L_adaptive) * f_grad\n",
    "        \n",
    "    plt.savefig('lr_adaptive_%02d.png' % i)\n",
    "    plt.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}