import numpy as np
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
from matplotlib import cm
X = [0.5,2.5]
Y = [0.2,0.9]
wx,by,er=[],[],[]
def f(w,b,x):
	return 1/(1 +np.exp(-(w*x+b)))

def grad_b(w,b,x,y):
	fx = f(w,b,x)
	return (fx-y)*(1-fx**2)

def error(w,b):
    err = 0
    for x,y in zip(X,Y):
        fx = f(w,b,x)
        err += 0.5*(fx-y)**2
    return err

def grad_w(w,b,x,y):
	fx = f(w,b,x)
	return (fx-y)*(1-fx**2)*x

def Normalgrad():
    w,b,eta,epoch = -2,-2,1,100
    for i in range(epoch):
        dw,db = 0,0
        for x,y in zip(X,Y):
            dw += grad_w(w,b,x,y)
            db += grad_b(w,b,x,y)
        w = w-eta*dw
        b = b-eta*db
        wx.append(w)
        by.append(b)
        er.append(error(w,b))

def Momentumgrad():
    w,b,eta,epoch,gama,updateb,updatew = -2,-2,1,100,0.1,0,0
    for i in range(epoch):
        dw,db = 0,0
        for x,y in zip(X,Y):
            dw += grad_w(w,b,x,y)
            db += grad_b(w,b,x,y)
        updatew = gama*updatew+eta*dw
        updateb = gama*updateb+eta*db
        w = w-updatew
        b = b-updateb
        wx.append(w)
        by.append(b)
        er.append(error(w,b))

def Nesterovgrad():
    w,b,eta,epoch,gama,updateb,updatew,wlookahead,blookahead = -2,-2,1,100,0.1,0,0,0,0
    for i in range(epoch):
        dw,db = 0,0
        wlookahead = w-gama*updatew
        blookahead = b-gama*updateb
        for x,y in zip(X,Y):
            dw += grad_w(wlookahead,blookahead,x,y)
            db += grad_b(wlookahead,blookahead,x,y)
        updatew = gama*updatew+eta*dw
        updateb = gama*updateb+eta*db
        w = w-updatew
        b = b-updateb
        wx.append(w)
        by.append(b)
        er.append(error(w,b))

def Stochasticgrad():
    w,b,eta,epoch = -2,-2,1,100
    for i in range(epoch):
        dw,db = 0,0
        for x,y in zip(X,Y):
            dw += grad_w(w,b,x,y)
            db += grad_b(w,b,x,y)
            w = w-eta*dw
            b = b-eta*db
            wx.append(w)
            by.append(b)
            er.append(error(w,b))

def Minibatchgrad():
    w,b,eta,epoch = -2,-2,1,100
    X = [0.5,2.5,1,1.5]
    Y = [0.2,0.9,0.29,0.4]
    count=0
    for i in range(epoch):
        dw,db = 0,0
        for x,y in zip(X,Y):
            dw += grad_w(w,b,x,y)
            db += grad_b(w,b,x,y)
            count += 1
            if count%2 == 0:
                w = w-eta*dw
                b = b-eta*db
                wx.append(w)
                by.append(b)
                er.append(error(w,b))

fig = plt.figure()
ax = fig.gca(projection='3d')
ax.set_title('Normal Gradient Descent')
Normalgrad()
X1 = np.arange(-6, 6, 0.5)
Y1 = np.arange(-6, 6, 0.5)
X1, Y1 = np.meshgrid(X1, Y1)
z1 = [-1]*len(wx)
Z = np.array([error(w,b) for w,b in zip(X1,Y1)])
ax.set_zlim3d(-1, 1)
surf = ax.plot_surface(X1, Y1, Z, cmap=cm.coolwarm)
ax.scatter(wx,by,z1)
plt.plot(wx,by,er)
plt.show()

wx,by,er = [],[],[]
fig = plt.figure()
ax = fig.gca(projection='3d')
ax.set_title('Momentum Gradient Descent with Gama=0.1')
Momentumgrad()
X1 = np.arange(-6, 6, 0.5)
Y1 = np.arange(-6, 6, 0.5)
X1, Y1 = np.meshgrid(X1, Y1)
z1 = [-1]*len(wx)
Z = np.array([error(w,b) for w,b in zip(X1,Y1)])
ax.set_zlim3d(-1, 1)
surf = ax.plot_surface(X1, Y1, Z, cmap=cm.coolwarm)
ax.scatter(wx,by,z1)
plt.plot(wx,by,er)
plt.show()

wx,by,er=[],[],[]
fig = plt.figure()
ax = fig.gca(projection='3d')
ax.set_title('Nesterov Accelerated Gradient Descent with Gama=0.1')
Nesterovgrad()
Z = np.array([error(w,b) for w,b in zip(X1,Y1)])
X1 = np.arange(-6, 6, 0.5)
Y1 = np.arange(-6, 6, 0.5)
X1, Y1 = np.meshgrid(X1, Y1)
z1=[-1]*len(wx)
Z = np.array([error(w,b) for w,b in zip(X1,Y1)])
ax.set_zlim3d(-1, 1)
surf = ax.plot_surface(X1, Y1, Z, cmap=cm.coolwarm)
ax.scatter(wx,by,z1)
plt.plot(wx,by,er)
plt.show()

wx,by,er=[],[],[]
fig = plt.figure()
ax = fig.gca(projection='3d')
ax.set_title('Stochastic Gradient Descent')
Stochasticgrad()
X1 = np.arange(-6, 6, 0.5)
Y1 = np.arange(-6, 6, 0.5)
X1, Y1 = np.meshgrid(X1, Y1)
z1 = [-1]*len(wx)
Z = np.array([error(w,b) for w,b in zip(X1,Y1)])
ax.set_zlim3d(-1, 1)
surf = ax.plot_surface(X1, Y1, Z, cmap=cm.coolwarm)
ax.scatter(wx,by,z1)
plt.plot(wx,by,er)
plt.show()

wx,by,er=[],[],[]
fig = plt.figure()
ax = fig.gca(projection='3d')
ax.set_title('Minibatch Gradient Dscent')
Minibatchgrad()
X1 = np.arange(-6, 6, 0.5)
Y1 = np.arange(-6, 6, 0.5)
X1, Y1 = np.meshgrid(X1, Y1)
z1 = [-1]*len(wx)
Z = np.array([error(w,b) for w,b in zip(X1,Y1)])
Z = np.array([error(w,b) for w,b in zip(X1,Y1)])
ax.set_zlim3d(-1, 1)
surf = ax.plot_surface(X1, Y1, Z, cmap=cm.coolwarm)
ax.scatter(wx,by,z1)
plt.plot(wx,by,er)
plt.show()