from sklearn import datasets
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
from numpy.linalg import inv

iris = datasets.load_iris()
X = iris.data[:100, :]
y = iris.target[:100].reshape((100, -1))


def logit(x):
    return 1. / (1 + np.exp(-x))


m, n = X.shape
alpha = 0.0065
theta_g = np.random.random((n, 1))
maxCycles = 30
J = pd.Series(np.arange(maxCycles, dtype=float))

for i in range(maxCycles):
    h = logit(np.dot(X, theta_g))
    J[i] = -(1 / 100.) * np.sum(y * np.log(h) + (1 - y) * np.log(1 - h))
    error = h - y
    grad = np.dot(X.T, error)
    theta_g -= alpha * grad
print theta_g
J.plot()
plt.show()

theta_n = np.random.random((n, 1))
maxCycles = 1
C = pd.Series(np.arange(maxCycles, dtype=float))
for i in range(maxCycles):
    h = logit(np.dot(X, theta_n))
    C[i] = -(1 / 100.) * np.sum(y * np.log(h) + (1 - y) * np.log(1 - h))
    error = h - y
    grad = np.dot(X.T, error)
    A = h * (1 - h) * np.eye(len(X))
    H = np.mat(X.T) * A * np.mat(X)
    theta_n -= inv(H) * grad
print theta_n
C.plot()
plt.show()