import numpy as np
import matplotlib.pyplot as plt

# Hypothesis
hypo = lambda _w, _x: 1 / (1 + np.exp(-_w * _x))
# Cost
cost = lambda _hypo, _y : -(1-_y) * np.log(1 - _hypo) -_y * np.log(_hypo)
# Gradient
def cost_gradient(w, X, Y):
    # Euler method to calculate derivative
    # h is delta of W
    h = 1e-2

    # For partial derivatives
    tmp_val = w

    # Calculate forward values
    W = [tmp_val + h for i in range(len(X))]
    _hypo = list(map(hypo, W, X))
    fxh1 = sum(list(map(cost, _hypo, y))) # Skip to use sum

    # Calculate backward values
    W = [tmp_val - h for i in range(len(X))]
    _hypo = list(map(hypo, W, X))
    fxh2 = sum(list(map(cost, _hypo, y))) # Skip to use sum

    # Calculate the diff
    grad = (fxh1 - fxh2) / (2*h)

    return grad

# Input
X = [i * 0.01 for i in range(-200, 200)]
# Answer
Y = [i > 0 for i in range(-100, 100)]
# Weight
w = 0.6
# Learning rate
lr = 0.001

costs = []
trials = [i for i in range(10)]

for t in trials:
    W = [w for i in range(len(X))]
    _hypo = list(map(hypo, W, X))
    _cost = sum(list(map(cost, _hypo, Y))) / len(_hypo)
    costs.append(_cost)

    grad = cost_gradient(w, X, Y)

    w = w - lr * grad
    plt.plot(t, _cost, "o", label="w = {0:4.3f}".format(w))

plt.plot(trials, costs)
plt.xlabel("trials")
plt.ylabel("cost")
plt.grid()
plt.legend(numpoints=1, loc="upper right", ncol=2)
plt.show()