from numpy import *
import numpy as np
from numpy.linalg import inv
from math import *
import matplotlib.pyplot as plt

# logistic function
def logist(x):
    return 1.0/(1 + np.exp(-x))

# logistic with regularization
def lgr(X, y, lamb):
    return dot(dot(inv(dot(X.transpose(), X) + lamb*eye(len(X[0]))), X.transpose()), y.reshape((len(y), 1)))

# 0/1 error
def Err(w, x, y):
    s = dot(w, x.transpose())
    p = where(s>=0, 1.0, -1.0)
    Eout = sum(p!=y)
    return Eout / len(x)

#read data
train, test = loadtxt("hw4_train.dat"), loadtxt("hw4_test.dat")
dim = len(train[0])
x = c_[ones(len(train)), delete(train, s_[dim-1], axis = 1 )]
y = delete(train, s_[0:dim-1 ], axis = 1 ).reshape((1,len(train)))[0]
xt = c_[ones(len(test)), delete(test, s_[dim-1], axis = 1 )]
yt = delete(test, s_[0:dim-1 ], axis = 1 ).reshape((1,len(test)))[0]

#No. 13
lam = 1.126
wreg = lgr(x, y, lam).reshape((1,3))[0]
print("13."), print("Ein = ", Err(wreg, x, y),", Eout = ", Err(wreg, xt, yt)), print()

#No. 14 & 15
A = range(2, -11, -1)
lamset = power(10*ones(len(A)), array(A))
Ein, Eout = zeros(len(A)), zeros(len(A))
mEin, mEout, Eom, Eim, lmi, lmo  = 1, 1, 1, 1, 1,1

for i in range(len(A)):
    wreg = lgr(x, y, lamset[i]).reshape((1,3))[0]
    Ein[i] = Err(wreg, x, y)
    Eout[i] = Err(wreg, xt, yt)
    if Ein[i] < mEin:
        mEin = Ein[i]
        Eom = Eout[i]
        lmi = i
    if Eout[i] < mEout:
        mEout = Eout[i]
        Eim = Ein[i]
        lmo = i

print("14."), print("Minimum Ein occurs when lambda =",lamset[lmi] ,", Ein =", mEin, ", Eout =",Eom), print()
print("15."), print("Minimum Eout occurs when lambda =",lamset[lmo],", Eout =", mEout, ", Ein =",Eim), print()

fig = plt.figure()
fig.suptitle("Relation between " + r"$\\lambda$" + " and "+r"$E_{in}$"+", "+r"$E_{out}$", fontsize=14, fontweight=\'bold\')
ax = fig.add_subplot(111)
plt.plot(lamset, Ein, \'k--\', label=r"$E_{in}$", color=\'#8F4586\')
plt.plot(lamset, Eout, \'k\', label=r"$E_{out}$", color=\'#64A600\')
legend = plt.legend(loc=\'upper left\', shadow=True)
ax.annotate(\'Minimum \'+r"$E_{in}$"+\' = \' + str(mEin), xy=(lamset[lmi], mEin), xytext=(lamset[lmi]*10, mEin+0.07),
 arrowprops=dict(facecolor=\'black\', shrink=0.05))
ax.annotate(\'Minimum \'+r"$E_{out}$"+\' = \' + str(mEout), xy=(lamset[lmo], mEout), xytext=(lamset[lmo]*10, mEin+0.05),
 arrowprops=dict(facecolor=\'black\', shrink=0.05))
plt.ylabel(\'Expected Value of 0/1 Error\'), plt.xlabel(r"$\\lambda$")
ax.set_xscale(\'log\')
legend.get_frame().set_facecolor(\'#CAFFFF\')


#No. 16 ~ 17
xtr, xv = x[0:120], x[120:200]
ytr, yv = y[0:120], y[120:200]
Etr, Ev, Eout = zeros(len(A)), zeros(len(A)), zeros(len(A))
mEtr, Etro, mEv, Etrv, lmtr, lmv  = 1, 1, 1, 1, 1, 1

for i in range(len(A)):
    wreg = lgr(xtr, ytr, lamset[i]).reshape((1,3))[0]
    Etr[i] = Err(wreg, xtr, ytr)
    Ev[i] = Err(wreg, xv, yv)
    Eout[i] = Err(wreg, xt, yt)
    if Etr[i] < mEtr:
        mEtr = Etr[i]
        Etro = Eout[i]
        lmtr = i
    if Ev[i] < mEv:
        mEv = Ev[i]
        Evo = Eout[i]
        lmv = i

print("16."), print("Minimum Etrain occurs when lambda =",lamset[lmtr] ,", Etrain =", mEtr, ", Eout =",Etro), print()
print("17."), print("Minimum Eval occurs when lambda =",lamset[lmv],", Eval =", mEv, ", Eout =",Evo), print()

fig2 = plt.figure()
fig2.suptitle("Relation between " + r"$\\lambda$" + " and "+ r"$E_{train}$"+", "+ r"$E_{val}$", fontsize=14, fontweight=\'bold\')
ax = fig2.add_subplot(111)
plt.plot(lamset, Etr, \'k--\', label=r"$E_{train}$", color=\'#0000E3\')
plt.plot(lamset, Ev, \'k\', label=r"$E_{val}$", color=\'#64A600\')
legend = plt.legend(loc=\'upper left\', shadow=True)
ax.annotate(\'Minimum \' +r"$E_{train}$"+\' = \' + str(mEin), xy=(lamset[lmtr], mEtr), xytext=(lamset[lmtr]*10, mEtr+0.09),
 arrowprops=dict(facecolor=\'black\', shrink=0.05))
ax.annotate(\'Minimum \' +r"$E_{val}$"+\' = \' + str(mEout), xy=(lamset[lmv], mEv), xytext=(lamset[lmv]/100, mEv+0.09),
 arrowprops=dict(facecolor=\'black\', shrink=0.05))
plt.ylabel(\'Expected Value of 0/1 Error\'), plt.xlabel(r"$\\lambda$")
ax.set_xscale(\'log\')
legend.get_frame().set_facecolor(\'#CAFFFF\')

#No. 18
wreg = lgr(x, y, lamset[lmv]).reshape((1,3))[0]
print("18."), print("For the wreg with optimal lambda achievd in 17. and trained in whole data:")
print("Ein = ", Err(wreg, x, y),", Eout = ", Err(wreg, xt, yt)), print()

#No. 19
Ecv, mEcv, lcv = zeros(len(A)), 1, 1
for i in range(len(A)):
    for j in range(5):
        ftrx, ftry = delete(x, s_[40*j:40*(j+1)], axis = 0 ), delete(y, s_[40*j:40*(j+1)], axis = 0 )
        ftex, ftey = x[40*j:40*(j+1)], y[40*j:40*(j+1)]
        wreg = lgr(ftrx, ftry, lamset[i]).reshape((1,3))[0]
        Ecv[i] += Err(wreg, ftex, ftey)
    Ecv[i] /= 5
    if Ecv[i] < mEcv:
        mEcv = Ecv[i]
        lcv = i

print("19."), print("Minimum Ecv occurs when lambda =",lamset[lcv] ,", Ecv =", mEcv), print()

fig3 = plt.figure()
fig3.suptitle("Relation between " + r"$\\lambda$" + " and " + r"$E_{cv}$", fontsize=14, fontweight=\'bold\')
ax = fig3.add_subplot(111)
plt.plot(lamset, Ecv, \'k\', label=\'Etrain\', color=\'#00A600\')
ax.annotate(\'Minimum \'+ r"$E_{cv}$"+\' = \' + str(mEcv), xy=(lamset[lcv], mEcv), xytext=(lamset[lcv]*10, mEcv+0.09),
 arrowprops=dict(facecolor=\'black\', shrink=0.05))
plt.ylabel(r\'$E_{cv}$\'+\' (Expected Value of 0/1 Error)\'), plt.xlabel(r"$\\lambda$")
ax.set_xscale(\'log\')

#No. 20
wreg = lgr(x, y, lamset[lcv]).reshape((1,3))[0]
print("20."), print("For the wreg with optimal lambda achievd in 19. and trained in whole data:")
print("Ein = ", Err(wreg, x, y),", Eout = ", Err(wreg, xt, yt)), print()

plt.show()