function in nnet

import theano
import theano.tensor as T
import numpy


class Layer(object):
    """
    this is a layer in the mlp
    it's not meant to predict the outcome hence it does not compute a loss.
    apply the functions for negative log likelihood = cost on the output of the last layer
    """

    def __init__(self, input, n_in, n_out):
        self.W = theano.shared(
                value=numpy.zeros(
                        (n_in, n_out),
                        dtype=theano.config.floatX
                ),
                name="W",
                borrow=True
        )
        self.b = theano.shared(
                value=numpy.zeros(n_out,
                                  dtype=theano.config.floatX),
                name="b",
                borrow=True
        )

        self.output = T.nnet.softmax(T.dot(input, self.W) + self.b)
        self.params = (self.W, self.b)
        self.input = input


def y_pred(output):
    return T.argmax(output, axis=1)


# this is meant to train the network for one specific function value
def custom_cost(output, y):
    if y==0:
        return -T.mean(T.log(output))
    else:
        return -T.mean(1-T.log(output))


def errors(output, y):
    # check if y has same dimension of y_pred
    if y.ndim != y_pred(output).ndim:
        raise TypeError(
                'y should have the same shape as self.y_pred',
                ('y', y.type, 'y_pred', y_pred(output).type)
        )
    # check if y is of the correct datatype
    if y.dtype.startswith('int'):
        # the T.neq operator returns a vector of 0s and 1s, where 1
        # represents a mistake in prediction
        return T.mean(T.neq(y_pred(output), y))
    else:
        raise NotImplementedError()

data_x = numpy.matrix([[0, 0],
                       [1, 0],
                       [0, 1],
                       [1, 1]])

data_y = numpy.array([0,
                      0,
                      0,
                      1])

train_set_x = theano.shared(numpy.asarray(data_x,
                         dtype=theano.config.floatX),
                         borrow=True)

train_set_y = T.cast(theano.shared(numpy.asarray(data_y,
                         dtype=theano.config.floatX),
                         borrow=True),"int32")

x = T.matrix("x")  # data
y = T.ivector("y")  # labels

classifier = Layer(input=x, n_in=2, n_out=1)

cost = custom_cost(classifier.output, y)

g_W = T.grad(cost=cost, wrt=classifier.W)
g_b = T.grad(cost=cost, wrt=classifier.b)
index = T.iscalar()

learning_rate = 0.1

updates = (
    (classifier.W, classifier.W - learning_rate * g_W),
    (classifier.b, classifier.b - learning_rate * g_b)
)

train_model = theano.function(
        inputs=[index],
        outputs=cost,
        updates=updates,
        givens={
            x: train_set_x[index:index + 1],
            y: train_set_y[index:index + 1]
        }
)
validate_model = theano.function(
        inputs=[index],
        outputs=errors(classifier.output, y),
        givens={
            x: train_set_x[index:index + 1],
            y: train_set_y[index:index + 1]
        }
)

print("before")
for i in range(4):
    print(validate_model(i))

n=50
last=10000000
sum=0
eps=0.01
#while abs(last-sum)>eps:
for i in range(5):
    last=sum
    sum=0
    for i in range(4):
        sum+=train_model(i)
    print(sum)

print("after")
for i in range(4):
    print(validate_model(i))