 # Logistic Regression with L2 regularization class

Apr 16th, 2021 (edited)
788
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
1. class LogisticRegression:
2.     def __init__(self, _lambda=0.05,
3.                  lr = 0.001,
4.                  n_iters = 100,
5.                  tolerance = 0.0001,
6.                  min_iter_tolerance = 10,
7.                  privacy_mechanism = None,
8.                  epsilon = 1e-2,
9.                  delta =1e-5):
10.
11.         if(min_iter_tolerance < 0):
12.             raise Exception('Minimum iteration can not be negative.')
13.         if privacy_mechanism:
14.             if type(privacy_mechanism) == str:
15.                 if not (privacy_mechanism.lower() == 'laplace' or privacy_mechanism.lower() == 'gaussian'):
16.                     raise Exception('Differential privacy mechanism should be Laplace or Gaussian.')
17.                 else:
18.                     self.differential_privacy_mechanism = privacy_mechanism.lower()
19.
20.                     if (delta <= 0 or epsilon <= 0):
21.                         raise Exception('Privacy parameter delta or epsilon should be positive.')
22.
23.                     self.epsilon = epsilon
24.                     self.delta = delta
25.             else:
26.                 raise Exception('Privacy mechanism should be a string.')
27.         else:
28.             self.differential_privacy_mechanism = privacy_mechanism
29.
30.         self._lambda = _lambda
31.         self.lr = lr
32.         self.n_iters = n_iters
33.         self.tolerance = tolerance
34.         self.iterTolerance = min_iter_tolerance
35.         self.weights = None
36.         self.J_train = None
37.         self.J_validation = None
39.         self.max_iteration_required = None
40.
41.     def fit(self, X, y, X_validation, y_validation):
42.         n_samples, n_features = X.shape # m x n
43.
44.         # weight bias in one vector -> [b, W]
45.
46.         # zero initialization
47.         # self.weights = np.zeros((1,n_features + 1)) # 1st element is bias 'b'
48.         # self.weights[0, 0] = np.random.rand() # self.bias # 1st element of weight (bias) random initialization
49.
50.         # random initialization
51.         self.weights = np.reshape(np.random.rand(n_features+1), (1,n_features+1)) # [b, W] # 1x(n+1)
52.
53.         X = np.concatenate((np.ones((n_samples, 1)), X), axis = 1) # 1st column all '1' to be multiplied by bias 'b'
54.         X_validation = np.concatenate((np.ones((X_validation.shape, 1)), X_validation), axis = 1)
55.
56.         self.J_train = []
57.         self.J_validation = []
59.
60.         converge_count = 0;
61.
62.         for i in tqdm(range(self.n_iters)):
63.             y_hat, dw, cost_train = self.calculateGradient(X, y, isTrainData=True) # db,
64.             y_hat_validation, cost_validation = self.calculateGradient(X_validation, y_validation)
65.
66.             # weight and bias update
67.             self.weights = self.weights - (self.lr*(dw.T))
68.
69.             # Norm of the Gradient(dJ/dw)
71.
72.             # train cost
73.             self.J_train.append(cost_train)
74.
75.             # validation cost
76.             self.J_validation.append(cost_validation)
77.
78.             print('Training loss = {} and Valaidation loss = {} after {} iteration'.format(cost_train, cost_validation, i))
79.
80.             # breaking condition after convergence (reach at tolerance level)
81. #             if(np.all(abs(dw)) <= self.tolerance):
82. #                 break
83.
84.             if(np.abs(self.J_train[int(i)-1] - self.J_train[int(i)]) <= self.tolerance
85.                and i != 0
86.                and i >= self.iterTolerance):
87.                 """ if only number of iteration is greater or equal to minimum number of iterations permitted
88.                    then the convergence will be checked"""
89.                 converge_count += 1
90.                 """print('{}  {}  {}  {}'.format(converge_count,
91.                                              self.J_train[int(i)-1],
92.                                              self.J_train[int(i)],
93.                                              np.abs(self.J_train[int(i)-1] - self.J_train[int(i)])))"""
94.
95.                 if converge_count >= 5:
96.                     self.max_iteration_required = i - converge_count + 1
97.                     break
98.                     """converge_count is used totest for consecutive 6 (5 counts) points if converges then the max_iteration
99.                    for converge will be current_iteration-count + 1 (+1 because converge_count increases before check
100.                    for break)"""
101.             else:
102.                 converge_count = 0 # to avoid increase count less than 3 consecutive value in tolerance level
103.
104.
105.
106.     def sigmoid_function(self, var):
107.         return 1.0 / (1.0 + np.exp(-(var)))
108.
109.     def hypothesis(self, X):
110.         linear_model = np.dot(self.weights, X.T) # + self.bias # m x 1
111.         y_h = self.sigmoid_function(linear_model)
112.         y_h[y_h==0.0] = 0.000000000001 # to prevent log(0) in cost
113.         y_h[y_h==1.0] = 0.999999999999 # to prevent log(1) in cost
114.
115.         return y_h
116.
117.     def calculateGradient(self, X, y, isTrainData = False):
118.         n_samples = X.shape
119.         y_hat = self.hypothesis(X)
120.
121.         # calculate cost
122.         cost = self.cost(y, y_hat, n_samples)
123.
124.         if isTrainData == True:
126.             dw = (1/n_samples)*np.dot(X.T,(y_hat-y.T).T) + np.sum((self._lambda/n_samples)*self.weights) # (n+1) x 1 # dJ/dw
127.             dw[0,0] = (1/n_samples)*np.sum(y_hat-y.T) # 1 x 1 # dJ/db  # bias
128.
129.             # print('shape of dw: ', dw.shape)
130.             # print(f'dw: {dw}\n db: {db}')
131.
132.             if self.differential_privacy_mechanism:
133.                 Delta_f = 1/n_samples
134.                 if self.differential_privacy_mechanism == 'laplace':
135.                     sigma = Delta_f / self.epsilon
136.                     dw += np.random.laplace(loc=0.0, scale=sigma, size=(X.shape, 1)) # (n_features + 1)x1
137.                 elif self.differential_privacy_mechanism == 'gaussian':
138.                     sigma = (Delta_f / self.epsilon)*math.sqrt(2*math.log(1.25/self.delta))
139.                     sigma = sigma**2  # sigma_square
140.                     dw += np.random.normal(loc=0.0, scale=sigma, size=(X.shape, 1))
141.
142.             return y_hat, dw, cost # ,db
143.         else:
144.             return y_hat,cost
145.
146.     def cost(self, y, y_hat,n_samples):
147.         cost = (-1.0/n_samples)
148.         cost *= np.sum((y.T * np.log(y_hat)) + ((1.0-y.T) * (np.log(1.0-y_hat))))
149.         # cost += (0.5*self._lambda) * np.sum(np.dot(self.weights.T,self.weights)) # (0.5*self._lambda/n_samples)
150.         cost += (0.5*(self._lambda/n_samples)) * np.sum(np.dot(self.weights.T,self.weights)) # (0.5*self._lambda/n_samples)
151.
152.         return cost
153.
154.
155.     def predict(self, X, weights, threshold = 0.5): # bias,
156.         bias = weights[0,0]
157.         weights_without_bias = weights[:, 1:]
158.         linear_model = np.dot(weights_without_bias, X.T) + bias
159.         y_predict = self.sigmoid_function(linear_model)
160.
161.         y_predict[y_predict >= threshold] = 1.0 # converts real to binary
162.         y_predict[y_predict < threshold] = 0.0
163.
164.         return y_predict
165.
166.     def prediction_result(self, y_true, y_predict):
167.         if (y_true.shape != y_predict.shape): # assumed y_true, y_predict to be row vector
168.             raise Exception("Label and prediction vector's length are not equal.")
169.
170.         tp = tn = fp = fn = 0
171.
172.         tp = np.sum((y_predict == y_true) & (y_predict == 1.0)) # true_value == 1 and predicted == 1
173.         tn = np.sum((y_predict == y_true) & (y_predict == 0.0)) # true_value == 0 and predicted == 0
174.         fp = np.sum((y_predict != y_true) & (y_predict == 1.0)) # true_value == 0 and predicted == 1
175.         fn = np.sum((y_predict != y_true) & (y_predict == 0.0)) # true_value == 1 and predicted == 0
176.
177.         """precision = tp / (tp+fp) ---> tp = true +ve, fp = false +ve
178.           recall or TPR = tp / (tp+fn) ---> tp = true +ve, fn = false -ve, TPR = true +ve rate
179.           FPR = fp / (tn+fp) ---> tn = true -ve, fp = false +ve"""
180.
181.         accuracy = (tp + tn) / (tp + tn + fp +fn) if(tp + tn + fp +fn) != 0 else 0
182.         precision = tp / (tp + fp) if(tp + fp) != 0 else 0
183.         recall_or_TPR = tp / (tp + fn) if(tp + fn) != 0 else 0
184.         FPR = fp / (tn + fp) if(tn + fp) != 0 else 0
185.
186.         return {'acc': accuracy, 'pre': precision, 'tpr': recall_or_TPR, 'fpr': FPR}
187.
188.     def summary(self):
189.         result = {
190.             'weights': self.weights ,
191.             'J_train': self.J_train,
192.             'J_validation': self.J_validation,
194.             'max_iteration_required': self.max_iteration_required
195.         }
196.
197.         return result
RAW Paste Data