mohtasim_nakib

Logistic Regression with L2 regularization class

Apr 16th, 2021 (edited)
788
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. class LogisticRegression:
  2.     def __init__(self, _lambda=0.05,
  3.                  lr = 0.001,
  4.                  n_iters = 100,
  5.                  tolerance = 0.0001,
  6.                  min_iter_tolerance = 10,
  7.                  privacy_mechanism = None,
  8.                  epsilon = 1e-2,
  9.                  delta =1e-5):
  10.        
  11.         if(min_iter_tolerance < 0):
  12.             raise Exception('Minimum iteration can not be negative.')
  13.         if privacy_mechanism:
  14.             if type(privacy_mechanism) == str:
  15.                 if not (privacy_mechanism.lower() == 'laplace' or privacy_mechanism.lower() == 'gaussian'):
  16.                     raise Exception('Differential privacy mechanism should be Laplace or Gaussian.')
  17.                 else:
  18.                     self.differential_privacy_mechanism = privacy_mechanism.lower()
  19.                    
  20.                     if (delta <= 0 or epsilon <= 0):
  21.                         raise Exception('Privacy parameter delta or epsilon should be positive.')
  22.                        
  23.                     self.epsilon = epsilon
  24.                     self.delta = delta
  25.             else:
  26.                 raise Exception('Privacy mechanism should be a string.')
  27.         else:
  28.             self.differential_privacy_mechanism = privacy_mechanism
  29.            
  30.         self._lambda = _lambda
  31.         self.lr = lr
  32.         self.n_iters = n_iters
  33.         self.tolerance = tolerance
  34.         self.iterTolerance = min_iter_tolerance
  35.         self.weights = None
  36.         self.J_train = None
  37.         self.J_validation = None
  38.         self.norm_of_gradient_dw = None
  39.         self.max_iteration_required = None
  40.        
  41.     def fit(self, X, y, X_validation, y_validation):  
  42.         n_samples, n_features = X.shape # m x n
  43.        
  44.         # weight bias in one vector -> [b, W]
  45.        
  46.         # zero initialization
  47.         # self.weights = np.zeros((1,n_features + 1)) # 1st element is bias 'b'
  48.         # self.weights[0, 0] = np.random.rand() # self.bias # 1st element of weight (bias) random initialization
  49.        
  50.         # random initialization
  51.         self.weights = np.reshape(np.random.rand(n_features+1), (1,n_features+1)) # [b, W] # 1x(n+1)
  52.        
  53.         X = np.concatenate((np.ones((n_samples, 1)), X), axis = 1) # 1st column all '1' to be multiplied by bias 'b'
  54.         X_validation = np.concatenate((np.ones((X_validation.shape[0], 1)), X_validation), axis = 1)
  55.        
  56.         self.J_train = []
  57.         self.J_validation = []
  58.         self.norm_of_gradient_dw = []
  59.        
  60.         converge_count = 0;
  61.        
  62.         for i in tqdm(range(self.n_iters)):
  63.             y_hat, dw, cost_train = self.calculateGradient(X, y, isTrainData=True) # db,
  64.             y_hat_validation, cost_validation = self.calculateGradient(X_validation, y_validation)
  65.            
  66.             # weight and bias update
  67.             self.weights = self.weights - (self.lr*(dw.T))
  68.            
  69.             # Norm of the Gradient(dJ/dw)
  70.             self.norm_of_gradient_dw.append(np.linalg.norm(dw))
  71.            
  72.             # train cost
  73.             self.J_train.append(cost_train)
  74.            
  75.             # validation cost
  76.             self.J_validation.append(cost_validation)
  77.            
  78.             print('Training loss = {} and Valaidation loss = {} after {} iteration'.format(cost_train, cost_validation, i))
  79.            
  80.             # breaking condition after convergence (reach at tolerance level)
  81. #             if(np.all(abs(dw)) <= self.tolerance):
  82. #                 break
  83.                
  84.             if(np.abs(self.J_train[int(i)-1] - self.J_train[int(i)]) <= self.tolerance
  85.                and i != 0
  86.                and i >= self.iterTolerance):
  87.                 """ if only number of iteration is greater or equal to minimum number of iterations permitted
  88.                    then the convergence will be checked"""
  89.                 converge_count += 1
  90.                 """print('{}  {}  {}  {}'.format(converge_count,
  91.                                              self.J_train[int(i)-1],
  92.                                              self.J_train[int(i)],
  93.                                              np.abs(self.J_train[int(i)-1] - self.J_train[int(i)])))"""
  94.                    
  95.                 if converge_count >= 5:
  96.                     self.max_iteration_required = i - converge_count + 1
  97.                     break
  98.                     """converge_count is used totest for consecutive 6 (5 counts) points if converges then the max_iteration
  99.                    for converge will be current_iteration-count + 1 (+1 because converge_count increases before check
  100.                    for break)"""
  101.             else:
  102.                 converge_count = 0 # to avoid increase count less than 3 consecutive value in tolerance level
  103.                    
  104.            
  105.    
  106.     def sigmoid_function(self, var):
  107.         return 1.0 / (1.0 + np.exp(-(var)))
  108.    
  109.     def hypothesis(self, X):
  110.         linear_model = np.dot(self.weights, X.T) # + self.bias # m x 1
  111.         y_h = self.sigmoid_function(linear_model)
  112.         y_h[y_h==0.0] = 0.000000000001 # to prevent log(0) in cost
  113.         y_h[y_h==1.0] = 0.999999999999 # to prevent log(1) in cost
  114.        
  115.         return y_h
  116.    
  117.     def calculateGradient(self, X, y, isTrainData = False):
  118.         n_samples = X.shape[0]
  119.         y_hat = self.hypothesis(X)
  120.        
  121.         # calculate cost
  122.         cost = self.cost(y, y_hat, n_samples)
  123.        
  124.         if isTrainData == True:
  125.             # calculate gradiant
  126.             dw = (1/n_samples)*np.dot(X.T,(y_hat-y.T).T) + np.sum((self._lambda/n_samples)*self.weights) # (n+1) x 1 # dJ/dw
  127.             dw[0,0] = (1/n_samples)*np.sum(y_hat-y.T) # 1 x 1 # dJ/db  # bias
  128.            
  129.             # print('shape of dw: ', dw.shape)
  130.             # print(f'dw: {dw}\n db: {db}')
  131.            
  132.             if self.differential_privacy_mechanism:
  133.                 Delta_f = 1/n_samples
  134.                 if self.differential_privacy_mechanism == 'laplace':
  135.                     sigma = Delta_f / self.epsilon
  136.                     dw += np.random.laplace(loc=0.0, scale=sigma, size=(X.shape[1], 1)) # (n_features + 1)x1
  137.                 elif self.differential_privacy_mechanism == 'gaussian':
  138.                     sigma = (Delta_f / self.epsilon)*math.sqrt(2*math.log(1.25/self.delta))
  139.                     sigma = sigma**2  # sigma_square
  140.                     dw += np.random.normal(loc=0.0, scale=sigma, size=(X.shape[1], 1))
  141.            
  142.             return y_hat, dw, cost # ,db
  143.         else:
  144.             return y_hat,cost
  145.        
  146.     def cost(self, y, y_hat,n_samples):
  147.         cost = (-1.0/n_samples)
  148.         cost *= np.sum((y.T * np.log(y_hat)) + ((1.0-y.T) * (np.log(1.0-y_hat))))
  149.         # cost += (0.5*self._lambda) * np.sum(np.dot(self.weights.T,self.weights)) # (0.5*self._lambda/n_samples)
  150.         cost += (0.5*(self._lambda/n_samples)) * np.sum(np.dot(self.weights.T,self.weights)) # (0.5*self._lambda/n_samples)
  151.        
  152.         return cost
  153.          
  154.    
  155.     def predict(self, X, weights, threshold = 0.5): # bias,
  156.         bias = weights[0,0]
  157.         weights_without_bias = weights[:, 1:]
  158.         linear_model = np.dot(weights_without_bias, X.T) + bias
  159.         y_predict = self.sigmoid_function(linear_model)
  160.        
  161.         y_predict[y_predict >= threshold] = 1.0 # converts real to binary
  162.         y_predict[y_predict < threshold] = 0.0
  163.        
  164.         return y_predict
  165.    
  166.     def prediction_result(self, y_true, y_predict):
  167.         if (y_true.shape[1] != y_predict.shape[1]): # assumed y_true, y_predict to be row vector
  168.             raise Exception("Label and prediction vector's length are not equal.")
  169.        
  170.         tp = tn = fp = fn = 0
  171.        
  172.         tp = np.sum((y_predict == y_true) & (y_predict == 1.0)) # true_value == 1 and predicted == 1
  173.         tn = np.sum((y_predict == y_true) & (y_predict == 0.0)) # true_value == 0 and predicted == 0
  174.         fp = np.sum((y_predict != y_true) & (y_predict == 1.0)) # true_value == 0 and predicted == 1
  175.         fn = np.sum((y_predict != y_true) & (y_predict == 0.0)) # true_value == 1 and predicted == 0
  176.        
  177.         """precision = tp / (tp+fp) ---> tp = true +ve, fp = false +ve
  178.           recall or TPR = tp / (tp+fn) ---> tp = true +ve, fn = false -ve, TPR = true +ve rate
  179.           FPR = fp / (tn+fp) ---> tn = true -ve, fp = false +ve"""
  180.        
  181.         accuracy = (tp + tn) / (tp + tn + fp +fn) if(tp + tn + fp +fn) != 0 else 0
  182.         precision = tp / (tp + fp) if(tp + fp) != 0 else 0
  183.         recall_or_TPR = tp / (tp + fn) if(tp + fn) != 0 else 0
  184.         FPR = fp / (tn + fp) if(tn + fp) != 0 else 0
  185.          
  186.         return {'acc': accuracy, 'pre': precision, 'tpr': recall_or_TPR, 'fpr': FPR}    
  187.    
  188.     def summary(self):
  189.         result = {
  190.             'weights': self.weights ,
  191.             'J_train': self.J_train,
  192.             'J_validation': self.J_validation,
  193.             'norm_of_gradient_dw': self.norm_of_gradient_dw,
  194.             'max_iteration_required': self.max_iteration_required
  195.         }
  196.        
  197.         return result
RAW Paste Data

Adblocker detected! Please consider disabling it...

We've detected AdBlock Plus or some other adblocking software preventing Pastebin.com from fully loading.

We don't have any obnoxious sound, or popup ads, we actively block these annoying types of ads!

Please add Pastebin.com to your ad blocker whitelist or disable your adblocking software.

×