Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import aux_fn
- import tensorflow as tf
- import datetime
- import numpy as np
- from statsmodels.tsa.seasonal import seasonal_decompose
- import scipy
- from itertools import groupby
- import copy
- class data_analyzer:
- def get_specificity(self):
- act=self.anomaly_actual
- mask=copy.copy(self.anomaly_pred)
- idx_zero = self.get_intervals(mask, 0)
- for i in idx_zero:
- if i[1] - i[0] < self.window_width:
- mask[i[0]:i[1]] = 1
- ress1=self.tmp_fun(act,mask)
- ress2=self.tmp_fun(mask,act)
- ress=np.zeros(4) # (TP,TN,FP,FN)
- ress[0] = (ress1[0]+ress2[0])//2
- ress[2] = ress1[1]
- ress[3] = ress2[1]
- ress[1] = len(self.anomaly_pred) // self.window_width - ress[0] - ress[1] - ress[3]
- return ress
- def get_intervals(self,data,value):
- lenss = [len(list(group)) for key, group in groupby(data)]
- keys = np.array([key for key, group in groupby(data)])
- lenss = np.cumsum(lenss)
- lenss = np.insert(lenss, 0, 0)
- idx = []
- for i in np.where(keys == value)[0]:
- idx.append(lenss[i:i + 2])
- return idx
- def tmp_fun(self,act,mask):
- idx = self.get_intervals(act, 1)
- ress=[0,0]
- for i in idx:
- end=min(i[1],len(mask))
- mask_sl = mask[i[0]:end]
- if (max(mask_sl)) == 1:
- ress[0] += 1
- else:
- ress[1] += 1
- return ress
- def get_prediction_from_score_quant(self,data):
- #if max(data)<3: #to be adjusted
- # return np.zeros(len(data), dtype=bool)
- threshold = np.quantile(data,0.999)
- pred = np.zeros(len(data), dtype=bool)
- pred[data > threshold] = True
- return pred
- def __init__(self,filename_raw,rolling_step=None,is_remove_trend=True):
- self.saved_column, timestamps = aux_fn.get_csv_data(filename_raw + '.csv')
- dates_list = [datetime.datetime.strptime(date, '%Y-%m-%d %H:%M:%S') for date in timestamps]
- lab = aux_fn.get_label_data(filename_raw + '.csv')
- self.anomaly_actual = aux_fn.get_anomaly_mask(dates_list, lab)
- self.model = tf.keras.models.load_model(filename_raw+'.h5')
- #model.summary()
- self.window_width = self.model.layers[0].input_shape[1]
- if rolling_step is None:
- rolling_step = self.window_width
- data = aux_fn.rolling(self.saved_column, self.window_width, rolling_step)
- data = np.expand_dims(data, axis=2)
- yhat = self.model.predict(data)
- len1 = yhat.shape[1] + rolling_step * (yhat.shape[0] - 1)
- self.saved_column = self.saved_column[0:len1]
- self.anomaly_actual = self.anomaly_actual[0:len1]
- self.resst = np.zeros(len1)
- cnt = np.zeros(len1)
- for i in range(yhat.shape[0]):
- self.resst[i * rolling_step:i * rolling_step + self.window_width] += yhat[i, :, 0]
- cnt[i * rolling_step:i * rolling_step + self.window_width] += 1
- self.resst = np.divide(self.resst, cnt)
- # resst=aux_fn.scaleData(resst)
- xc = aux_fn.xcorr(self.saved_column, self.resst, rolling_step)
- self.opt_shift = np.argmax(xc) - rolling_step
- if self.opt_shift > 0:
- self.saved_column = np.pad(self.saved_column, (0, self.opt_shift), 'mean')
- self.anomaly_actual = np.pad(self.anomaly_actual, (0, self.opt_shift), 'minimum')
- self.resst = np.pad(self.resst, (self.opt_shift, 0), 'mean')
- elif self.opt_shift < 0:
- self.saved_column = np.pad(self.saved_column, (-self.opt_shift, 0), 'mean')
- self.anomaly_actual = np.pad(self.anomaly_actual, (-self.opt_shift, 0),'minimum')
- self.resst = np.pad(self.resst, (0, -self.opt_shift), 'mean')
- self.anomaly_score = (self.resst - self.saved_column)
- self.anomaly_score = aux_fn.smooth(self.anomaly_score,
- 50) # Smoothing parameter. Can be changed to get better results, but usually 20 works well
- if is_remove_trend:
- _ , psd = scipy.signal.periodogram(self.anomaly_score)
- psd[0:self.window_width//2] = 0
- result = seasonal_decompose(self.anomaly_score, model='additive', freq=np.argmax(psd))
- self.anomaly_score = result.resid
- self.anomaly_score=np.nan_to_num(self.anomaly_score)
- self.anomaly_score = (self.anomaly_score-np.mean(self.anomaly_score))/np.std(self.anomaly_score)
- self.anomaly_score = abs(self.anomaly_score)
- self.anomaly_pred = self.get_prediction_from_score_quant(self.anomaly_score)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement