Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predi
- cted samples.
- 'precision', 'predicted', average, warn_for)
- precision recall f1-score support
- non-VPN 0.81 1.00 0.89 29432
- VPN 0.00 0.00 0.00 6973
- micro avg 0.81 0.81 0.81 36405
- macro avg 0.40 0.50 0.45 36405
- weighted avg 0.65 0.81 0.72 36405
- def data_generotto(path: str, batchsize: int):
- while True:
- chunks = pd.read_csv(os.path.join(
- path, "shuffled.csv"), sep=';', chunksize=batchsize)
- for i, chunk in enumerate(chunks):
- X, y = preprocess.preprocess(chunk)
- # X = np.array(X).reshape(X.shape[0], 1, X.shape[1])
- yield (X, y)
- # sorry for messy code
- def balance_train_data(data, fold_count=3):
- """Balance the data using sklearn.utils resample to max sentiment count."""
- balanced_data = pd.DataFrame()
- data_dict = dict(data['label'].value_counts())
- for label in data_dict.keys():
- df = data[data.label == label]
- samples_count = int(
- (max(data_dict.values()) - data_dict[label])/fold_count)
- df_up = resample(df, replace=True,
- n_samples=samples_count, random_state=42)
- print("Resampled {} tweets: {} + {} = {}".format(label,
- len(df), len(df_up), len(df)+len(df_up)))
- balanced_data = pd.concat([balanced_data, df, df_up])
- return shuffle(balanced_data, random_state=42)
- def create_model(model_folder_name):
- global folder_name
- folder_name = model_folder_name
- model = Sequential()
- model.add(Dense(8, activation='relu', input_dim=4))
- model.add(Dense(4, kernel_initializer='uniform', activation='relu'))
- model.add(Dense(1, kernel_initializer='uniform', activation='sigmoid'))
- optimizer = optimizers.Adam(lr=0.0001)
- model.compile(optimizer=optimizer, loss="binary_crossentropy",
- metrics=['accuracy'])
- model.summary(print_fn=myprint)
- return model, optimizer.get_config(), "ann"
- model.fit_generator(data_generotto(
- "./complete_csv", BS), steps_per_epoch=TRAIN_SIZE // BS, epochs=EPOCHS, callbacks=[es])
- save_model(model, f"./models/{model_folder_name}/MODEL.h5")
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement