daily pastebin goal
45%
SHARE
TWEET

Untitled

a guest Dec 11th, 2018 58 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import numpy as np # linear algebra
  2. import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
  3. import os
  4. import re
  5. import torchtext
  6. import torch
  7. import random
  8. from torch import nn, optim
  9. from sklearn.metrics import f1_score
  10. from nltk import word_tokenize
  11.  
  12.  
  13. random_state = random.getstate()
  14. batch_size = 64
  15.  
  16. text = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize)
  17. qid = torchtext.data.Field()
  18. target = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)
  19. train = torchtext.data.TabularDataset(path='../input/train.csv', format='csv',
  20.                                       fields={'question_text': ('text',text),
  21.                                               'target': ('target',target)})
  22. ???????????????????????????????????????????????????????????????????????
  23. class SimpleModel(nn.Module):
  24.     def __init__(self, pretrained_lm, padding_idx, hidden_dim, static=True):
  25.         super(SimpleModel, self).__init__()
  26.         self.hidden_dim = 32
  27.         self.embedding = nn.Embedding.from_pretrained(pretrained_lm)
  28.         self.embedding.padding_idx = padding_idx
  29.         if static:
  30.             self.embedding.weight.requires_grad = False
  31.         self.lstm = nn.LSTM(input_size=self.embedding.embedding_dim,
  32.                             hidden_size=self.hidden_dim,
  33.                             num_layers=1)
  34.         self.lstm_to_linear = nn.Linear(self.hidden_dim, 1)
  35.        
  36.     def forward(self, sents):
  37.         x = self.embedding(sents)
  38.         x = torch.transpose(x, dim0=1, dim1=0)
  39.         lstm_out, (h_n, c_n) = self.lstm(x)
  40.         # gather all the hidden states
  41.         x = torch.cat([c_n[i,:, :] for i in range(c_n.shape[0])], dim=1)
  42.         # feed linear layer
  43.         output = self.lstm_to_linear(x)
  44.         return output
  45.  
  46.    
  47.  
  48. text.build_vocab(train, min_freq=1)
  49. text.vocab.load_vectors(torchtext.vocab.Vectors('../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'))
  50. print(text.vocab.vectors.shape)
  51.  
  52.  
  53. train, val = train.split(split_ratio=0.8, random_state=random_state)
  54.  
  55.  
  56. train_iter = torchtext.data.BucketIterator(dataset=train,
  57.                                            batch_size=batch_size,
  58.                                            sort_key=lambda x: x.text.__len__(),
  59.                                            shuffle=True,
  60.                                            sort=False)
  61.  
  62. val_iter = torchtext.data.BucketIterator(dataset=val,
  63.                                          batch_size=batch_size,
  64.                                          sort_key=lambda x: x.text.__len__(),
  65.                                          train=False,
  66.                                          sort=False)
  67. ???????????????????????????????????????????????????????????????????????
  68. def training(epoch, model, loss_func, optimizer, train_iter, val_iter):
  69.     step = 0
  70.     train_record = []
  71.     losses = []
  72.     val_record = []
  73.     for e in range(epoch):
  74.         train_iter.init_epoch()
  75.         for train_batch in iter(train_iter):
  76.             step += 1
  77.             model.train()
  78.             x = train_batch.text.cuda()
  79.             y = train_batch.target.type(torch.Tensor).cuda()
  80.             model.zero_grad()
  81.             pred = model.forward(x).view(-1)
  82.             loss = loss_function(pred, y)
  83.             loss_data = loss.cpu().data.numpy()
  84.             train_record.append(loss_data)
  85.             loss.backward()
  86.             optimizer.step()
  87.             if step % 1000 == 0:
  88.                 print("Step: {:06}, loss {:.4f}".format(step, loss_data))
  89.             if step % 10000 == 0:
  90.                 model.eval()
  91.                 model.zero_grad()
  92.                 val_loss = []
  93.                 for val_batch in iter(val_iter):
  94.                     val_x = val_batch.text.cuda()
  95.                     val_y = val_batch.target.type(torch.Tensor).cuda()
  96.                     val_pred = model.forward(val_x).view(-1)
  97.                     val_loss.append(loss_function(val_pred, val_y).cpu().data.numpy())
  98.                 val_record.append({'step': step, 'loss': np.mean(val_loss)})
  99.                 print('Epoch {:02} - step {:06} - train_loss {:.4f} - val_loss {:.4f} '.format(
  100.                             e, step, np.mean(train_record), val_record[-1]['loss']))
  101.                 train_record = []
  102.  
  103. model = SimpleModel(text.vocab.vectors,
  104.                     padding_idx=text.vocab.stoi[text.pad_token],
  105.                     hidden_dim=128).cuda()
  106. loss_function = nn.BCEWithLogitsLoss()
  107. optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),lr=1e-3)
  108.  
  109. training(model=model,
  110.          epoch=20,
  111.          loss_func=loss_function,
  112.          optimizer=optimizer,
  113.          train_iter=train_iter,
  114.          val_iter=val_iter)
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top