Advertisement
Guest User

Untitled

a guest
Apr 24th, 2017
73
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.51 KB | None | 0 0
  1. def load_data(filename):
  2.  
  3. traina = cPickle.load( open( filename, "rb" ) )
  4.  
  5. j = 0
  6. i = 0
  7. temp = None
  8. pairs = []
  9.  
  10. for r in traina:
  11. d = spacy.tokens.doc.Doc(nlp.vocab)
  12. d.from_bytes(r)
  13. e = spacy.tokens.doc.Doc(nlp.vocab, words=[unicode(w) for w in d if not w.is_stop])
  14.  
  15. if i % 2 == 1:
  16. pairs.append((e, temp))
  17. j += 1
  18. else:
  19. temp = e
  20.  
  21. if i % 100 == 0:
  22. print(i)
  23.  
  24. i += 1
  25.  
  26. return pairs
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement