Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def load_data(filename):
- traina = cPickle.load( open( filename, "rb" ) )
- j = 0
- i = 0
- temp = None
- pairs = []
- for r in traina:
- d = spacy.tokens.doc.Doc(nlp.vocab)
- d.from_bytes(r)
- e = spacy.tokens.doc.Doc(nlp.vocab, words=[unicode(w) for w in d if not w.is_stop])
- if i % 2 == 1:
- pairs.append((e, temp))
- j += 1
- else:
- temp = e
- if i % 100 == 0:
- print(i)
- i += 1
- return pairs
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement