Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- x = ['company', 'arriving', 'wednesday', 'and', 'then', 'beach', 'how', 'are', 'you', 'any', 'warmer', 'there', 'enjoy', 'your', 'day', 'follow', 'back', 'please', 'everyone', 'go', 'watch', 's', 'new', 'video', 'you', 'know', 'the', 'deal', 'make', 'sure', 'to', 'subscribe', 'and', 'like', '<http>', 'you', 'said', 'next', 'week', 'you', 'will', 'be', 'the', 'one', 'picking', 'me', 'up', 'lol', 'hindi', 'na', 'tl', 'huehue', 'that', 'works', 'you', 'said', 'everyone', 'of', 'us', 'my', 'little', 'cousin', 'keeps', 'asking', 'if', 'i', 'wanna', 'play', 'and', "i'm", 'like', 'yes', 'but', 'with', 'my', 'pals', 'not', 'you', "you're", 'welcome', 'pas', 'quand', 'tu', 'es', 'vers', '<num>', 'i', 'never', 'get', 'good', 'mornng', 'texts', 'sad', 'sad', 'moment', 'i', 'think', 'ima', 'go', 'get', 'a', 'glass', 'of', 'milk', 'ahah', 'for', 'the', 'first', 'time', 'i', 'actually', 'know', 'what', 'their', 'doing', 'd', 'thank', 'you', 'happy', 'birthday', 'hope', "you're"...........]
- types = []
- for word in x:
- a.append(type(word))
- print set(a)
- >>>set([<type 'str'>])
- import nltk
- porter = nltk.PorterStemmer()
- stemmed_x = [porter.stem(word) for word in x]
- File "/Library/Python/2.7/site-packages/nltk-3.0.0b2-py2.7.egg/nltk/stem/porter.py", line 633, in stem
- stem = self.stem_word(word.lower(), 0, len(word) - 1)
- File "/Library/Python/2.7/site-packages/nltk-3.0.0b2-py2.7.egg/nltk/stem/porter.py", line 591, in stem_word
- word = self._step1ab(word)
- File "/Library/Python/2.7/site-packages/nltk-3.0.0b2-py2.7.egg/nltk/stem/porter.py", line 289, in _step1ab
- if word.endswith("ied"):
- UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 12: ordinal not in range(128)
- for w in x:
- try:
- porter.stem(w)
- except UnicodeDecodeError:
- print w
- #sagittarius”
- #instadane…
- #bleedblue”
- #pr챕cieux
- #على_شرفة_الماضي
- #exploringsf…
- #fishing…
- #sindhubestfriend…
- #الإستعداد_لإنهيار_ال_سعود
- #jaredpreslar…
- #femalepains”
- #gobillings”
- #juicing…
- #instamood…
- print [value for value in x if 'xe2' in x]
- import nltk
- porter = nltk.PorterStemmer()
- stemmed_x = [porter.stem(word.decode('utf-8')) for word in x]
Add Comment
Please, Sign In to add comment