Guest User

Untitled

a guest
Nov 22nd, 2017
84
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.18 KB | None | 0 0
  1. x = ['company', 'arriving', 'wednesday', 'and', 'then', 'beach', 'how', 'are', 'you', 'any', 'warmer', 'there', 'enjoy', 'your', 'day', 'follow', 'back', 'please', 'everyone', 'go', 'watch', 's', 'new', 'video', 'you', 'know', 'the', 'deal', 'make', 'sure', 'to', 'subscribe', 'and', 'like', '<http>', 'you', 'said', 'next', 'week', 'you', 'will', 'be', 'the', 'one', 'picking', 'me', 'up', 'lol', 'hindi', 'na', 'tl', 'huehue', 'that', 'works', 'you', 'said', 'everyone', 'of', 'us', 'my', 'little', 'cousin', 'keeps', 'asking', 'if', 'i', 'wanna', 'play', 'and', "i'm", 'like', 'yes', 'but', 'with', 'my', 'pals', 'not', 'you', "you're", 'welcome', 'pas', 'quand', 'tu', 'es', 'vers', '<num>', 'i', 'never', 'get', 'good', 'mornng', 'texts', 'sad', 'sad', 'moment', 'i', 'think', 'ima', 'go', 'get', 'a', 'glass', 'of', 'milk', 'ahah', 'for', 'the', 'first', 'time', 'i', 'actually', 'know', 'what', 'their', 'doing', 'd', 'thank', 'you', 'happy', 'birthday', 'hope', "you're"...........]
  2.  
  3. types = []
  4. for word in x:
  5. a.append(type(word))
  6. print set(a)
  7.  
  8. >>>set([<type 'str'>])
  9.  
  10. import nltk
  11. porter = nltk.PorterStemmer()
  12. stemmed_x = [porter.stem(word) for word in x]
  13.  
  14. File "/Library/Python/2.7/site-packages/nltk-3.0.0b2-py2.7.egg/nltk/stem/porter.py", line 633, in stem
  15. stem = self.stem_word(word.lower(), 0, len(word) - 1)
  16. File "/Library/Python/2.7/site-packages/nltk-3.0.0b2-py2.7.egg/nltk/stem/porter.py", line 591, in stem_word
  17. word = self._step1ab(word)
  18. File "/Library/Python/2.7/site-packages/nltk-3.0.0b2-py2.7.egg/nltk/stem/porter.py", line 289, in _step1ab
  19. if word.endswith("ied"):
  20. UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 12: ordinal not in range(128)
  21.  
  22. for w in x:
  23. try:
  24. porter.stem(w)
  25. except UnicodeDecodeError:
  26. print w
  27.  
  28. #sagittarius”
  29. #instadane…
  30. #bleedblue”
  31. #pr챕cieux
  32. #على_شرفة_الماضي
  33. #exploringsf…
  34. #fishing…
  35. #sindhubestfriend…
  36. #الإستعداد_لإنهيار_ال_سعود
  37. #jaredpreslar…
  38. #femalepains”
  39. #gobillings”
  40. #juicing…
  41. #instamood…
  42.  
  43. print [value for value in x if 'xe2' in x]
  44.  
  45. import nltk
  46. porter = nltk.PorterStemmer()
  47. stemmed_x = [porter.stem(word.decode('utf-8')) for word in x]
Add Comment
Please, Sign In to add comment