HITOMIi23

Untitled

Feb 6th, 2019
112
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 0.45 KB | None | 0 0
  1. def txtED(s):
  2.     container = set()
  3.     word_list = s.split()
  4.     for i in word_list:
  5.       a = i.lower()
  6.       b = a.join([c for c in s if c.isalpha()])
  7.       container.add(b)
  8.     return container          
  9.  
  10.  
  11. txtFile3  = sc.wholeTextFiles(args.input)
  12.         numOfDoc = txtFile3.keys().count()
  13.         RDD = txtFile3.map(lambda (doc, vals):(doc, txtED(vals))).values().flatMap(lambda word: (word, 1))\
  14.                .saveAsTextFile(args.output)
Advertisement
Add Comment
Please, Sign In to add comment