Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def txtED(s):
- container = set()
- word_list = s.split()
- for i in word_list:
- a = i.lower()
- b = a.join([c for c in s if c.isalpha()])
- container.add(b)
- return container
- txtFile3 = sc.wholeTextFiles(args.input)
- numOfDoc = txtFile3.keys().count()
- RDD = txtFile3.map(lambda (doc, vals):(doc, txtED(vals))).values().flatMap(lambda word: (word, 1))\
- .saveAsTextFile(args.output)
Advertisement
Add Comment
Please, Sign In to add comment