Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # html-to-freq-2.py
- import fileinput
- import argparse
- import re
- from collections import defaultdict
- parser = argparse.ArgumentParser(description='Count word frequencies')
- parser.add_argument('files', metavar='file.txt', nargs='+',
- help='files to read')
- parser.add_argument('--word', dest='word',
- help='word to lookup')
- parser.add_argument('--bigram', dest='bigram',
- help='bigram to lookup')
- args = parser.parse_args()
- stopwords = ['a', 'about', 'above', 'across', 'after', 'afterwards']
- stopwords += ['again', 'against', 'all', 'almost', 'alone', 'along']
- stopwords += ['already', 'also', 'although', 'always', 'am', 'among']
- stopwords += ['amongst', 'amoungst', 'amount', 'an', 'and', 'another']
- stopwords += ['any', 'anyhow', 'anyone', 'anything', 'anyway', 'anywhere']
- stopwords += ['are', 'around', 'as', 'at', 'back', 'be', 'became']
- stopwords += ['because', 'become', 'becomes', 'becoming', 'been']
- stopwords += ['before', 'beforehand', 'behind', 'being', 'below']
- stopwords += ['beside', 'besides', 'between', 'beyond', 'bill', 'both']
- stopwords += ['bottom', 'but', 'by', 'call', 'can', 'cannot', 'cant']
- stopwords += ['co', 'computer', 'con', 'could', 'couldnt', 'cry', 'de']
- stopwords += ['describe', 'detail', 'did', 'do', 'done', 'down', 'due']
- stopwords += ['during', 'each', 'eg', 'eight', 'either', 'eleven', 'else']
- stopwords += ['elsewhere', 'empty', 'enough', 'etc', 'even', 'ever']
- stopwords += ['every', 'everyone', 'everything', 'everywhere', 'except']
- stopwords += ['few', 'fifteen', 'fifty', 'fill', 'find', 'fire', 'first']
- stopwords += ['five', 'for', 'former', 'formerly', 'forty', 'found']
- stopwords += ['four', 'from', 'front', 'full', 'further', 'get', 'give']
- stopwords += ['go', 'had', 'has', 'hasnt', 'have', 'he', 'hence', 'her']
- stopwords += ['here', 'hereafter', 'hereby', 'herein', 'hereupon', 'hers']
- stopwords += ['herself', 'him', 'himself', 'his', 'how', 'however']
- stopwords += ['hundred', 'i', 'ie', 'if', 'in', 'inc', 'indeed']
- stopwords += ['interest', 'into', 'is', 'it', 'its', 'itself', 'keep']
- stopwords += ['last', 'latter', 'latterly', 'least', 'less', 'ltd', 'made']
- stopwords += ['many', 'may', 'me', 'meanwhile', 'might', 'mill', 'mine']
- stopwords += ['more', 'moreover', 'most', 'mostly', 'move', 'much']
- stopwords += ['must', 'my', 'myself', 'name', 'namely', 'neither', 'never']
- stopwords += ['nevertheless', 'next', 'nine', 'no', 'nobody', 'none']
- stopwords += ['noone', 'nor', 'not', 'nothing', 'now', 'nowhere', 'of']
- stopwords += ['off', 'often', 'on','once', 'one', 'only', 'onto', 'or']
- stopwords += ['other', 'others', 'otherwise', 'our', 'ours', 'ourselves']
- stopwords += ['out', 'over', 'own', 'part', 'per', 'perhaps', 'please']
- stopwords += ['put', 'rather', 're', 's', 'same', 'see', 'seem', 'seemed']
- stopwords += ['seeming', 'seems', 'serious', 'several', 'she', 'should']
- stopwords += ['show', 'side', 'since', 'sincere', 'six', 'sixty', 'so']
- stopwords += ['some', 'somehow', 'someone', 'something', 'sometime']
- stopwords += ['sometimes', 'somewhere', 'still', 'such', 'system', 'take']
- stopwords += ['ten', 'than', 'that', 'the', 'their', 'them', 'themselves']
- stopwords += ['then', 'thence', 'there', 'thereafter', 'thereby']
- stopwords += ['therefore', 'therein', 'thereupon', 'these', 'they']
- stopwords += ['thick', 'thin', 'third', 'this', 'those', 'though', 'three']
- stopwords += ['three', 'through', 'throughout', 'thru', 'thus', 'to']
- stopwords += ['together', 'too', 'top', 'toward', 'towards', 'twelve']
- stopwords += ['twenty', 'two', 'un', 'under', 'until', 'up', 'upon']
- stopwords += ['us', 'very', 'via', 'was', 'we', 'well', 'were', 'what']
- stopwords += ['whatever', 'when', 'whence', 'whenever', 'where']
- stopwords += ['whereafter', 'whereas', 'whereby', 'wherein', 'whereupon']
- stopwords += ['wherever', 'whether', 'which', 'while', 'whither', 'who']
- stopwords += ['whoever', 'whole', 'whom', 'whose', 'why', 'will', 'with']
- stopwords += ['within', 'without', 'would', 'yet', 'you', 'your']
- stopwords += ['yours', 'yourself', 'yourselves']
- def stripNonAlphaNum(text):
- #the characters between the [] will be stripped from the text, single quotes and dashes are not included
- #since they're in the middle of compount words or contractions
- return re.compile(r'[.,"?!@#$%^&*()]', re.UNICODE).sub('', text)
- def splitWords(text):
- return re.compile(r'\s+', re.UNICODE).split(re.compile(r'\n', re.UNICODE).sub(' ', text))
- def wordListToFreqDict(wordlist):
- #change the .2 to change the number of decimal places displayed
- wordfreq = ['{:.4%} ({}/{})'.format((wordlist.count(p)/len(wordlist)), wordlist.count(p), len(wordlist)) for p in wordlist]
- return dict(zip(wordlist,wordfreq))
- def sortFreqDict(freqdict):
- aux = [(freqdict[key], key) for key in freqdict]
- aux.sort()
- aux.reverse()
- return aux
- def removeStopwords(wordlist, stopwords):
- return [w for w in wordlist if w not in stopwords]
- def findNextWordFreq(text, word):
- nextWords = defaultdict(lambda: 0)
- search = f"{word}\s+([\w|'|-]+)\s+"
- #print(search)
- for m in re.finditer(search, text):
- #print(m.group(0))
- #print(m.group(1))
- nextWords[m.group(1)] += 1
- return nextWords
- text = ""
- for line in fileinput.input(args.files):
- text += line
- t = stripNonAlphaNum(text.lower())
- if args.word is not None:
- nextWords = findNextWordFreq(t, args.word)
- print(nextWords.items())
- total = 0
- for w in nextWords: total += nextWords[w]
- for w in nextWords: print([w, nextWords[w], '{:.4%}'.format(nextWords[w]/total)])
- elif args.bigram is not None:
- nextWords = findNextWordFreq(t, args.bigram)
- for w in nextWords: print(f"{args.bigram} {w}")
- else:
- fullwordlist = splitWords(t)
- strippedWordlist = removeStopwords(fullwordlist, stopwords)
- dictionary = wordListToFreqDict(strippedWordlist)
- sorteddict = sortFreqDict(dictionary)
- #print("Total words: %i" % len(strippedWordlist))
- for s in sorteddict: print(str(s))
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement