Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import math
- # -----
- # This function takes a word and a dictionary of
- # word counts, and it generates a score that
- # approximates the relevance of the word
- # in the document from which the word counts
- # were generated. The higher the score, the more
- # relevant the word.
- #
- # In many cases, the score returned will be
- # negative. Note that the "higher" of two
- # negative scores is the one that is less
- # negative, or the one that is closer to zero.
- def get_score(word, counts):
- denominator = float(1 + counts["_total"])
- if word in counts:
- return math.log((1 + counts[word]) / denominator)
- else:
- return math.log(1 / denominator)
- # normalize
- # CODE FROM PART 2
- # This function takes a word and returns the same word
- # with:
- # - All non-letters removed
- # - All letters converted to lowercase
- def normalize(word):
- return "".join(letter for letter in word if letter.isalpha()).lower()
- # get_counts
- # -----
- # This function takes a filename and generates a dictionary
- # whose keys are the unique words in the file and whose
- # values are the counts for those words.
- def get_counts(filename):
- word_count = 0
- result_dict = {}
- file = open(filename)
- for line in file:
- for word in line.strip().split():
- if word == " ":
- continue
- else:
- word = normalize(word)
- if word in result_dict:
- result_dict["_total"] = word_count
- else:
- result_dict[word] = 1
- return result_dict
- print("Word count: " + len(result_dict))
- print("Total word count: " + word_count)
- # Get the counts for the two shortened versions
- # of the texts
- shakespeare_counts = get_counts(open("hamlet.txt", "r"))
- austen_counts = get_counts(open("pride_and_prejudice.txt", "r"))
- #Text to predict
- TextInput = str(input("Enter the text to be predicted: "))
- #Get the counts for the text input by the user
- text_counts = get_counts(TextInput)
- #Split the text sample into words
- SplitInput = TextInput.split()
- for word in SplitInput:
- if len(word) >= 1:
- normalize(word)
- else:
- continue
Advertisement
Add Comment
Please, Sign In to add comment