Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import math
- # the concordance of a document is a frequency map of the words in the document
- class VectorCompare:
- # return the frequency map of words in a document
- def concordance(self, document):
- if type(document) != str:
- raise ValueError('Supplied argument should be of type string')
- con = {}
- for word in document.split(' '):
- if word in con:
- con[word] = con[word] + 1
- else:
- con[word] = 1
- return con
- # calculate the Euclidean norm of the vector (it's length, by the Pythagorean theorem)
- def magnitude(self, concordance):
- if type(concordance) != dict:
- raise ValueError('Supplied argument should be of type dict')
- total = 0
- for word, count in concordance.items():
- total += count ** 2
- return math.sqrt(total)
- def relation(self, concordance1, concordance2):
- if type(concordance1) != dict:
- raise ValueError('Supplied Argument 1 should be of type dict')
- if type(concordance2) != dict:
- raise ValueError('Supplied Argument 2 should be of type dict')
- relevance = 0
- # calculate topvalue = max{count1*count2}
- topvalue = 0
- for word, count in concordance1.items():
- if word in concordance2:
- topvalue += count * concordance2[word]
- # return topvalue / (|v1||v2|) which is a measurement of how relevant the two
- # vectors are to each other
- if (self.magnitude(concordance1) * self.magnitude(concordance2)) != 0:
- return topvalue / (self.magnitude(concordance1) * self.magnitude(concordance2))
- else:
- return 0
- # test our class
- v = VectorCompare()
- # add some test documents
- documents = {
- 0:'''At Scale You Will Hit Every Performance Issue I used to think I knew a bit about performance scalability and how to keep things trucking when you hit large amounts of data Truth is I know diddly squat on the subject since the most I have ever done is read about how its done To understand how I came about realising this you need some background''',
- 1:'''Richard Stallman to visit Australia Im not usually one to promote events and the like unless I feel there is a genuine benefit to be had by attending but this is one stands out Richard M Stallman the guru of Free Software is coming Down Under to hold a talk You can read about him here Open Source Celebrity to visit Australia''',
- 2:'''MySQL Backups Done Easily One thing that comes up a lot on sites like Stackoverflow and the like is how to backup MySQL databases The first answer is usually use mysqldump This is all fine and good till you start to want to dump multiple databases You can do this all in one like using the all databases option however this makes restoring a single database an issue since you have to parse out the parts you want which can be a pain''',
- 3:'''Why You Shouldnt roll your own CAPTCHA At a TechEd I attended a few years ago I was watching a presentation about Security presented by Rocky Heckman read his blog its quite good In it he was talking about security algorithms The part that really stuck with me went like this''',
- 4:'''The Great Benefit of Test Driven Development Nobody Talks About The feeling of productivity because you are writing lots of code Think about that for a moment Ask any developer who wants to develop why they became a developer One of the first things that comes up is I enjoy writing code This is one of the things that I personally enjoy doing Writing code any code especially when its solving my current problem makes me feel productive It makes me feel like Im getting somewhere Its empowering''',
- 5:'''Setting up GIT to use a Subversion SVN style workflow Moving from Subversion SVN to GIT can be a little confusing at first I think the biggest thing I noticed was that GIT doesnt have a specific workflow you have to pick your own Personally I wanted to stick to my Subversion like work-flow with a central server which all my machines would pull and push too Since it took a while to set up I thought I would throw up a blog post on how to do it''',
- 6:'''Why CAPTCHA Never Use Numbers 0 1 5 7 Interestingly this sort of question pops up a lot in my referring search term stats Why CAPTCHAs never use the numbers 0 1 5 7 Its a relativity simple question with a reasonably simple answer Its because each of the above numbers are easy to confuse with a letter See the below''',
- }
- # calculate the concordances of each document
- index = {}
- for key in documents.keys():
- index[key] = v.concordance(documents[key].lower())
- # prompt the user with "Enter Search Term: " and collect their input
- searchterm = input('Enter Search Term: ')
- # list of relevant search results
- matches = []
- # for each document, calculate its relevance to the search term
- for i in range(len(index)):
- relation = v.relation(v.concordance(searchterm.lower()), index[i])
- if relation != 0:
- matches.append((relation, documents[i][:100]))
- # sort matches in descending order (most relevant first)
- matches.sort(reverse=True)
- for i in matches:
- print (i[0],i[1])
Advertisement
Add Comment
Please, Sign In to add comment