Guest User

Untitled

a guest
Oct 20th, 2017
87
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 5.35 KB | None | 0 0
  1. class Indexer:
  2. def __init__(self, numberOfFeatures=75, numberOfClasses=3):
  3. self.numberOfFeatures = numberOfFeatures
  4. self.numberOfClasses = numberOfClasses
  5. self.Xindex = AnnoyIndex(75)
  6. self.Yindex = AnnoyIndex(3)
  7.  
  8. self.Xindex.verbose(1)
  9. self.Yindex.verbose(1)
  10.  
  11. self.bufferXlist = []
  12. self.bufferYlist = []
  13.  
  14. self.built = False
  15. self.searchNumber = 25
  16.  
  17. def buildIndex(self, numOfTrees=500):
  18. print("X Items before build : %s" % self.Xindex.get_n_items())
  19. print("Y Items before build : %s" % self.Yindex.get_n_items())
  20. print("\nBuilding index with %s trees ..." % numOfTrees)
  21. self.Xindex.build(numOfTrees)
  22. self.Yindex.build(10)
  23. self.built = True
  24. print("X Items after build : %s" % self.Xindex.get_n_items())
  25. print("Y Items after build : %s" % self.Yindex.get_n_items())
  26. print("Index built OK\n")
  27.  
  28. def addToIndex(self, X, Y):
  29. if (self.built == False):
  30. index = self.generateIndexNumber()
  31. self.Xindex.add_item(index, X)
  32. self.Yindex.add_item(index, Y)
  33. else:
  34. raise Exception('Cannot add items to built indices')
  35.  
  36. def generateIndexNumber(self):
  37. return self.Xindex.get_n_items() + 1
  38.  
  39. def predict(self, X, debug=False):
  40. testSearch = self.Xindex.get_nns_by_vector(X, self.searchNumber, include_distances=True)
  41. indices, distances = testSearch
  42.  
  43. if debug:
  44. print("\n Indices")
  45. print(indices)
  46. print("\n Distances")
  47. print(distances)
  48. print("\n Length")
  49. print(self.Yindex.get_n_items())
  50.  
  51. for inex in indices:
  52. if inex > self.Yindex.get_n_items():
  53. print(inex)
  54.  
  55. mappedAnswers = [self.Yindex.get_item_vector(indexInTrainSet) for indexInTrainSet in indices]
  56.  
  57. weightedMovementLow = np.rint(np.average([a[0] for a in mappedAnswers], weights=[1 - x for x in distances]))
  58. weightedMovementHigh = np.rint(np.average([a[1] for a in mappedAnswers], weights=[1 - x for x in distances]))
  59. weightedResult = np.rint(np.average([a[2] for a in mappedAnswers], weights=[1 - x for x in distances]))
  60.  
  61. return [weightedMovementLow, weightedMovementHigh, weightedResult]
  62.  
  63. def addItems(self, Xlist, Ylist):
  64.  
  65. if (len(Xlist) != len(Ylist)):
  66. print("Xlist length does not match Ylist length")
  67. return False
  68.  
  69. for i in range(len(Xlist)):
  70. self.addToIndex(Xlist[i], Ylist[i])
  71.  
  72. return True
  73.  
  74. def testItems(self, XlistTest, YlistTest, printDebug=False):
  75.  
  76. if (len(XlistTest) != len(YlistTest)):
  77. print("XlistTest length does not match YlistTest length")
  78. return False
  79.  
  80. results = []
  81. # for i in tqdm(range(len(XlistTest)), desc="Running tests", ncols=100, leave=True):
  82. for i in range(len(XlistTest)):
  83. prediction = self.predict(XlistTest[i])
  84. results.append(prediction[2] == YlistTest[i][2])
  85.  
  86. all = len(results)
  87. good = results.count(True)
  88. percentage = ((good / all) * 100)
  89. if (printDebug == True):
  90. print("All %s" % all)
  91. print("Good %s" % good)
  92. print("Ok to %s %% " % percentage)
  93.  
  94. return percentage
  95.  
  96. def bufferItemsToTest(self, Xlist, Ylist):
  97. self.bufferXlist.extend(Xlist)
  98. self.bufferYlist.extend(Ylist)
  99.  
  100. def runBufferedTests(self):
  101. return self.testItems(self.bufferXlist, self.bufferYlist, True)
  102.  
  103. def saveIndex(self, fileName):
  104. print("\nSaving indices...")
  105. dirPath = path.dirname(fileName)
  106. filePath = path.basename(fileName)
  107. if not path.exists(dirPath):
  108. makedirs(dirPath, exist_ok=True)
  109.  
  110. print(self.Xindex)
  111. print(self.Yindex)
  112.  
  113. # 1516931
  114. print("\t saving to : %s/X-%s" % (dirPath, filePath))
  115. print("\t saving to : %s/Y-%s" % (dirPath, filePath))
  116.  
  117. print("X Items before save : %s" % self.Xindex.get_n_items())
  118. print("Y Items before save : %s" % self.Yindex.get_n_items())
  119.  
  120. a = self.Xindex.save("%s/X-%s" % (dirPath, filePath))
  121. b = self.Yindex.save("%s/Y-%s" % (dirPath, filePath))
  122.  
  123. print("Save resulted : %s" % a)
  124. print("Save resulted : %s" % b)
  125.  
  126. print("X Items after save : %s" % self.Xindex.get_n_items())
  127. print("Y Items after save : %s" % self.Yindex.get_n_items())
  128.  
  129. print("\nIndices saved OK")
  130. return True
  131.  
  132. def load(self, fileName):
  133. dirPath = path.dirname(fileName)
  134. filePath = path.basename(fileName)
  135.  
  136. # 1516931
  137. print("\t loading from : %s/X-%s" % (dirPath, filePath))
  138. print("\t loading from : %s/Y-%s" % (dirPath, filePath))
  139.  
  140. self.Xindex.unload()
  141. self.Yindex.unload()
  142.  
  143. o = AnnoyIndex(self.numberOfFeatures)
  144. p = AnnoyIndex(self.numberOfClasses)
  145.  
  146. a = o.load("%s/X-%s" % (dirPath, filePath))
  147. b = p.load("%s/Y-%s" % (dirPath, filePath))
  148.  
  149. print("Load resulted : %s" % a)
  150. print("Load resulted : %s" % b)
  151. print("\nLoaded %s" % (fileName))
  152. print("Loaded %i items to X index" % (self.Xindex.get_n_items()))
  153. print("Loaded %i items to Y index" % (self.Yindex.get_n_items()))
  154. print("\nIndices loaded OK")
  155.  
  156. self.Xindex = o
  157. # self.Yindex = p
  158. return True
Add Comment
Please, Sign In to add comment