Advertisement
Guest User

Untitled

a guest
Apr 15th, 2018
118
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.08 KB | None | 0 0
  1. import requests
  2. import simplejson as json
  3. import psycopg2 as psycopg2
  4. def getDB():
  5. return "dbname='{}' user='{}' host='{}' password='{}'".format("deps", "jason", "localhost", "redsasd" )
  6.  
  7.  
  8. def connect():
  9. try:
  10. db_info = getDB()
  11. con = psycopg2.connect(db_info)
  12. cur = con.cursor()
  13. except Exception as e:
  14. print("ERROR ON CURSOR")
  15. print(str(e))
  16. return con, cur
  17.  
  18.  
  19. def commit_n_close(con):
  20. try:
  21. con.commit()
  22. con.close()
  23. except Exception as e:
  24. print("ERROR ON CLOSE!!")
  25. print(str(e))
  26.  
  27. def commit(con):
  28. try:
  29. con.commit()
  30. except Exception as e:
  31. print("ERROR ON CLOSE!!")
  32. print(str(e))
  33.  
  34.  
  35.  
  36. def fetch(sql, params):
  37. con, cur = connect()
  38. cur.execute(sql, params)
  39. rows = cur.fetchall()
  40. cols = [desc[0] for desc in cur.description]
  41. con.close()
  42. return rows, cols
  43.  
  44. con, cur = connect()
  45. rows, cols = fetch("SELECT phrase, vector, sims, sim_dists FROM w2v", ())
  46.  
  47.  
  48. import pickle
  49. def save_object(obj, name):
  50. with open( name + '.pkl', 'wb') as f:
  51. pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)
  52.  
  53. def load_object(name):
  54. with open( name + '.pkl', 'rb') as f:
  55. return pickle.load(f, encoding="UTF8")
  56.  
  57. depsNpen_cache = {} #load_object("depsNpen_cache")
  58. #freqs = load_object("freqs")
  59.  
  60. # i = 0
  61. # for key in depsNpen_cache:
  62. # i += 1
  63. # #if i > 50:break
  64. # phrase = key.split("|")[0]
  65. # if phrase in freqs:
  66. # ants = freqs[phrase]["antonyms"]
  67. # sentiment = freqs[phrase]["sentiment"]
  68. # depsNpen_cache[key]["antonyms"] = ants
  69. # depsNpen_cache[key]["sentiment"] = sentiment
  70.  
  71.  
  72. j = 0
  73. vectors = {}
  74. sims = {}
  75. vocabulary = {}
  76. for r in rows:
  77. j += 1
  78. phrase = r[0]
  79. print(j, phrase)
  80. vocabulary[phrase] = True
  81. vectors[phrase] = [float(x) for x in r[1]]
  82. sims[phrase] = {r[2][i]: float(r[3][i]) for i in range(0, len(r[2]))}
  83.  
  84. save_object(vocabulary, "deps_vocabulary")
  85. save_object(vectors, "deps_vectors")
  86. save_object(sims, "deps_sims")
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement