Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import requests
- import simplejson as json
- import psycopg2 as psycopg2
- def getDB():
- return "dbname='{}' user='{}' host='{}' password='{}'".format("deps", "jason", "localhost", "redsasd" )
- def connect():
- try:
- db_info = getDB()
- con = psycopg2.connect(db_info)
- cur = con.cursor()
- except Exception as e:
- print("ERROR ON CURSOR")
- print(str(e))
- return con, cur
- def commit_n_close(con):
- try:
- con.commit()
- con.close()
- except Exception as e:
- print("ERROR ON CLOSE!!")
- print(str(e))
- def commit(con):
- try:
- con.commit()
- except Exception as e:
- print("ERROR ON CLOSE!!")
- print(str(e))
- def fetch(sql, params):
- con, cur = connect()
- cur.execute(sql, params)
- rows = cur.fetchall()
- cols = [desc[0] for desc in cur.description]
- con.close()
- return rows, cols
- con, cur = connect()
- rows, cols = fetch("SELECT phrase, vector, sims, sim_dists FROM w2v", ())
- import pickle
- def save_object(obj, name):
- with open( name + '.pkl', 'wb') as f:
- pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)
- def load_object(name):
- with open( name + '.pkl', 'rb') as f:
- return pickle.load(f, encoding="UTF8")
- depsNpen_cache = {} #load_object("depsNpen_cache")
- #freqs = load_object("freqs")
- # i = 0
- # for key in depsNpen_cache:
- # i += 1
- # #if i > 50:break
- # phrase = key.split("|")[0]
- # if phrase in freqs:
- # ants = freqs[phrase]["antonyms"]
- # sentiment = freqs[phrase]["sentiment"]
- # depsNpen_cache[key]["antonyms"] = ants
- # depsNpen_cache[key]["sentiment"] = sentiment
- j = 0
- vectors = {}
- sims = {}
- vocabulary = {}
- for r in rows:
- j += 1
- phrase = r[0]
- print(j, phrase)
- vocabulary[phrase] = True
- vectors[phrase] = [float(x) for x in r[1]]
- sims[phrase] = {r[2][i]: float(r[3][i]) for i in range(0, len(r[2]))}
- save_object(vocabulary, "deps_vocabulary")
- save_object(vectors, "deps_vectors")
- save_object(sims, "deps_sims")
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement