Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # library for calling GET Requests
- import urllib.request
- # library for mapping JSON Requests
- import json
- # library for hadling postgreSQl
- import psycopg2
- # this is the main function
- from psycopg2 import sql
- urlData1 = "https://redirect.viglink.com/?format=go&jsonp=vglnk_157494196953111&key=949efb41171ac6ec1bf7f206d57e90b8&libId=k3iaga3z01021u9s000DAauhujszi&loc=https%3A%2F%2Fwww.r-bloggers.com%2Fopen-trade-statistics%2F&v=1&type=U&out=https%3A%2F%2Fapi.tradestatistics.io%2Fcountries&ref=https%3A%2F%2Fwww.google.com%2F&title=Open%20Trade%20Statistics%20%7C%20R-bloggers&txt=https%3A%2F%2Fapi.tradestatistics.io%2Fcountries"
- countryCode = "ken"
- year1 = "2016"
- year2 = "2017"
- year3 = "2018"
- urlData3 = "https://api.tradestatistics.io/products"
- version1 = "country"
- version2 = "products"
- version3 = "trade"
- version4 = "tradevalue"
- def main():
- #fetchRowsFromDB()
- print(getResponse(urlData3))
- # if(getResponseAndStoreOnDatabase1(urlData1,version1) == True):
- # print("Version one is Complete")
- # elif(getResponseAndStoreOnDatabase2(urlData2,version2) == True):
- # print("Version two is complete")
- # else:
- # getResponseAndStoreOnDatabase2(urlData3,version3)
- # print("Version three is complete")
- def fetchRowsFromDB():
- try:
- connection = psycopg2.connect(user="postgres",
- password="123",
- host="127.0.0.1",
- port="5432",
- database="ItcImports")
- cursor = connection.cursor()
- cursor.execute(sql.SQL("SELECT * FROM {}.{} ORDER BY {} ASC").format(sql.Identifier('public'),
- sql.Identifier('Country'),
- sql.Identifier('country_id'), ))
- print("Selecting rows from importValue table using cursor.fetchall")
- importvalue_records = cursor.fetchall()
- print("Print each row and it's columns values")
- k=0
- for row in importvalue_records:
- k=+1
- print("the number of times = ", k)
- print("1 = ", row[0])
- getResponseAndStoreOnDatabase1(row[0])
- # here we are going to do the necessary
- except (Exception, psycopg2.Error) as error:
- print("Error while fetching data from PostgreSQL", error)
- finally:
- # closing database connection.
- if (connection):
- cursor.close()
- connection.close()
- print("PostgreSQL connection is closed")
- def getResponseAndStoreOnDatabase1(reporter):
- urlData2 = "https://api.tradestatistics.io/yrpc?y=" + year1 + "&r=" + reporter + "&p=" + countryCode + ""
- if getResponse(urlData2) is not '':
- conectToPostGres(getResponse(urlData2))
- return True
- def getResponse(url):
- # opens the url
- operUrl = urllib.request.urlopen(url)
- # cheks if the GET response is successfull
- if (operUrl.getcode() == 200):
- # opens the url and reads the content
- data = operUrl.read()
- # decods the JSON content
- jsonData = json.loads(data)
- else:
- print("Error receiving data", operUrl.getcode())
- return jsonData
- # this function handles successfull connection and storing of data to postgreSQL
- def conectToPostGres(jsonData):
- # handles connection
- # in your case you will change the password and database name umeshika!
- connection = psycopg2.connect(user="postgres",
- password="123",
- host="127.0.0.1",
- port="5432",
- database="ItcImports")
- if (version1 == "country"):
- for i in jsonData:
- try:
- # this calls the cursor
- cursor = connection.cursor()
- # this query helps in storying values to the table
- # in this case you will just change the household name to your table name and its column names
- postgres_insert_query = """ INSERT INTO public.household("householdNumber", "safeWater", "treatedWater") VALUES (%s,%s,%s)"""
- # define here the json objects and store them
- record_to_insert = (i["country_iso"], i["country_name_english"], i["continent"])
- # execute the cursor
- cursor.execute(postgres_insert_query, record_to_insert)
- # commit the connection
- connection.commit()
- # this counts the rows
- count = cursor.rowcount
- # this prints them if its successfull
- print(count, "Record inserted successfully into member table")
- # this handles errors if it occurs
- except(Exception, psycopg2.Error) as error:
- if (connection):
- print("Failed to insert record into the table", error)
- finally:
- # closing database connection
- if (connection):
- cursor.close()
- connection.close()
- print("PostgresQl connection is closed")
- elif (version2 == "products"):
- for i in jsonData:
- try:
- # this calls the cursor
- cursor = connection.cursor()
- # this query helps in storying values to the table
- # in this case you will just change the household name to your table name and its column names
- postgres_insert_query = """ INSERT INTO public.household("householdNumber", "safeWater", "treatedWater") VALUES (%s,%s,%s)"""
- # define here the json objects and store them
- record_to_insert = (int(i["product_code"]), i["group_code"], i["group_name"])
- # execute the cursor
- cursor.execute(postgres_insert_query, record_to_insert)
- # commit the connection
- connection.commit()
- # this counts the rows
- count = cursor.rowcount
- # this prints them if its successfull
- print(count, "Record inserted successfully into member table")
- # this handles errors if it occurs
- except(Exception, psycopg2.Error) as error:
- if (connection):
- print("Failed to insert record into the table", error)
- finally:
- # closing database connection
- if (connection):
- cursor.close()
- connection.close()
- print("PostgresQl connection is closed")
- elif (version3 == "trade"):
- for i in jsonData:
- try:
- # this calls the cursor
- cursor = connection.cursor()
- # this query helps in storying values to the table
- # in this case you will just change the household name to your table name and its column names
- postgres_insert_query = """ INSERT INTO public.household("householdNumber", "safeWater", "treatedWater") VALUES (%s,%s,%s)"""
- # define here the json objects and store them
- record_to_insert = (i["reporter_iso"], int(i["product_code"]), int(i["import_value_usd"]))
- # execute the cursor
- cursor.execute(postgres_insert_query, record_to_insert)
- # commit the connection
- connection.commit()
- # this counts the rows
- count = cursor.rowcount
- # this prints them if its successfull
- print(count, "Record inserted successfully into member table")
- # this handles errors if it occurs
- except(Exception, psycopg2.Error) as error:
- if (connection):
- print("Failed to insert record into the table", error)
- finally:
- # closing database connection
- if (connection):
- cursor.close()
- connection.close()
- print("PostgresQl connection is closed")
- elif (version4 == "tradevalue"):
- k=0
- for i in jsonData:
- try:
- # this calls the cursor
- cursor = connection.cursor()
- # this query helps in storying values to the table
- # in this case you will just change the household name to your table name and its column names
- postgres_insert_query = """ INSERT INTO public."Importvalue"("ReporterIso", "Productcode", "ImportUSD", "Year", "ExportUSD") VALUES (%s,%s,%s,%s,%s) """
- # define here the json objects and store them
- record_to_insert = (i["reporter_iso"], int(i["product_code"]), int(i["import_value_usd"]), int(i["year"]),int(i["export_value_usd"]))
- # execute the cursor
- cursor.execute(postgres_insert_query, record_to_insert)
- # commit the connection
- connection.commit()
- # this counts the rows
- count = cursor.rowcount
- # this prints them if its successfull
- print(count, "Record inserted successfully into member table")
- k=+1
- print(k, " the number of the function called!!!")
- # this handles errors if it occurs
- except(Exception, psycopg2.Error) as error:
- if (connection):
- print("Failed to insert record into the table", error)
- finally:
- # closing database connection
- if (connection):
- cursor.close()
- connection.close()
- print("PostgresQl connection is closed")
- # this one calls the main function
- if __name__ == '__main__':
- # this is the main function
- main()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement