Advertisement
LemmyNjaria

Python Version For Your Challenge Update2!!!

Dec 16th, 2019
184
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 10.04 KB | None | 0 0
  1. # library for calling GET Requests
  2. import urllib.request
  3. # library for mapping JSON Requests
  4. import json
  5. # library for hadling postgreSQl
  6. import psycopg2
  7. # this is the main function
  8. from psycopg2 import sql
  9.  
  10. urlData1 = "https://redirect.viglink.com/?format=go&jsonp=vglnk_157494196953111&key=949efb41171ac6ec1bf7f206d57e90b8&libId=k3iaga3z01021u9s000DAauhujszi&loc=https%3A%2F%2Fwww.r-bloggers.com%2Fopen-trade-statistics%2F&v=1&type=U&out=https%3A%2F%2Fapi.tradestatistics.io%2Fcountries&ref=https%3A%2F%2Fwww.google.com%2F&title=Open%20Trade%20Statistics%20%7C%20R-bloggers&txt=https%3A%2F%2Fapi.tradestatistics.io%2Fcountries"
  11. countryCode = "ken"
  12. year1 = "2016"
  13. year2 = "2017"
  14. year3 = "2018"
  15.  
  16. urlData3 = "https://api.tradestatistics.io/products"
  17.  
  18. version1 = "country"
  19. version2 = "products"
  20. version3 = "trade"
  21. version4 = "tradevalue"
  22.  
  23.  
  24. def main():
  25.     #fetchRowsFromDB()
  26.     print(getResponse(urlData3))
  27.  
  28.     # if(getResponseAndStoreOnDatabase1(urlData1,version1) == True):
  29.     # print("Version one is Complete")
  30.     # elif(getResponseAndStoreOnDatabase2(urlData2,version2) == True):
  31.     # print("Version two is complete")
  32.     # else:
  33.     # getResponseAndStoreOnDatabase2(urlData3,version3)
  34.     # print("Version three is complete")
  35.  
  36.  
  37. def fetchRowsFromDB():
  38.     try:
  39.         connection = psycopg2.connect(user="postgres",
  40.                                       password="123",
  41.                                       host="127.0.0.1",
  42.                                       port="5432",
  43.                                       database="ItcImports")
  44.         cursor = connection.cursor()
  45.  
  46.         cursor.execute(sql.SQL("SELECT * FROM {}.{} ORDER BY {} ASC").format(sql.Identifier('public'),
  47.                                                                              sql.Identifier('Country'),
  48.                                                                              sql.Identifier('country_id'), ))
  49.         print("Selecting rows from importValue table using cursor.fetchall")
  50.         importvalue_records = cursor.fetchall()
  51.  
  52.         print("Print each row and it's columns values")
  53.         k=0
  54.         for row in importvalue_records:
  55.             k=+1
  56.             print("the number of times = ", k)
  57.             print("1 = ", row[0])
  58.  
  59.             getResponseAndStoreOnDatabase1(row[0])
  60.             # here we are going to do the necessary
  61.  
  62.  
  63.  
  64.     except (Exception, psycopg2.Error) as error:
  65.         print("Error while fetching data from PostgreSQL", error)
  66.  
  67.     finally:
  68.         # closing database connection.
  69.         if (connection):
  70.             cursor.close()
  71.             connection.close()
  72.             print("PostgreSQL connection is closed")
  73.  
  74.  
  75. def getResponseAndStoreOnDatabase1(reporter):
  76.     urlData2 = "https://api.tradestatistics.io/yrpc?y=" + year1 + "&r=" + reporter + "&p=" + countryCode + ""
  77.  
  78.     if getResponse(urlData2) is not '':
  79.         conectToPostGres(getResponse(urlData2))
  80.         return True
  81.  
  82.  
  83. def getResponse(url):
  84.     # opens the url
  85.     operUrl = urllib.request.urlopen(url)
  86.     # cheks if the GET response is successfull
  87.     if (operUrl.getcode() == 200):
  88.         # opens the url and reads the content
  89.         data = operUrl.read()
  90.         # decods the JSON content
  91.         jsonData = json.loads(data)
  92.     else:
  93.         print("Error receiving data", operUrl.getcode())
  94.     return jsonData
  95.  
  96.  
  97. # this function handles successfull connection and storing of data to postgreSQL
  98. def conectToPostGres(jsonData):
  99.     # handles connection
  100.     # in your case you will change the password and database name umeshika!
  101.     connection = psycopg2.connect(user="postgres",
  102.                                   password="123",
  103.                                   host="127.0.0.1",
  104.                                   port="5432",
  105.                                   database="ItcImports")
  106.  
  107.     if (version1 == "country"):
  108.         for i in jsonData:
  109.             try:
  110.                 # this calls the cursor
  111.                 cursor = connection.cursor()
  112.  
  113.                 # this query helps in storying values to the table
  114.                 # in this case you will just change the household name to your table name and its column names
  115.                 postgres_insert_query = """ INSERT INTO public.household("householdNumber", "safeWater", "treatedWater") VALUES (%s,%s,%s)"""
  116.                 # define here the json objects and store them
  117.                 record_to_insert = (i["country_iso"], i["country_name_english"], i["continent"])
  118.                 # execute the cursor
  119.                 cursor.execute(postgres_insert_query, record_to_insert)
  120.                 # commit the connection
  121.                 connection.commit()
  122.                 # this counts the rows
  123.                 count = cursor.rowcount
  124.                 # this prints them if its successfull
  125.                 print(count, "Record inserted successfully into member table")
  126.  
  127.             # this handles errors if it occurs
  128.             except(Exception, psycopg2.Error) as error:
  129.                 if (connection):
  130.                     print("Failed to insert record into the table", error)
  131.  
  132.             finally:
  133.                 # closing database connection
  134.                 if (connection):
  135.                     cursor.close()
  136.                     connection.close()
  137.                     print("PostgresQl connection is closed")
  138.  
  139.     elif (version2 == "products"):
  140.  
  141.         for i in jsonData:
  142.             try:
  143.                 # this calls the cursor
  144.                 cursor = connection.cursor()
  145.  
  146.                 # this query helps in storying values to the table
  147.                 # in this case you will just change the household name to your table name and its column names
  148.                 postgres_insert_query = """ INSERT INTO public.household("householdNumber", "safeWater", "treatedWater") VALUES (%s,%s,%s)"""
  149.                 # define here the json objects and store them
  150.                 record_to_insert = (int(i["product_code"]), i["group_code"], i["group_name"])
  151.                 # execute the cursor
  152.                 cursor.execute(postgres_insert_query, record_to_insert)
  153.                 # commit the connection
  154.                 connection.commit()
  155.                 # this counts the rows
  156.                 count = cursor.rowcount
  157.                 # this prints them if its successfull
  158.                 print(count, "Record inserted successfully into member table")
  159.  
  160.             # this handles errors if it occurs
  161.             except(Exception, psycopg2.Error) as error:
  162.                 if (connection):
  163.                     print("Failed to insert record into the table", error)
  164.  
  165.             finally:
  166.                 # closing database connection
  167.                 if (connection):
  168.                     cursor.close()
  169.                     connection.close()
  170.                     print("PostgresQl connection is closed")
  171.  
  172.     elif (version3 == "trade"):
  173.  
  174.         for i in jsonData:
  175.             try:
  176.                 # this calls the cursor
  177.                 cursor = connection.cursor()
  178.  
  179.                 # this query helps in storying values to the table
  180.                 # in this case you will just change the household name to your table name and its column names
  181.                 postgres_insert_query = """ INSERT INTO public.household("householdNumber", "safeWater", "treatedWater") VALUES (%s,%s,%s)"""
  182.                 # define here the json objects and store them
  183.                 record_to_insert = (i["reporter_iso"], int(i["product_code"]), int(i["import_value_usd"]))
  184.                 # execute the cursor
  185.                 cursor.execute(postgres_insert_query, record_to_insert)
  186.                 # commit the connection
  187.                 connection.commit()
  188.                 # this counts the rows
  189.                 count = cursor.rowcount
  190.                 # this prints them if its successfull
  191.                 print(count, "Record inserted successfully into member table")
  192.  
  193.             # this handles errors if it occurs
  194.             except(Exception, psycopg2.Error) as error:
  195.                 if (connection):
  196.                     print("Failed to insert record into the table", error)
  197.  
  198.             finally:
  199.                 # closing database connection
  200.                 if (connection):
  201.                     cursor.close()
  202.                     connection.close()
  203.                     print("PostgresQl connection is closed")
  204.  
  205.     elif (version4 == "tradevalue"):
  206.         k=0
  207.         for i in jsonData:
  208.             try:
  209.                 # this calls the cursor
  210.                 cursor = connection.cursor()
  211.  
  212.                 # this query helps in storying values to the table
  213.                 # in this case you will just change the household name to your table name and its column names
  214.                 postgres_insert_query = """ INSERT INTO public."Importvalue"("ReporterIso", "Productcode", "ImportUSD", "Year", "ExportUSD") VALUES (%s,%s,%s,%s,%s) """
  215.                 # define here the json objects and store them
  216.  
  217.                 record_to_insert = (i["reporter_iso"], int(i["product_code"]), int(i["import_value_usd"]), int(i["year"]),int(i["export_value_usd"]))
  218.                 # execute the cursor
  219.                 cursor.execute(postgres_insert_query, record_to_insert)
  220.                 # commit the connection
  221.                 connection.commit()
  222.                 # this counts the rows
  223.                 count = cursor.rowcount
  224.                 # this prints them if its successfull
  225.                 print(count, "Record inserted successfully into member table")
  226.                 k=+1
  227.                 print(k, " the number of the function called!!!")
  228.  
  229.             # this handles errors if it occurs
  230.             except(Exception, psycopg2.Error) as error:
  231.                 if (connection):
  232.                     print("Failed to insert record into the table", error)
  233.  
  234.             finally:
  235.                 # closing database connection
  236.                 if (connection):
  237.                     cursor.close()
  238.                     connection.close()
  239.                     print("PostgresQl connection is closed")
  240.  
  241.  
  242. # this one calls the main function
  243. if __name__ == '__main__':
  244.     # this is the main function
  245.     main()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement