Advertisement
Guest User

Untitled

a guest
Feb 21st, 2019
112
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.54 KB | None | 0 0
  1. import datetime
  2. import pickle
  3. import time
  4. from datetime import timedelta
  5. import ftplib
  6. import MySQLdb
  7. import sys
  8. from sys import argv
  9. import os
  10. import urllib
  11. import subprocess
  12. from ftplib import FTP
  13. from time import sleep
  14. import requests
  15. from rets import Session
  16.  
  17.  
  18.  
  19. a = datetime.datetime.strftime(datetime.datetime.now(),'%Y-%m-%d %H:%m:%S')
  20.  
  21.  
  22. db2 = MySQLdb.connect(host="127.0.0.1", # your host, usually localhost
  23. charset='utf8',
  24. user="", # your username
  25. passwd="", # your password
  26. db="") # name of the data base
  27.  
  28. cur2 = db2.cursor()
  29.  
  30. cur2.execute('SET NAMES utf8;')
  31. cur2.execute('SET CHARACTER SET utf8;')
  32. cur2.execute('SET character_set_connection=utf8;')
  33.  
  34. cur2.execute("SELECT MAX(lastupdate) FROM updates where name = 'media'")
  35. max_time_string = cur2.fetchone()[0]
  36. max_time_string = str(max_time_string)
  37. max_datetime = datetime.strptime(max_time_string, "%Y-%m-%d %H:%M:%S")
  38. search_datetime = max_datetime - timedelta(minutes=0)
  39. search_time_string = datetime.strftime(search_datetime, "%Y-%m-%dT%H:%M:%S")
  40.  
  41. login_url = ''
  42. username = ''
  43. password = ''
  44. rets_client = Session(login_url, username, password)
  45. rets_client.login()
  46. system_data = rets_client.get_system_metadata()
  47. print system_data
  48. resources = rets_client.get_resource_metadata(resource='Agent')
  49. resources
  50. print "Getting records since", max_time_string
  51. search_results = rets_client.search(resource='media', resource_class='media', dmql_query='(Modified='+search_time_string+'+)') #change search query here
  52. print "Inserting", len(search_results), "records now"
  53. chunk_size = 40000
  54. chunks = []
  55. chunk_size = max(1, chunk_size)
  56. for i in range(0, len(search_results), chunk_size):
  57. chunk = search_results[i:i+chunk_size]
  58. chunks.append(chunk)
  59.  
  60. for chunk in chunks:
  61.  
  62. placeholders = ', '.join(['%s'] * len(chunk[0]))
  63. columns = "`%s`" % '`,`'.join(chunk[0].keys())
  64. update_string = ""
  65. for key in chunk[0]:
  66. update_string = update_string + "`" + key + "`=VALUES(`" + key + "`),"
  67. update_string = update_string[:-1]
  68.  
  69. sql = "INSERT IGNORE INTO media ( %s ) VALUES ( %s ) ON DUPLICATE KEY UPDATE %s" % (columns, placeholders, update_string) #change table here
  70.  
  71. chunk_values = []
  72. for result in chunk:
  73. chunk_values.append(tuple(result.values()))
  74.  
  75. success = False
  76.  
  77. try:
  78. cur2.executemany(sql, chunk_values)
  79. db2.commit()
  80. print ("Records inserted")
  81. success = True
  82. except:
  83. db2.rollback()
  84.  
  85. if not success:
  86.  
  87. for result in chunk:
  88.  
  89. placeholders = ', '.join(['%s'] * len(result))
  90. columns = "`%s`" % '`,`'.join(result.keys())
  91. update_string = ""
  92. for key in result:
  93. update_string = update_string + "`" + key + "`=VALUES(`" + key + "`),"
  94. update_string = update_string[:-1]
  95.  
  96. sql = "INSERT INTO media ( %s ) VALUES ( %s ) ON DUPLICATE KEY UPDATE %s" % (columns, placeholders, update_string) #change table here
  97.  
  98. print (sql)
  99. cur2.execute(sql, result.values())
  100. print len(search_results), "records inserted!"
  101. db2.commit()
  102. print "Processed", len(search_results), "results, quitting."
  103. cur2.execute("INSERT INTO updates (name, lastupdate, lastupdatecount) VALUES (%s, %s, '%s')",("media", len(search_results), a))
  104. db2.commit()
  105. exit()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement