Guest User

Untitled

a guest
Apr 19th, 2018
117
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.80 KB | None | 0 0
  1. import pyodbc
  2. SQL_ALL_TABLE_QUERY="SELECT TABLE_NAME FROM {}.INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE';"
  3. SELECT_ALL_SQL_QUERY="SELECT * FROM [{0}] ORDER BY 1 OFFSET {1} ROWS FETCH NEXT {2} ROWS ONLY;"
  4.  
  5. class MongoStorage:
  6.  
  7. def __init__(self, db_name):
  8. self.client = MongoClient('localhost:27017',username='test',password='test123', authSource='admin',authMechanism='SCRAM-SHA-1')
  9. self.db = self.client[db_name]
  10.  
  11. def custom_insert_bulk(self, data, table_name):
  12. collection = self.db[table_name]
  13. return collection.insert_many(data)
  14.  
  15. mongo_handle = MongoStorage(db_name="test_db"))
  16.  
  17. conn = pyodbc.connect(
  18. "DRIVER={ODBC Driver 17 for SQL Server};" +
  19. "SERVER={0};DATABASE={1};UID={2};PWD={3};PORT={4}".format(
  20. db.get("hostname"),
  21. db.get("db_name"),
  22. self.username,
  23. self.password,
  24. int(db.get("port"))
  25. )
  26. )
  27. row_limit = 1000
  28. for table in table_list:
  29. offset = 0
  30. page = 2
  31. next_page = True
  32. while next_page:
  33. if sql:
  34. query = settings.SELECT_ALL_SQL_QUERY.format(table, offset, row_limit)
  35. elif mysql:
  36. query = settings.SELECT_ALL_MYSQL_QUERY.format(table, offset, row_limit)
  37. else:
  38. continue
  39. cursor.execute(query)
  40. columns = [column[0].encode("utf-8") for column in cursor.description]
  41. result = []
  42. query_result = cursor.fetchall()
  43. if 0 < len(query_result) <= row_limit:
  44. if len(query_result) < row_limit:
  45. next_page = False
  46. for row in query_result:
  47. row_items = []
  48. for item in row:
  49. if isinstance(item, decimal.Decimal):
  50. row_items.append(Decimal128(item))
  51. elif isinstance(item, datetime.date):
  52. row_items.append( datetime.datetime.combine( item, datetime.time()))
  53. elif isinstance(item, str):
  54. item = item.strip()
  55. if "" in item or len(item) >= 10000000:
  56. continue
  57. row_items.append(item.decode("utf-8", errors='ignore').encode("utf-8"))
  58. else:
  59. if not sys.getsizeof(item) >= 10000000:
  60. row_items.append(item)
  61. result.append(dict(zip(columns, row_items)))
  62. mongo_handle.custom_insert_bulk(data=result, table_name=table)
  63.  
  64. total used free shared buff/cache available
  65. Mem: 7982 4566 221 28 3194 3073
  66. Swap: 0 0 0
Add Comment
Please, Sign In to add comment