Advertisement
Typhoon

ZVJS SQLITE to ELK 5

Apr 24th, 2015
320
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.75 KB | None | 0 0
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3.  
  4. import sqlite3
  5. import json
  6. import unicodedata
  7. from elasticsearch import Elasticsearch
  8.  
  9. conn = sqlite3.connect('SK_Prison_and_Court_Guard.sqlite')
  10. conn.row_factory = sqlite3.Row
  11. curs = conn.cursor()
  12. cursor = conn.execute('select * from data;')
  13. last_sqlite_id = len(cursor.fetchall())
  14.  
  15. print "Last SQLite ID from DB: ", last_sqlite_id
  16.  
  17. es = Elasticsearch(host='elk.myserver.sk', port=9200, http_auth=('auth_user', 'auth_pass'),)
  18.  
  19. response = es.search(
  20.     index="testpy",
  21.     body={
  22.         "query": {
  23.             "bool": {
  24.                 "must": [{"match": {"_type": "testpy"}}]
  25.             }
  26.         },
  27.         "aggs": {
  28.             "max_price": {"max": {"field": "invoice_id"}}},
  29.         "sort": {"invoice_id": {"order": "desc"}},
  30.         "size": 1
  31.     }
  32. )
  33.  
  34. for hit in response['hits']['hits']:
  35.     last_elastic_id = (hit['_source']['invoice_id'])
  36.     print "Last Elasticsearch ID is: ", (last_elastic_id)
  37.  
  38. # Temp WorkAround if isn't nothing in index
  39. # If is empty use counter with number 1
  40. # last_sqlite_id = 999999
  41. counter = last_elastic_id + 1
  42.  
  43. while counter < last_sqlite_id:
  44.     try:
  45.         d = str(counter)
  46.         curs.execute("SELECT * FROM data WHERE invoice_id=" + d + "")
  47.         recs = curs.fetchall()
  48.         rows = [dict(rec) for rec in recs]
  49.         rows_json = json.dumps(rows)
  50.         chunk = unicodedata.normalize('NFKD', unicode(rows_json, 'utf-8', 'ignore')).encode('ASCII', 'ignore').replace(
  51.             '[', '').replace(']', '').replace('"null"', 'null').replace('999999.99', '0.0')
  52.         print "Next try to push Document to ELK with ID: ", (counter)
  53.  
  54.         res = es.index(index="testpy", doc_type='testpy', body=chunk)
  55.         print "Result - Created : ", (res['created']), "OK\n"
  56.  
  57.         counter += 1
  58.     except:
  59.         counter += 1
  60.         print ("Something is wrong with ELK or Data in Table")
  61.  
  62. print ("Finish")
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement