Advertisement
Typhoon

ZVJS SQLITE to ELK 3

Apr 7th, 2015
262
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.87 KB | None | 0 0
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3.  
  4. import sqlite3
  5. import json
  6. import unicodedata
  7. from elasticsearch import Elasticsearch
  8.  
  9. conn = sqlite3.connect('SK_Prison_and_Court_Guard.sqlite')
  10. conn.row_factory = sqlite3.Row
  11. curs = conn.cursor()
  12. conn.commit()
  13.  
  14. es = Elasticsearch(host='elk.myserver.sk', port=9200, http_auth=('auth_user', 'auth_pass'),)
  15.  
  16. response = es.search(
  17.     index="testpy",
  18.     body={
  19.         "query": {
  20.             "bool": {
  21.                 "must": [{"match": {"_type": "testpy"}}]
  22.             }
  23.         },
  24.         "aggs": {
  25.             "max_price": {"max": {"field": "invoice_id"}}},
  26.         "sort": {"invoice_id": {"order": "desc"}},
  27.         "size": 1
  28.     }
  29. )
  30.  
  31. for hit in response['hits']['hits']:
  32.     last_elastic_id = (hit['_source']['invoice_id'])
  33.     print "Last Elasticsearch ID is: ",(last_elastic_id)
  34.  
  35. # Temp WorkAround if isn't nothing in index
  36. # If is empty use counter with number 1
  37. last_sqlite_id = 999999
  38. counter = last_elastic_id + 1
  39. try:
  40.     while counter < last_sqlite_id :
  41.        
  42.         conn = sqlite3.connect('SK_Prison_and_Court_Guard.sqlite')
  43.         conn.row_factory = sqlite3.Row
  44.         d = str(counter)
  45.         curs.execute("SELECT * FROM data WHERE invoice_id=" + d + "")
  46.         recs = curs.fetchall()
  47.         rows = [dict(rec) for rec in recs]
  48.         rows_json = json.dumps(rows)
  49.         chunk = unicodedata.normalize('NFKD', unicode(rows_json, 'utf-8', 'ignore')).encode('ASCII', 'ignore').replace('[','').replace(']', '').replace('"null"', 'null').replace('999999.99', '0.0')
  50.         print "Next try ID: ",(counter)
  51.      
  52.         es = Elasticsearch(host='elk.myserver.sk', port=9200, http_auth=('auth_user', 'auth_pass'),)
  53.         res = es.index(index="testpy", doc_type='testpy', body=chunk)
  54.         print(res['created']), "OK\n"
  55.      
  56.         counter = counter + 1
  57. except:
  58.     print "Finish"
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement