Advertisement
Typhoon

ZVJS SQLITE to ELK 2

Mar 16th, 2015
295
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.50 KB | None | 0 0
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3.  
  4. import sqlite3
  5. import json
  6. import unicodedata
  7.  
  8. from elasticsearch import Elasticsearch
  9.  
  10.  
  11. conn = sqlite3.connect('my_database.sqlite')
  12. conn.row_factory = sqlite3.Row
  13. curs = conn.cursor()
  14. conn.commit()
  15.  
  16. curs.execute("SELECT MAX(invoice_id) FROM data")
  17. recs = curs.fetchall()
  18. last_sqlite_id = recs
  19.  
  20. es = Elasticsearch(host='elk.myhost.com', port=9200)
  21.  
  22. response = es.search(
  23.     index="testpy",
  24.     body={
  25.         "query": {
  26.             "bool": {
  27.                 "must": [{"match": {"_type": "testpy"}}]
  28.             }
  29.         },
  30.         "aggs": {
  31.             "max_price": {"max": {"field": "invoice_id"}}},
  32.         "sort": {"invoice_id": {"order": "desc"}},
  33.         "size": 1
  34.     }
  35. )
  36.  
  37. for hit in response['hits']['hits']:
  38.     last_elastic_id = (hit['_source']['invoice_id'])
  39.     print (last_elastic_id)
  40.  
  41. counter = last_elastic_id
  42. while counter < last_sqlite_id:
  43.     d = str(counter)
  44.     curs.execute("SELECT * FROM data WHERE invoice_id=" + d + "")
  45.     recs = curs.fetchall()
  46.     rows = [dict(rec) for rec in recs]
  47.     rows_json = json.dumps(rows)
  48.     chunk = unicodedata.normalize('NFKD', unicode(rows_json, 'utf-8', 'ignore')).encode('ASCII', 'ignore').replace('[','').replace(']', '').replace('"null"', 'null').replace('999999.99', '0.0')
  49.     print (counter)
  50.  
  51.     es = Elasticsearch(host='elk.myhost.com', port=9200)
  52.     res = es.index(index="testpy", doc_type='testpy', body=chunk)
  53.     print(res['created'])
  54.  
  55.     counter = counter + 1
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement