Advertisement
Typhoon

ZVJS SQLITE to ELK

Mar 8th, 2015
263
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 0.95 KB | None | 0 0
  1. #!/usr/bin/env python2
  2. # -*- coding: utf-8 -*-
  3.  
  4. from datetime import datetime
  5. from elasticsearch import Elasticsearch
  6. import sqlite3
  7. import json
  8. import sys
  9. from bs4 import BeautifulSoup
  10. import chardet
  11. import string
  12. import unicodedata
  13. import hashlib
  14.  
  15. conn = sqlite3.connect('zvjs.sqlite')
  16. conn.row_factory = sqlite3.Row
  17. curs = conn.cursor()
  18. conn.commit()
  19.  
  20. counter = 1
  21. while counter<121488:
  22.  
  23.     d = str(counter)
  24.     curs.execute("SELECT * FROM data WHERE invoice_id="+d+"")
  25.     recs = curs.fetchall()
  26.     rows = [ dict(rec) for rec in recs ]
  27.     rows_json = json.dumps(rows)
  28.     chunk = unicodedata.normalize('NFKD', unicode(rows_json, 'utf-8','ignore')).encode('ASCII', 'ignore').replace('[', '').replace(']', '').replace('"null"', 'null').replace('999999.99', '0.0')
  29.     print (counter)
  30.  
  31.     es = Elasticsearch(host='elasticsearch.com', port=9200)
  32.     res = es.index(index="testpy", doc_type='testpy', body=chunk)
  33.     print(res['created'])
  34.  
  35.     counter = counter + 1
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement