Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/env python
- # -*- coding: utf-8 -*-
- import sqlite3
- import json
- import unicodedata
- from elasticsearch import Elasticsearch
- conn = sqlite3.connect('my_database.sqlite')
- conn.row_factory = sqlite3.Row
- curs = conn.cursor()
- conn.commit()
- curs.execute("SELECT MAX(invoice_id) FROM data")
- recs = curs.fetchall()
- last_sqlite_id = recs
- es = Elasticsearch(host='elk.myhost.com', port=9200)
- response = es.search(
- index="testpy",
- body={
- "query": {
- "bool": {
- "must": [{"match": {"_type": "testpy"}}]
- }
- },
- "aggs": {
- "max_price": {"max": {"field": "invoice_id"}}},
- "sort": {"invoice_id": {"order": "desc"}},
- "size": 1
- }
- )
- for hit in response['hits']['hits']:
- last_elastic_id = (hit['_source']['invoice_id'])
- print (last_elastic_id)
- counter = last_elastic_id
- while counter < last_sqlite_id:
- d = str(counter)
- curs.execute("SELECT * FROM data WHERE invoice_id=" + d + "")
- recs = curs.fetchall()
- rows = [dict(rec) for rec in recs]
- rows_json = json.dumps(rows)
- chunk = unicodedata.normalize('NFKD', unicode(rows_json, 'utf-8', 'ignore')).encode('ASCII', 'ignore').replace('[','').replace(']', '').replace('"null"', 'null').replace('999999.99', '0.0')
- print (counter)
- es = Elasticsearch(host='elk.myhost.com', port=9200)
- res = es.index(index="testpy", doc_type='testpy', body=chunk)
- print(res['created'])
- counter = counter + 1
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement