Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # -*- coding: utf-8 -*-
- from datetime import date, datetime
- from collections import defaultdict
- import re
- from json import dumps as json_dumps
- from dateutil.relativedelta import relativedelta
- from sqlalchemy import create_engine
- from sqlalchemy.sql import text
- from psycopg2.extras import execute_values
- ITEMS_VALUES = 'items'
- COMPLEX_NUM = 'complex_num'
- NODES_HEALTH = 'nodes_health'
- CONNECTORS_VALUES = 'connectors_values'
- TIMESTAMP = 'date_create'
- LINE = (r'^(?P<date_create>0\d{10})'
- '(?P<complex_num>\d{2})'
- '(?P<items>\d{6})'
- '(?P<nodes_health>\d{5})'
- '.{10}'
- '(?P<connectors_values>\d{11})$'
- )
- matcher = re.compile(LINE)
- def timestamp_to_date(timestamp):
- return datetime.fromtimestamp(
- int(timestamp)
- ).strftime('%Y-%m-%d')
- def get_3_months_back_in_days():
- """ returns 3s month back days count according today
- """
- today = date.today()
- back_then = today + relativedelta(months=-3)
- return (today - back_then).days
- def parse_line():
- """ parse income line
- позиции 1 - 11 - unix timestamp (первый символ ноль)
- позиции 12 - 13 - номер комплекса
- позиции 14 - 19 - состояние оборудования, каждая цифра отвечает за
- отдельное
- устройство (значения 0 - 9)
- позиции 20 - 24 - характеристики основных узлов комплекса (не является
- целевым
- устройством) (значения 0, 1)
- позиции 35 - 45 - состояние коннекторов (не является целевым устройством)
- """
- pass
- def unpack_values(data, key):
- d = defaultdict(set)
- for item_ind, item_val in enumerate(data[key]):
- d[item_ind].add(int(item_val))
- return d
- def parse(afile):
- """ main entrypoint to parse whole file
- """
- d = defaultdict(dict)
- timestamp = None
- with open(afile, 'r') as thefile:
- for line in thefile:
- found = matcher.match(line)
- if found:
- data = found.groupdict()
- complex_num = data[COMPLEX_NUM]
- if timestamp is None:
- timestamp = timestamp_to_date(data[TIMESTAMP])
- for k in (NODES_HEALTH, CONNECTORS_VALUES, ITEMS_VALUES):
- d[complex_num].update({k: unpack_values(data, k)})
- result = []
- for complex_num, data in d.items():
- data.update({COMPLEX_NUM: int(complex_num)})
- data.update({TIMESTAMP: timestamp})
- result.append(data)
- return result
- def dump_to_db(data):
- conn = psycopg2.connect(dbname="omd", host="localhost",
- user="user", password="password")
- print data
- with conn.cursor() as cur:
- for to_insert in data:
- cur.execute("""
- INSERT INTO complex_data
- (date_create, complex_num, items, nodes_health, connectors_values)
- VALUES (%(date_create)s, %(complex_num)s, %(items)s,
- %(nodes_health)s, %(connectors_values)s);
- """, to_insert,)
- def dump_to_db(data):
- engine = create_engine('postgresql://user:password@localhost/omd')
- con = engine.connect()
- SQL = text("""
- INSERT INTO complex_data
- (date_create, complex_num, items, nodes_health, connectors_values)
- VALUES :date_create, :complex_num, :items,
- :nodes_health, :connectors_values;
- """)
- con.execute(SQL, **data[0])
- dump_to_db(parse('test.txt'))
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement