Advertisement
Guest User

Untitled

a guest
Jun 28th, 2016
89
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 15.90 KB | None | 0 0
  1. # -*- coding: utf-8 -*-
  2. import os
  3. import sys
  4. import json
  5. sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  6. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "bx.settings")
  7. import django
  8. django.setup()
  9. from dateutil import parser
  10. from decimal import Decimal, InvalidOperation
  11. from django.core.validators import validate_email
  12. from django import forms
  13. from django.conf import settings
  14. from bx.metadata.models import Metadata
  15. from bx.myauth.models import User
  16. from mod.extra import load_schema, validate_json
  17. from bx.org.models import (
  18. Org, Group, Employee, OrgGroup, OrgEmployee, EmployeeUser,
  19. GroupEmployee, Superiority
  20. )
  21.  
  22. DEBUG = False
  23.  
  24.  
  25. def delete_empty_dict_values(dict_obj):
  26. d = dict()
  27. for k, v in dict_obj.iteritems():
  28. if isinstance(v, dict):
  29. v = delete_empty_dict_values(v)
  30. if isinstance(v, list):
  31. v = delete_empty_list_values(v)
  32. if v:
  33. d[k] = v
  34. return d
  35.  
  36.  
  37. def delete_empty_list_values(list_obj):
  38. l = list()
  39. for v in list_obj:
  40. if isinstance(v, list):
  41. v = delete_empty_list_values(v)
  42. if isinstance(v, dict):
  43. v = delete_empty_dict_values(v)
  44. if v:
  45. l.append(v)
  46. return l
  47.  
  48.  
  49. def is_valid_email(email):
  50. try:
  51. validate_email(email)
  52. except forms.ValidationError:
  53. return False
  54.  
  55. return True
  56.  
  57.  
  58. class Singleton(type):
  59. _instances = {}
  60.  
  61. def __call__(cls, *args, **kwargs):
  62. if cls not in cls._instances:
  63. cls._instances[cls] = super(Singleton, cls).__call__(*args,
  64. **kwargs)
  65. return cls._instances[cls]
  66.  
  67.  
  68. class BaseManager(object):
  69. __metaclass__ = Singleton
  70.  
  71. def __init__(self, old_data=None, json_schema=None):
  72. """
  73. :param old_data: old data for model
  74. :param json_schema: json schema validator
  75. :return:
  76. """
  77. self.data = old_data
  78. self.schema = json_schema
  79.  
  80. def get_extra_by_schema(self, extra_data, schema_object):
  81. extra_fields = schema_object.keys()
  82. return {k: v for k, v in extra_data.iteritems() if k in extra_fields}
  83.  
  84. def get_extra_legacy(self, extra_data, legacy_model_name=None):
  85. """ Get from dict only elements which is in json schema
  86. :param extra_data: dict
  87. :param legacy_model_name: if model include some old models
  88. :return: dict
  89. """
  90. schema_legacy = self.schema['properties']['legacy']['properties']
  91. if legacy_model_name:
  92. schema_legacy = schema_legacy[legacy_model_name]['properties']
  93. return self.get_extra_by_schema(extra_data, schema_legacy)
  94.  
  95. def clean_and_validate_extra(self, extra):
  96. extra = delete_empty_dict_values(extra)
  97. validate_json(extra, self.schema)
  98. return extra
  99.  
  100. def get_or_create(self, **kwargs):
  101. raise NotImplementedError()
  102.  
  103.  
  104. class UserManager(BaseManager):
  105. def get_by_old_user_id(self, user_id):
  106. return self.get_or_create(**self.data[user_id])
  107.  
  108. def get_or_create(self, email, **kwargs):
  109. """ Get or create User from json
  110. :param email: email
  111. :param kwargs: params for extra field
  112. :return: User object
  113. """
  114. def get_email():
  115. # if email doesn't exist, generate it
  116. return email or '{0}@beneple.com'.format(kwargs['username'])
  117.  
  118. try:
  119. obj = User.objects.get(email=get_email())
  120. except User.DoesNotExist:
  121. obj = User(
  122. email=get_email(), password=kwargs['password'],
  123. clearance=self.convert_group_permission(kwargs['groups']),
  124. extra={'legacy': {
  125. 'date_joined': kwargs['date_joined'],
  126. 'first_name': kwargs['first_name'],
  127. 'last_name': kwargs['last_name'],
  128. 'username': kwargs['username'],
  129. }}
  130. )
  131. obj.save(metadata=Metadata.empty())
  132.  
  133. return obj
  134.  
  135. @staticmethod
  136. def convert_group_permission(group_ids):
  137. old_groups = []
  138. for group in group_ids:
  139. old_groups.append({
  140. 1: 'Admin',
  141. 4: 'CLevel',
  142. 6: 'Employee',
  143. 3: 'HR',
  144. 5: 'LineManager',
  145. 2: 'SuperHR',
  146. }.get(group))
  147. if 'HR' in old_groups or 'SuperHR' in old_groups:
  148. return 'edit'
  149. elif 'Admin' in old_groups:
  150. return 'audit'
  151. else:
  152. return ''
  153.  
  154.  
  155. class OrgManager(BaseManager):
  156. def get_or_create(self, title, url, **kwargs):
  157. """ Get or create Org from json.
  158. :param title: name filed
  159. :param url: domain field
  160. :param kwargs: other params
  161. :return: Org object
  162. """
  163. try:
  164. obj = Org.objects.get(name=title, domain=url)
  165. except Org.DoesNotExist:
  166. extra_legacy = self.get_extra_legacy(kwargs)
  167. obj = Org(name=title, domain=url, extra={'legacy': extra_legacy})
  168.  
  169. working_start_hour = kwargs.get('working_start_hour')
  170. if working_start_hour:
  171. obj.extra["working_hours_start"] = parser.parse(
  172. working_start_hour).time().isoformat()
  173. working_hours_end = kwargs.get('working_hours_end')
  174. if working_hours_end:
  175. obj.extra["working_hours_end"] = parser.parse(
  176. working_hours_end).time().isoformat()
  177.  
  178. validate_json(obj.extra, self.schema)
  179. obj.save(metadata=Metadata.empty())
  180.  
  181. return obj
  182.  
  183.  
  184. class EmployeeManager(BaseManager):
  185. def get_by_old_employee_id(self, employee_id):
  186. old_user_id = self.data[employee_id]['user']
  187. user = UserManager().get_by_old_user_id(old_user_id)
  188. return Employee.objects.get(employeeuser__user=user)
  189.  
  190. def get_user_by_old_employee_id(self, employee_id):
  191. old_user_id = self.data[employee_id]['user']
  192. return UserManager().get_by_old_user_id(old_user_id)
  193.  
  194. def get_or_create(self, user_obj, org_obj=None, **kwargs):
  195. """ Get or create Employee object.
  196. :param user_obj: User model instance
  197. :param org_obj: Org model instance
  198. :param kwargs: other params
  199. :return: Employee object
  200. """
  201. try:
  202. obj = Employee.objects.get(employeeuser__user=user_obj)
  203. except Employee.DoesNotExist:
  204. m = Metadata.empty()
  205. pers_email = kwargs['personal_email']
  206. gender = kwargs['gender']
  207. obj = Employee(
  208. displayname=u'{0} {1}'.format(
  209. user_obj.extra['legacy']['first_name'],
  210. user_obj.extra['legacy']['last_name']
  211. ).strip(),
  212. active=kwargs['status'],
  213. extra=delete_empty_dict_values({
  214. 'birth_date': kwargs['birth_date'],
  215. 'emergency_contact': {
  216. 'name': kwargs['ec_name'],
  217. 'phone_number': kwargs['ec_phone'],
  218. },
  219. 'gender': gender.lower() if gender else None,
  220. 'contact': {'home': {
  221. 'address': kwargs['home_address'],
  222. 'email': pers_email if is_valid_email(pers_email) else '',
  223. 'phone_number': kwargs['phone'],
  224. }},
  225. 'documents': {
  226. 'labor_card': {
  227. 'number': kwargs['labour_card'],
  228. 'expiry_date': kwargs['labour_expiry_date'],
  229. },
  230. 'passport': {
  231. 'number': kwargs['passport_number'],
  232. 'expiry_date': kwargs['passport_expiry_date'],
  233. },
  234. 'emirates_id': {
  235. 'number': kwargs['emirates_id'],
  236. 'expiry_date': kwargs['emirates_id_expiry_date'],
  237. },
  238. },
  239. 'languages': [
  240. lang.strip() for lang in kwargs['languages'].split(',')
  241. ] if kwargs['languages'] else [],
  242.  
  243. 'nationality': kwargs['nationality'],
  244.  
  245. 'legacy': {
  246. 'employee': self.get_extra_legacy(kwargs, 'employee')
  247. }
  248. }),
  249. )
  250. obj.extra['marital_status'] = {
  251. 'Married': 'married',
  252. 'Single': 'unmarried',
  253. }.get(kwargs['marital_status']) or "other"
  254.  
  255. obj.extra['religion'] = {
  256. 'Christianity': 'Christian',
  257. 'Islam': 'Muslim',
  258. }.get(kwargs['religion']) or 'Other'
  259.  
  260. validate_json(obj.extra, self.schema)
  261. obj.save(metadata=m)
  262.  
  263. EmployeeUser(employee=obj, user=user_obj).save(metadata=m)
  264. if org_obj:
  265. OrgEmployee(employee=obj, org=org_obj).save(metadata=m)
  266.  
  267. return obj
  268.  
  269. def complement_from_employment(self, employee_obj, **kwargs):
  270. """ Update Employee object, get data from employee.employment
  271. :param employee_obj: Employee model instance
  272. :param kwargs: data params
  273. :return:
  274. """
  275. obj = employee_obj
  276. try:
  277. salary = Decimal(kwargs['salary'].replace(',', '')) \
  278. if kwargs['salary'] else Decimal(0)
  279. if len(salary.as_tuple().digits) > 10:
  280. # if count of digits more 10 - don't save in db
  281. salary = Decimal(0)
  282. obj.monthlysalary = salary
  283. except InvalidOperation:
  284. pass
  285.  
  286. obj.extra['title'] = kwargs['position'] or '---'
  287. if 'documents' not in obj.extra:
  288. obj.extra['documents'] = {}
  289. obj.extra['documents'].update({
  290. 'visa': {
  291. 'number': kwargs['visa_number'],
  292. 'expiry_date': kwargs['visa_expiry_date'],
  293. 'image_url': kwargs['visa_document'],
  294. },
  295. 'work_contract': {
  296. 'start_date': kwargs['contract_start_date'],
  297. 'end_date': kwargs['contract_end_date'],
  298. },
  299. })
  300. extra_legacy = self.get_extra_legacy(kwargs, 'employment')
  301. obj.extra['legacy'].update({'employment': extra_legacy})
  302.  
  303. obj.extra = self.clean_and_validate_extra(obj.extra)
  304. obj.save(metadata=Metadata.empty())
  305.  
  306. return obj
  307.  
  308. def complement_form_bankinfo(self, employee_obj, **kwargs):
  309. obj = employee_obj
  310. bank_schema = self.schema['properties']['bank_account']['properties']
  311. obj.extra['bank_account'] = self.get_extra_by_schema(kwargs,
  312. bank_schema)
  313. obj.extra['legacy']['bank_account'] = self.get_extra_legacy(
  314. kwargs, 'bank_account'
  315. )
  316. obj.extra = self.clean_and_validate_extra(obj.extra)
  317. obj.save(metadata=Metadata.empty())
  318.  
  319. return obj
  320.  
  321. def complement_form_dependent(self, employee_obj, **kwargs):
  322. obj = employee_obj
  323. obj.extra.setdefault('dependants', [])
  324. dependant_names = [d['name'] for d in obj.extra['dependants']]
  325. if kwargs['name'] not in dependant_names:
  326. relation = kwargs['relation']
  327. relation = relation if relation and relation != 'None' else 'Other'
  328. obj.extra['dependants'].append({
  329. 'birth_date': kwargs['birth_date'],
  330. 'name': kwargs['name'],
  331. 'nationality': kwargs['nationality'],
  332. 'relation': relation,
  333. 'emirates_id': {
  334. 'number': kwargs['emirates_id'],
  335. 'expiry_date': kwargs['emirates_id_expiry_date'],
  336. },
  337. 'passport': {
  338. 'number': kwargs['passport_number'],
  339. 'expiry_date': kwargs['passport_expiry_date'],
  340. },
  341. 'visa': {
  342. 'number': kwargs['visa_number'],
  343. 'expiry_date': kwargs['visa_expiry_date'],
  344. 'image_url': kwargs['visa_document']
  345. },
  346. })
  347. obj.extra['legacy']['dependants'] = self.get_extra_legacy(
  348. kwargs, 'dependants'
  349. )
  350.  
  351. obj.extra = self.clean_and_validate_extra(obj.extra)
  352. obj.save(metadata=Metadata.empty())
  353.  
  354. return obj
  355.  
  356.  
  357. def load_old_data():
  358. with open(os.path.join(settings.BASE_DIR, 'old_db_data.json')) as f:
  359. json_data = json.load(f)
  360.  
  361. # loading old data
  362. data = {}
  363. for i in json_data:
  364. data.setdefault(i['model'], {})
  365. data[i['model']].update({i['pk']: i['fields']})
  366. print 'data for import loaded'
  367.  
  368. # load json schema
  369. org_schema = load_schema(__file__, 'org/org.schema.json')
  370. employee_schema = load_schema(__file__, 'org/employee.schema.json')
  371. del(employee_schema['required'])
  372.  
  373. # init converters
  374. user_manager = UserManager(data['auth.user'])
  375. org_manager = OrgManager(data['aythan.organization'], org_schema)
  376. employee_manager = EmployeeManager(data['employee.employee'],
  377. employee_schema)
  378.  
  379. # import auth.user
  380. for user_id, user_data in data['auth.user'].iteritems():
  381. user_manager.get_or_create(**user_data)
  382. if DEBUG:
  383. break
  384. print 'import auth.user is complete'
  385.  
  386. # import aythan.organization
  387. for org_id, org_data in data['aythan.organization'].iteritems():
  388. user_id = org_data.get('user')
  389. if user_id:
  390. # link to org user equivalent of admin rights in new systems
  391. user = user_manager.get_by_old_user_id(user_id)
  392. user.clearance = 'admin'
  393. user.save(metadata=Metadata.empty())
  394.  
  395. org_manager.get_or_create(**org_data)
  396.  
  397. if DEBUG:
  398. break
  399. print 'import aythan.organization is complete'
  400.  
  401. # load employee.employee
  402. for employee_id, employee_data in data['employee.employee'].iteritems():
  403. user = user_manager.get_by_old_user_id(employee_data['user'])
  404. org = None
  405. org_id = employee_data.get('organization')
  406. if org_id:
  407. org = org_manager.get_or_create(
  408. **data['aythan.organization'][org_id]
  409. )
  410.  
  411. employee_manager.get_or_create(user_obj=user, org_obj=org,
  412. **employee_data)
  413.  
  414. if DEBUG:
  415. break
  416. print 'import employee.employee is complete'
  417.  
  418. # load employee.employment
  419. for e_id, employment_data in data['employee.employment'].iteritems():
  420. employee = employee_manager.get_by_old_employee_id(
  421. employment_data['employee']
  422. )
  423. if employee_data.get('line_manager'):
  424. # create Superiority link for User
  425. line_manager_user = employee_manager.get_user_by_old_employee_id(
  426. employee_data['line_manager'])
  427. line_manager = employee_manager.get_or_create(
  428. user_obj=line_manager_user)
  429. Superiority(
  430. subordinate=employee, superior=line_manager
  431. ).save(metadata=Metadata.empty())
  432.  
  433. employee_manager.complement_from_employment(
  434. employee, **employment_data
  435. )
  436.  
  437. if DEBUG:
  438. break
  439. print 'import employee.employment is complete'
  440.  
  441. # load employee.bankinfo
  442. for eb_id, eb_data in data['employee.bankinfo'].iteritems():
  443. employee = employee_manager.get_by_old_employee_id(eb_data['employee'])
  444. employee_manager.complement_form_bankinfo(employee, **eb_data)
  445.  
  446. if DEBUG:
  447. break
  448. print 'import employee.bankinfo is complete'
  449.  
  450. # load employee.dependent
  451. for ed_id, ed_data in data['employee.dependent'].iteritems():
  452. employee = employee_manager.get_by_old_employee_id(ed_data['employee'])
  453. employee_manager.complement_form_dependent(employee, **ed_data)
  454.  
  455. if DEBUG:
  456. break
  457. print 'import employee.dependent is complete'
  458.  
  459.  
  460. if __name__ == "__main__":
  461. load_old_data()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement