Advertisement
Guest User

Untitled

a guest
Nov 15th, 2016
105
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 13.55 KB | None | 0 0
  1. # Copyright (C) 2016 iNuron NV
  2. #
  3. # This file is part of Open vStorage Open Source Edition (OSE),
  4. # as available from
  5. #
  6. # http://www.openvstorage.org and
  7. # http://www.openvstorage.com.
  8. #
  9. # This file is free software; you can redistribute it and/or modify it
  10. # under the terms of the GNU Affero General Public License v3 (GNU AGPLv3)
  11. # as published by the Free Software Foundation, in version 3 as it comes
  12. # in the LICENSE.txt file of the Open vStorage OSE distribution.
  13. #
  14. # Open vStorage is distributed in the hope that it will be useful,
  15. # but WITHOUT ANY WARRANTY of any kind.
  16.  
  17. """
  18. OVS migration module
  19. """
  20.  
  21. import hashlib
  22. import random
  23. import string
  24.  
  25.  
  26. class OVSMigrator(object):
  27. """
  28. Handles all model related migrations
  29. """
  30.  
  31. identifier = 'ovs'
  32. THIS_VERSION = 12
  33.  
  34. def __init__(self):
  35. """ Init method """
  36. pass
  37.  
  38. @staticmethod
  39. def migrate(previous_version):
  40. """
  41. Migrates from a given version to the current version. It uses 'previous_version' to be smart
  42. wherever possible, but the code should be able to migrate any version towards the expected version.
  43. When this is not possible, the code can set a minimum version and raise when it is not met.
  44. :param previous_version: The previous version from which to start the migration
  45. :type previous_version: float
  46. """
  47.  
  48. working_version = previous_version
  49.  
  50. if working_version == 0:
  51. # Initial version:
  52. # * Set the version to THIS RELEASE version
  53.  
  54. from ovs.dal.hybrids.user import User
  55. from ovs.dal.hybrids.group import Group
  56. from ovs.dal.hybrids.role import Role
  57. from ovs.dal.hybrids.client import Client
  58. from ovs.dal.hybrids.j_rolegroup import RoleGroup
  59. from ovs.dal.hybrids.j_roleclient import RoleClient
  60. from ovs.dal.hybrids.servicetype import ServiceType
  61. from ovs.dal.hybrids.branding import Branding
  62. from ovs.dal.lists.backendtypelist import BackendTypeList
  63.  
  64. # Create groups
  65. admin_group = Group()
  66. admin_group.name = 'administrators'
  67. admin_group.description = 'Administrators'
  68. admin_group.save()
  69. viewers_group = Group()
  70. viewers_group.name = 'viewers'
  71. viewers_group.description = 'Viewers'
  72. viewers_group.save()
  73.  
  74. # Create users
  75. admin = User()
  76. admin.username = 'admin'
  77. admin.password = hashlib.sha256('admin').hexdigest()
  78. admin.is_active = True
  79. admin.group = admin_group
  80. admin.save()
  81.  
  82. # Create internal OAuth 2 clients
  83. admin_pw_client = Client()
  84. admin_pw_client.ovs_type = 'INTERNAL'
  85. admin_pw_client.grant_type = 'PASSWORD'
  86. admin_pw_client.user = admin
  87. admin_pw_client.save()
  88. admin_cc_client = Client()
  89. admin_cc_client.ovs_type = 'INTERNAL'
  90. admin_cc_client.grant_type = 'CLIENT_CREDENTIALS'
  91. admin_cc_client.client_secret = ''.join(random.choice(string.ascii_letters +
  92. string.digits +
  93. '|_=+*#@!/-[]{}<>.?,\'";:~')
  94. for _ in range(128))
  95. admin_cc_client.user = admin
  96. admin_cc_client.save()
  97.  
  98. # Create roles
  99. read_role = Role()
  100. read_role.code = 'read'
  101. read_role.name = 'Read'
  102. read_role.description = 'Can read objects'
  103. read_role.save()
  104. write_role = Role()
  105. write_role.code = 'write'
  106. write_role.name = 'Write'
  107. write_role.description = 'Can write objects'
  108. write_role.save()
  109. manage_role = Role()
  110. manage_role.code = 'manage'
  111. manage_role.name = 'Manage'
  112. manage_role.description = 'Can manage the system'
  113. manage_role.save()
  114.  
  115. # Attach groups to roles
  116. mapping = [
  117. (admin_group, [read_role, write_role, manage_role]),
  118. (viewers_group, [read_role])
  119. ]
  120. for setting in mapping:
  121. for role in setting[1]:
  122. rolegroup = RoleGroup()
  123. rolegroup.group = setting[0]
  124. rolegroup.role = role
  125. rolegroup.save()
  126. for user in setting[0].users:
  127. for role in setting[1]:
  128. for client in user.clients:
  129. roleclient = RoleClient()
  130. roleclient.client = client
  131. roleclient.role = role
  132. roleclient.save()
  133.  
  134. # Add service types
  135. for service_type_info in [ServiceType.SERVICE_TYPES.MD_SERVER, ServiceType.SERVICE_TYPES.ALBA_PROXY, ServiceType.SERVICE_TYPES.ARAKOON]:
  136. service_type = ServiceType()
  137. service_type.name = service_type_info
  138. service_type.save()
  139.  
  140. # Branding
  141. branding = Branding()
  142. branding.name = 'Default'
  143. branding.description = 'Default bootstrap theme'
  144. branding.css = 'bootstrap-default.min.css'
  145. branding.productname = 'Open vStorage'
  146. branding.is_default = True
  147. branding.save()
  148. slate = Branding()
  149. slate.name = 'Slate'
  150. slate.description = 'Dark bootstrap theme'
  151. slate.css = 'bootstrap-slate.min.css'
  152. slate.productname = 'Open vStorage'
  153. slate.is_default = False
  154. slate.save()
  155.  
  156. # From here on, all actual migration should happen to get to the expected state for THIS RELEASE
  157. elif working_version < OVSMigrator.THIS_VERSION:
  158. # Migrate unique constraints
  159. from ovs.dal.helpers import HybridRunner, Descriptor
  160. from ovs.extensions.storage.persistentfactory import PersistentFactory
  161. client = PersistentFactory.get_client()
  162. hybrid_structure = HybridRunner.get_hybrids()
  163. for class_descriptor in hybrid_structure.values():
  164. cls = Descriptor().load(class_descriptor).get_object()
  165. classname = cls.__name__.lower()
  166. unique_key = 'ovs_unique_{0}_{{0}}_'.format(classname)
  167. uniques = []
  168. # noinspection PyProtectedMember
  169. for prop in cls._properties:
  170. if prop.unique is True and len([k for k in client.prefix(unique_key.format(prop.name))]) == 0:
  171. uniques.append(prop.name)
  172. if len(uniques) > 0:
  173. prefix = 'ovs_data_{0}_'.format(classname)
  174. for key in client.prefix(prefix):
  175. data = client.get(key)
  176. for property_name in uniques:
  177. ukey = '{0}{1}'.format(unique_key.format(property_name), hashlib.sha1(str(data[property_name])).hexdigest())
  178. client.set(ukey, key)
  179.  
  180. # Complete rework of the way we detect devices to assign roles or use as ASD
  181. # Allow loop-, raid-, nvme-, ??-devices and logical volumes as ASD (https://github.com/openvstorage/framework/issues/792)
  182. from ovs.dal.lists.storagerouterlist import StorageRouterList
  183. from ovs.extensions.generic.sshclient import SSHClient, UnableToConnectException
  184. from ovs.lib.disk import DiskController
  185.  
  186. for storagerouter in StorageRouterList.get_storagerouters():
  187. try:
  188. client = SSHClient(storagerouter, username='root')
  189. except UnableToConnectException:
  190. raise
  191.  
  192. # Retrieve all symlinks for all devices
  193. # Example of name_alias_mapping:
  194. # {'/dev/md0': ['/dev/disk/by-id/md-uuid-ad2de634:26d97253:5eda0a23:96986b76', '/dev/disk/by-id/md-name-OVS-1:0'],
  195. # '/dev/sda': ['/dev/disk/by-path/pci-0000:03:00.0-sas-0x5000c295fe2ff771-lun-0'],
  196. # '/dev/sda1': ['/dev/disk/by-uuid/e3e0bc62-4edc-4c6b-a6ce-1f39e8f27e41', '/dev/disk/by-path/pci-0000:03:00.0-sas-0x5000c295fe2ff771-lun-0-part1']}
  197. name_alias_mapping = {}
  198. alias_name_mapping = {}
  199. for path_type in client.dir_list(directory='/dev/disk'):
  200. if path_type in ['by-uuid', 'by-partuuid']: # UUIDs can change after creating a filesystem on a partition
  201. continue
  202. directory = '/dev/disk/{0}'.format(path_type)
  203. for symlink in client.dir_list(directory=directory):
  204. symlink_path = '{0}/{1}'.format(directory, symlink)
  205. link = client.file_read_link(symlink_path)
  206. if link not in name_alias_mapping:
  207. name_alias_mapping[link] = []
  208. name_alias_mapping[link].append(symlink_path)
  209. alias_name_mapping[symlink_path] = link
  210.  
  211. for disk in storagerouter.disks:
  212. if disk.aliases is None:
  213. # noinspection PyProtectedMember
  214. device_path = '/dev/{0}'.format(disk.name)
  215. disk.aliases = name_alias_mapping.get(device_path, [device_path])
  216. disk.save()
  217. for partition in disk.partitions:
  218. if partition.aliases is None:
  219. # noinspection PyProtectedMember
  220. partition_device = alias_name_mapping.get(partition._data.get('path'))
  221. if partition_device is None:
  222. partition.aliases = []
  223. partition.save()
  224. continue
  225. partition.aliases = name_alias_mapping.get(partition_device, [])
  226. partition.save()
  227.  
  228. DiskController.sync_with_reality(storagerouter_guid=storagerouter.guid)
  229.  
  230. # Only support ALBA backend type
  231. from ovs.dal.lists.backendtypelist import BackendTypeList
  232. for backend_type in BackendTypeList.get_backend_types():
  233. if backend_type.code != 'alba':
  234. backend_type.delete()
  235.  
  236. # Reformat the vpool.metadata information
  237. from ovs.dal.lists.vpoollist import VPoolList
  238. for vpool in VPoolList.get_vpools():
  239. new_metadata = {}
  240. for metadata_key, value in vpool.metadata.items():
  241. new_info = {}
  242. storagerouter_guids = [key for key in vpool.metadata.keys() if not key.startswith('backend')]
  243. if isinstance(value, dict):
  244. read_cache = value.get('backend_info', {}).get('fragment_cache_on_read', True)
  245. write_cache = value.get('backend_info', {}).get('fragment_cache_on_write', False)
  246. new_info['backend_info'] = {'alba_backend_guid': value.get('backend_guid'),
  247. 'backend_guid': None,
  248. 'frag_size': value.get('backend_info', {}).get('frag_size'),
  249. 'name': value.get('name'),
  250. 'policies': value.get('backend_info', {}).get('policies'),
  251. 'preset': value.get('preset'),
  252. 'sco_size': value.get('backend_info', {}).get('sco_size'),
  253. 'total_size': value.get('backend_info', {}).get('total_size')}
  254. new_info['arakoon_config'] = value.get('arakoon_config')
  255. new_info['connection_info'] = {'host': value.get('connection', {}).get('host', ''),
  256. 'port': value.get('connection', {}).get('port', ''),
  257. 'local': value.get('connection', {}).get('local', ''),
  258. 'client_id': value.get('connection', {}).get('client_id', ''),
  259. 'client_secret': value.get('connection', {}).get('client_secret', '')}
  260. if metadata_key == 'backend':
  261. new_info['caching_info'] = dict((sr_guid, {'fragment_cache_on_read': read_cache, 'fragment_cache_on_write': write_cache}) for sr_guid in storagerouter_guids)
  262. if metadata_key in storagerouter_guids:
  263. metadata_key = 'backend_aa_{0}'.format(metadata_key)
  264. new_metadata[metadata_key] = new_info
  265. vpool.metadata = new_metadata
  266. vpool.save()
  267.  
  268. # Removal of READ role
  269. from ovs.dal.lists.diskpartitionlist import DiskPartitionList
  270. for partition in DiskPartitionList.get_partitions():
  271. if 'READ' in partition.roles:
  272. partition.roles.remove('READ')
  273. partition.save()
  274.  
  275. return OVSMigrator.THIS_VERSION
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement