Advertisement
Guest User

Untitled

a guest
Mar 24th, 2019
96
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.89 KB | None | 0 0
  1. # -*- coding: utf-8 -*-
  2. from __future__ import unicode_literals, print_function
  3.  
  4. import logging
  5. import sys
  6.  
  7. from udata.app import create_app, standalone
  8. from udata.commands import init_logging
  9. from udata.search import es, commands as cmd
  10.  
  11. import objgraph
  12.  
  13. LIMIT = 10000
  14. MAX_COMMON_TYPES = 15
  15. MAX_GROW = 5
  16. INDEX = 'udata-profiling'
  17.  
  18. app = standalone(create_app(init_logging=init_logging))
  19. log = logging.getLogger('__main__')
  20.  
  21.  
  22. def before():
  23. log.info('Object graph before')
  24. objgraph.show_most_common_types(limit=MAX_COMMON_TYPES)
  25. objgraph.show_growth(limit=MAX_GROW)
  26.  
  27.  
  28. def perform():
  29. log.info('Initiliazing index "{0}"'.format(INDEX))
  30.  
  31. if es.indices.exists(INDEX):
  32. es.indices.delete(INDEX)
  33.  
  34. es.initialize(INDEX)
  35.  
  36. with cmd.handle_error(INDEX):
  37. cmd.disable_refresh(INDEX)
  38. index_some_datasets()
  39. es.indices.delete(index=INDEX)
  40.  
  41.  
  42. def index_some_datasets():
  43. from udata.core.dataset.models import Dataset
  44. from udata.core.dataset.search import DatasetSearch
  45.  
  46. qs = Dataset.objects.visible()
  47. qs = qs.exclude(*DatasetSearch.exclude_fields)
  48.  
  49. qs = qs
  50.  
  51. docs = cmd.iter_qs(qs[:LIMIT], DatasetSearch)
  52. docs = cmd.iter_for_index(docs, INDEX)
  53.  
  54. for ok, info in cmd.streaming_bulk(es.client, docs, raise_on_error=False):
  55. print('.' if ok else 'x', end='')
  56. sys.stdout.flush()
  57. print('')
  58.  
  59.  
  60. def after(*types):
  61. log.info('Object graph after')
  62. objgraph.show_most_common_types(limit=MAX_COMMON_TYPES)
  63. objgraph.show_growth()
  64.  
  65. for typ in types:
  66. log.info('%ss %s', typ, objgraph.count(typ))
  67. log.info('%ss without leaking ones: %s', typ,
  68. objgraph.count(typ, objgraph.get_leaking_objects()))
  69. objects = objgraph.by_type(typ)
  70. leakings = objgraph.get_leaking_objects(objects)[:5]
  71. objgraph.show_backrefs(leakings, max_depth=4)
  72.  
  73.  
  74. before()
  75. with app.app_context():
  76. perform()
  77. after('Dataset')
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement