Advertisement
One1L

Untitled

Feb 4th, 2017
154
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 5.19 KB | None | 0 0
  1. from django.conf import settings
  2. import requests
  3. from main.models import Application, ApplicationInstance, Platform, Category, Device, InputControl, Url
  4. from media_object.models import MediaObject
  5. from data_mining.util import prepare_file
  6. import logging
  7. from requests.exceptions import Timeout
  8. from django import db
  9. from memory_profiler import profile
  10.  
  11. data_update_logger = logging.getLogger('data_update')
  12.  
  13.  
  14. @profile
  15. def handle_app_instance(app, app_instance_data):
  16.     if 'url' not in app_instance_data:
  17.         return
  18.     app_instance = ApplicationInstance.objects.create(app=app)
  19.     app_instance.file_size = app_instance_data.get('fileSize')
  20.     data_update_logger.debug(u'Set app instance url')
  21.     try:
  22.         app_instance.url = requests.get(app_instance_data['url'], verify=False, timeout=settings.REQUEST_TIMEOUT_S).url
  23.     except Timeout:
  24.         return
  25.     app_instance.source = app_instance_data['source']
  26.     app_instance.platforms.add(*Platform.objects.filter(id__in=app_instance_data['platformIds']))
  27.     app_instance.input_controls.add(*InputControl.objects.filter(id__in=app_instance_data['inputControlIds']))
  28.     app_instance.price = app_instance_data['price']
  29.     app_instance.save()
  30.     app_instance.devices.add(*Device.objects.filter(slug__in=app_instance_data['deviceIdentifiers']))
  31.  
  32.  
  33. @profile
  34. def handle_app(app_data):
  35.     data_update_logger.info(u'Application: {}'.format(app_data['name']))
  36.  
  37.     data_update_logger.debug(u'Creating application')
  38.     app, created = Application.objects.get_or_create(id=app_data['id'])
  39.  
  40.     data_update_logger.debug(u'Set name')
  41.     app.name = app_data['name']
  42.  
  43.     data_update_logger.debug(u'Set description')
  44.     app.description = app_data['description']
  45.     try:
  46.         data_update_logger.debug(u'Set website')
  47.         app.website = requests.get(app_data.get('website'), verify=False, timeout=settings.REQUEST_TIMEOUT_S).url
  48.     except Exception:
  49.         app.website = app_data.get('website')
  50.  
  51.     data_update_logger.debug(u'Set slug')
  52.     app.slug = app_data['urlSlug']
  53.  
  54.     data_update_logger.debug(u'Set minimum requirements, setup instructions, playing instructions')
  55.     app.minimum_requirements = app_data.get('minimumRequirements')
  56.     app.setup_instructions = app_data.get('setupInstructions')
  57.     app.playing_instruction = app_data.get('playingInstruction')
  58.  
  59.     data_update_logger.debug(u'Prepare thumbnail')
  60.     thumbnail_unique_name, thumbnail_file = prepare_file(app_data['thumbnailUrl'])
  61.  
  62.     data_update_logger.debug(u'Delete current thumbnail')
  63.     app.thumbnail.delete()
  64.  
  65.     data_update_logger.debug(u'Save thumbnail')
  66.     app.thumbnail.save(thumbnail_unique_name, thumbnail_file)
  67.  
  68.     data_update_logger.debug(u'Delete all current instances')
  69.     app.applicationinstance_set.all().delete()
  70.     for app_instance_data in app_data['downloads']:
  71.         data_update_logger.debug(u'Save thumbnail')
  72.         handle_app_instance(app, app_instance_data)
  73.  
  74.     data_update_logger.debug(u'Set content warning')
  75.     app.content_warning = app_data['contentWarning']
  76.  
  77.     data_update_logger.debug(u'Add categories')
  78.     app.categories.add(*Category.objects.filter(id__in=map(lambda _: _['id'], app_data['genres'])))
  79.  
  80.     data_update_logger.debug(u'Delete all videos')
  81.     app.videos.all().delete()
  82.     for video_data in app_data['videos']:
  83.         data_update_logger.debug(u'Add video')
  84.         app.videos.add(Url.objects.create(value=video_data['embedUrl']))
  85.  
  86.     data_update_logger.debug(u'Remove all screenshots')
  87.     for screenshot in app.screenshots.all():
  88.         screenshot.delete()
  89.  
  90.     data_update_logger.debug(u'Save screenshots')
  91.     for screenshot_data in app_data['screenshots']:
  92.         data_update_logger.debug(u'Prepare screenshot')
  93.         screenshot_unique_name, screenshot_file = prepare_file(screenshot_data['url'])
  94.         data_update_logger.debug(u'Create media object')
  95.         screenshot = MediaObject.objects.create()
  96.         data_update_logger.debug(u'Save screenshot')
  97.         screenshot.file.save(screenshot_unique_name, screenshot_file)
  98.         data_update_logger.debug(u'Add screenshot')
  99.         app.screenshots.add(screenshot)
  100.     data_update_logger.debug(u'Save app')
  101.     app.save()
  102.     db.reset_queries()
  103.  
  104.  
  105. @profile
  106. def handle_search_page(search_data):
  107.     for app_data in search_data:
  108.         try:
  109.             handle_app(app_data)
  110.         except Exception:
  111.             continue
  112.  
  113.  
  114. @profile
  115. def update(page=1):
  116.     last_page_reached = False
  117.     while not last_page_reached:
  118.         try:
  119.             params = {'page': page}
  120.             data_update_logger.info('Page: {}'.format(page))
  121.             search_page = requests.get(settings.WEARVR_API_URL_TEMPLATE.format('apps/search'),
  122.                                        params=params,
  123.                                        verify=False,
  124.                                        timeout=settings.REQUEST_TIMEOUT_S).json()
  125.             if not search_page['apps']:
  126.                 last_page_reached = True
  127.                 continue
  128.             handle_search_page(search_page['apps'])
  129.         except Exception:
  130.             data_update_logger.exception('')
  131.         finally:
  132.             page += 1
  133.     data_update_logger.info('Updated'.format(page))
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement