Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- from django.conf import settings
- import requests
- from main.models import Application, ApplicationInstance, Platform, Category, Device, InputControl, Url
- from media_object.models import MediaObject
- from data_mining.util import prepare_file
- import logging
- from requests.exceptions import Timeout
- from django import db
- from memory_profiler import profile
- data_update_logger = logging.getLogger('data_update')
- @profile
- def handle_app_instance(app, app_instance_data):
- if 'url' not in app_instance_data:
- return
- app_instance = ApplicationInstance.objects.create(app=app)
- app_instance.file_size = app_instance_data.get('fileSize')
- data_update_logger.debug(u'Set app instance url')
- try:
- app_instance.url = requests.get(app_instance_data['url'], verify=False, timeout=settings.REQUEST_TIMEOUT_S).url
- except Timeout:
- return
- app_instance.source = app_instance_data['source']
- app_instance.platforms.add(*Platform.objects.filter(id__in=app_instance_data['platformIds']))
- app_instance.input_controls.add(*InputControl.objects.filter(id__in=app_instance_data['inputControlIds']))
- app_instance.price = app_instance_data['price']
- app_instance.save()
- app_instance.devices.add(*Device.objects.filter(slug__in=app_instance_data['deviceIdentifiers']))
- @profile
- def handle_app(app_data):
- data_update_logger.info(u'Application: {}'.format(app_data['name']))
- data_update_logger.debug(u'Creating application')
- app, created = Application.objects.get_or_create(id=app_data['id'])
- data_update_logger.debug(u'Set name')
- app.name = app_data['name']
- data_update_logger.debug(u'Set description')
- app.description = app_data['description']
- try:
- data_update_logger.debug(u'Set website')
- app.website = requests.get(app_data.get('website'), verify=False, timeout=settings.REQUEST_TIMEOUT_S).url
- except Exception:
- app.website = app_data.get('website')
- data_update_logger.debug(u'Set slug')
- app.slug = app_data['urlSlug']
- data_update_logger.debug(u'Set minimum requirements, setup instructions, playing instructions')
- app.minimum_requirements = app_data.get('minimumRequirements')
- app.setup_instructions = app_data.get('setupInstructions')
- app.playing_instruction = app_data.get('playingInstruction')
- data_update_logger.debug(u'Prepare thumbnail')
- thumbnail_unique_name, thumbnail_file = prepare_file(app_data['thumbnailUrl'])
- data_update_logger.debug(u'Delete current thumbnail')
- app.thumbnail.delete()
- data_update_logger.debug(u'Save thumbnail')
- app.thumbnail.save(thumbnail_unique_name, thumbnail_file)
- data_update_logger.debug(u'Delete all current instances')
- app.applicationinstance_set.all().delete()
- for app_instance_data in app_data['downloads']:
- data_update_logger.debug(u'Save thumbnail')
- handle_app_instance(app, app_instance_data)
- data_update_logger.debug(u'Set content warning')
- app.content_warning = app_data['contentWarning']
- data_update_logger.debug(u'Add categories')
- app.categories.add(*Category.objects.filter(id__in=map(lambda _: _['id'], app_data['genres'])))
- data_update_logger.debug(u'Delete all videos')
- app.videos.all().delete()
- for video_data in app_data['videos']:
- data_update_logger.debug(u'Add video')
- app.videos.add(Url.objects.create(value=video_data['embedUrl']))
- data_update_logger.debug(u'Remove all screenshots')
- for screenshot in app.screenshots.all():
- screenshot.delete()
- data_update_logger.debug(u'Save screenshots')
- for screenshot_data in app_data['screenshots']:
- data_update_logger.debug(u'Prepare screenshot')
- screenshot_unique_name, screenshot_file = prepare_file(screenshot_data['url'])
- data_update_logger.debug(u'Create media object')
- screenshot = MediaObject.objects.create()
- data_update_logger.debug(u'Save screenshot')
- screenshot.file.save(screenshot_unique_name, screenshot_file)
- data_update_logger.debug(u'Add screenshot')
- app.screenshots.add(screenshot)
- data_update_logger.debug(u'Save app')
- app.save()
- db.reset_queries()
- @profile
- def handle_search_page(search_data):
- for app_data in search_data:
- try:
- handle_app(app_data)
- except Exception:
- continue
- def app_pages(page=1):
- last_page_reached = False
- while not last_page_reached:
- try:
- params = {'page': page}
- data_update_logger.info('Page: {}'.format(page))
- search_page = requests.get(settings.WEARVR_API_URL_TEMPLATE.format('apps/search'),
- params=params,
- verify=False,
- timeout=settings.REQUEST_TIMEOUT_S).json()
- yield search_page
- if not search_page['apps']:
- last_page_reached = True
- except Exception:
- data_update_logger.exception('')
- finally:
- page += 1
- raise StopIteration
- @profile
- def update(page=1):
- for search_page in app_pages(page):
- try:
- handle_search_page(search_page['apps'])
- except Exception:
- data_update_logger.exception('')
- data_update_logger.info('Updated')
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement