utils.py

import os
# import json, requests
from typing import List
import requests
# from urllib import request

from django.core.files.base import ContentFile
from django.conf import settings
# from django.core.files import File
# from django.core.files.temp import NamedTemporaryFile

from apps.realty.models import (
    RealtyObject,

    ObjectBeside,
    MetroStation,

    SingleRowAttribute,
    MultipleRowAttribute,
    Attribute,


    AttrValue,    SliderImage,
)
from .megabaz_api import (
    Client as MegabazClient,
    ItemsCollection as MegabazItemsCollection,
    ItemNormalizer as MegabazItemNormalizer,
    TaskType as MegabazTaskType,
    from_timestamp,
)
from .const import (
    STATUS
)

__all__ = (
    'realty_import_controller',
)


class ServiceImporterInterface:
    service_name = None

    class Meta:
        abstract = True

    def get_service_name(self):
        if not self.service_name:
            raise NotImplementedError('Property "service_name" have to be established')
        return self.service_name

    def parse(self):
        raise NotImplementedError('There is no implementation for method "parse"')

    def get_object_contact(self):
        raise NotImplementedError('There is no implementation for method "get_service_object_contact"')


class MegabazServiceImporter(ServiceImporterInterface):
    service_name = 'megabaz'
    api_key = getattr(settings, 'MEGABAZ_PASSWORD', '')
    parsed = False

    name_convert_list = {
        'kids': 'children',
        'pets': 'animals',

        'washer': 'washer',
        'tv': 'tv',
        'fridge': 'fridge',
        'kitchen_furniture': 'kitchen_furniture',
        'furniture': 'furniture',
    }

    _rooms = None

    def get_object_contact(self, contact_id):
        return None

    def parse(self):
        print('Start to parse...\n')

        self.client = MegabazClient(self.api_key)
        self.item_normalizer = MegabazItemNormalizer()

        while not self.parsed:
            print('Parsing page...')
            self.handle_data()
        print(f'Ended parse {self.service_name}')
        self.parsed = False

    def handle_data(self):
        data = self.client.receive_items()

        print(f'data: {data}')

        items = data['items']
        tasks = {
            task['item_id']: task['type'] for task in data['tasks']
        }


        if not len(items):
            self.parsed = True

        i=0
        for item in items:
            # if i > 10:
            #     break

            print('***************************')
            # print(items[item])
            # print('---------------------------')
            _data = self.item_normalizer.normalize(items[item])
            # _data['item_id'] = task['item_id']
            print('item:', _data)
            # print('task: ', tasks)
            _operation = tasks.get(_data['id'], MegabazTaskType.ADD)
            print('_operation: ', _operation)
            if _operation == MegabazTaskType.ADD:
                print('_operation: ADD')
                print('type: ', _data['type'])
                print('contract_type: ', _data['contract_type'])
                if (
                    _data['type'] == 'rent'
                    and _data['contract_type'] == 'long'
                ):
                    self.save(_data)

            if _operation == MegabazTaskType.UPDATE:
                print('_operation: UPDATE')
                print(f"data['id']: {data['id']}")
                self.update(_data)

            if _operation == MegabazTaskType.DELETE:
                print('_operation: DELETE')
                print(f"data['id']: {data['id']}")
                self.delete(_data['id'])

            print('***************************\n')
            i +=1

    def delete(self, obj_id):
        print(f'Deleting object {obj_id}...\n')

        _log = RealtyObject.objects.filter(
            alien_id=obj_id,
            alien_name=self.service_name
        ).delete()
        print('Deleted: ', _log, '\n')

    @staticmethod
    def str_to_int(data:str = None):
        try:
            return int(data)
        except ValueError:
            return None

    def update(self, data):
        update_data = {
            'status': self.get_status(data),
            'price': self.get_price(data),
        }

        def add_update(key, value):
            if value:
                update_data[key] = value

        address = f"{data['street']}, {data['street_number'] if data['street_number'] else ''}" if data['street'] else ''
        add_update('address', address)
        add_update('number_of_storeys', data['floors'])
        add_update('floor', int(data['floor']) if data['floor'] else None)
        add_update('area', float(data['square']) if data['square'] else None)
        add_update('location_lon', float(data['longitude']) if data['longitude'] else None)
        add_update('location_lat', float(data['latitude']) if data['latitude'] else None)
        add_update('description', data['description'])
        add_update('alien_creator_name', data['contact_name'])
        add_update('alien_creator_contact', data['phone'])

        RealtyObject.objects.filter(
            alien_id=int(data['id'])
        ).update(**update_data)

    def is_dublicate(self, data, title):
        if not (data['street'] and data['street_number']):
            return False
        if title:
            return RealtyObject.objects.filter(title=title).exists()
        return False

    def get_status(self, data, title=None):
        status = data['status']
        if self.is_dublicate(data, title):
            return STATUS.dublicate
        if status == 'checked':
            return STATUS.checked
        if status in [
            'not_checked',
            'not_called',
            'planned'
        ]:
            return STATUS.moderate
        return STATUS.archive

    def get_price(self, data):
        _price = data['price'] if data['price'] else None
        if not _price and data['price_from'] and data['price_to']:
            _price = (int(data['price_from']) + int(data['price_to'])) / 2
        return int(_price) if _price else None

    def save(self, data):
        print('Executing "save" method...\n')

        _title_parts = []

        print('object_type: ', data['object_type'])
        print('image: ', data['image'])
        if not (data['object_type'] and data['image'] and data['status'] in [
            'not_checked',
            'checked',
            'not_called',
            'planned'
        ]):
            return

        _obj_type = data['object_type'][0]
        rooms = self.str_to_int(_obj_type)
        print('rooms: ', _obj_type)
        if rooms:
            self._rooms = rooms
            _title_parts.append(f'Сдам {self._rooms} комнатную квартиру')
        else:
            return

        _address=f"{data['street']}, {data['street_number'] if data['street_number'] else ''}" if data['street'] else ''
        if _address:
            _title_parts.append(_address)

        title = ', '.join(_title_parts)

        _obj = RealtyObject.objects.create(
            status=self.get_status(data, title),
            title=title,
            description=data['description'],

            number_of_storeys=int(data['floors']) if data['floors'] else None,
            floor=int(data['floor']) if data['floor'] else None,
            area=float(data['square']) if data['square'] else None,
            price=self.get_price(data),

            address=_address,
            created=data['created_at'],

            location_lon=float(data['longitude']) if data['longitude'] else None,
            location_lat=float(data['latitude']) if data['latitude'] else None,

            alien_name=self.service_name,
            alien_id=int(data['id']),
            alien_creator_id=0,
            alien_creator_name=data['contact_name'],
            alien_creator_contact=data['phone'],
        )
        print(f'Created object ({data["id"]}): ', _obj)
        self.save_metro(data, _obj)
        self.save_single_attrs(data, _obj)
        self.save_multiple_attrs(data, _obj)
        self.save_images(data, _obj)

    @staticmethod
    def save_metro(data, obj):
        print('Start to save metro stations...\n')
        stations = data['station']
        print('metro stations to save: ', stations)
        for station in stations:
            metro = MetroStation.objects.filter(name=station['name']).first()
            if not metro:
                print(f'Metro with name "{station["name"]}" not found!')
                continue
            metro_obj = ObjectBeside.objects.create(
                metro=metro,
                model=obj,
                time=station['time']['walk']
            )
            print('Added metro object: ', metro_obj)

    def save_single_attrs(self, data, obj):
        print('Start to save single attrs...\n')

        if not self._rooms:
            return

        attr = Attribute.objects.filter(slug='rooms').first()
        print('try find rooms: ', self._rooms)
        attr_value = AttrValue.objects.filter(
            attribute=attr,
            name__contains=self._rooms
        ).first()
        if attr and attr_value:
            row_attr = SingleRowAttribute.objects.create(
                product=obj,
                attribute=attr,
                value=attr_value
            )
            print('Saved attr "rooms": ', row_attr)
        else:
            print('Attr "rooms" was NOT save: ', attr, attr_value)

    def save_multiple_attrs(self, data, obj):
        print('Start to save multiple attrs...')
        print('Options: ', data['options'])

        row_attr = None
        for key in self.name_convert_list:
            attr = Attribute.objects.filter(slug='features').first()
            attr_values = attr.attr_values.all()

            if self.name_convert_list[key] not in data['options']:
                continue

            attr_value = attr_values.filter(
                slug=key
            ).first()
            print('attr_value: ', attr_value)

            if not row_attr and attr_value:
                row_attr = MultipleRowAttribute.objects.create(
                    product=obj,
                    attribute=attr,
                )
                print('Created multiple attr: ', row_attr)

            if attr_value:
                row_attr.values.add(attr_value)
                print(f'Added attr_value: ', attr_value)

    @staticmethod
    def save_images(data, obj):
        print('Start to save obj images...')
        for img_url in data['image']:
            response = requests.get(img_url)
            if response.status_code == requests.codes.ok:
                _img = ContentFile(response.content)
                img_filename = os.path.basename(img_url)
                img_obj = SliderImage.objects.create(
                    model=obj
                )
                img_obj.image.save(img_filename, _img, save=True)
                print('Created image: ', img_obj)


class SmartAgentImporter(ServiceImporterInterface):
    service_name = 'smart_agent'
    login = getattr(settings, 'SMART_AGENT_LOGIN', '')
    password = getattr(settings, 'SMART_AGENT_PASSWORD', '')

    name_convert_list = {
        'for_whom': {
            'duo': [
                'for_couple'
            ],
            'one': [
                'for_1_woman',
                'for_1_man'
            ],
            'family': [
                'for_many',
                'for_couple'
            ],
            'students': [
                'for_many',
                'for_1_woman',
                'for_1_man',
                'for_2_woman',
                'for_2_man'
            ]
        },
        'features': {
            'children': [
                'children'
            ],
            'pets': [
                'pets'
            ],
            'fridge': [
                'fridge'
            ],
            'washer': [
                'washer'
            ],
            'balcony': [
                'balcony'
            ],
            'tv': [
                'tv'
            ],
        },
    }
    _rooms = None
    _rooms_attr = None

    def parse(self):
        self.delete()
        self.create()

    @staticmethod
    def check_response(response, method_name='not_set'):
        if not response["success"]:
            print(f'Method "{method_name}"  refused!')
            return False
        if not response['payload']["items"]:
            print(f'Item list empty for method "{method_name}"')
            return False
        return True

    @staticmethod
    def check_object_response(response, method_name='get_object'):
        if not response["success"]:
            print(f'Method "{method_name}"  refused!')
            return False
        if not response['payload']:
            print(f'Attrs list empty for method "{method_name}"')
            return False
        return True

    def get_object_contact(self, contact_id):
        print('Getting contact...\n')

        url = f'http://smartagent.ru/api/phone' \
              f'?id={contact_id}' \
              f'&auth[login]={self.login}' \
              f'&auth[password]={self.password}' \
              f'&property=1'
        response = requests.get(url).json()
        print('V get_object_contact response:', response)
        if not self.check_object_response(response, 'get_object_contact'):
            return {'errors': [item['desc'] for item in response['msg']['errors']]}
        # print('Deleted: ', _log, '\n')
        return response

    def delete(self):
        print('Deleting...\n')

        url = f'http://smartagent.ru/api/deleted' \
              f'?auth[login]={self.login}' \
              f'&auth[password]={self.password}' \
              f'&property=1&region=1'
        response = requests.get(url).json()
        print('V delete response:', response)
        if not self.check_response(response, 'delete'):
            return
        _ids = [item['id'] for item in response['payload']['items']]
        print('V delete _ids:', _ids)
        _log = RealtyObject.objects.filter(
            alien_id__in=_ids,
            alien_name=self.service_name
        ).delete()
        print('Deleted: ', _log, '\n')

    def create(self):
        print('Creating...\n')

        self._rooms_attr = Attribute.objects.filter(slug='rooms').first()

        _new = self.get_new()
        print('Actual: ', _new)
        if not _new:
            print('Nothing to create')
            return

        for i, item in enumerate(_new):
            print(f'======================== START {i} ========================')
            # if i > 10:
            #     break
            self.save(self.get_object(item['id']))
            print(f'======================== END {i} ========================\n')

    def get_new(self) -> List:
        print('Getting actual...\n')

        url = f'http://smartagent.ru/api/new' \
              f'?auth[login]={self.login}' \
              f'&auth[password]={self.password}' \
              f'&property=1&region=1'
        response = requests.get(url).json()
        print('V get_new response:', response)
        print('--- get_new ["success"]:', response["success"])
        if not self.check_response(response, 'get_new'):
            return
        items = response['payload']['items']
        return items

    def get_object(self, _id: int):
        print('Getting Object with id: ', _id, '\n')

        url = f'http://smartagent.ru/api/get' \
              f'?id={_id}' \
              f'&auth[login]={self.login}' \
              f'&auth[password]={self.password}' \
              f'&property=1&region=1'
        response = requests.get(url).json()
        print('V get_object response:', response)
        return response

    def save(self, _data):
        print('Executing "save" method...\n')

        if not self.check_object_response(_data, 'save'):
            return

        data = _data['payload']

        print('try find rooms: ', data['rooms'])
        self._rooms = AttrValue.objects.filter(
            attribute=self._rooms_attr,
            name__contains=data['rooms']
        ).first()
        if not self._rooms:
            print('Obj don`t have attr "rooms"! Skiped...')
            return

        if not data['images']:
            print('Obj don`t have images! Skiped...')
            return

        _obj = RealtyObject.objects.create(
            status=STATUS.moderate,
            title=data['note_generated'].split('.')[0],
            description=data['note_generated'],
            # owners_text=data['note_generated'],

            number_of_storeys=int(data['floors']),
            floor=int(data['floor']),
            area=float(data['total_area']),
            price=int(data['price']),

            address=f"{data['street_cache']}, {data['building']}'"
                    + f", корпус {data['corpus']}" if data.get('corpus') else '',
            location_lon=float(data['lon']),
            location_lat=float(data['lat']),

            alien_name=self.service_name,
            alien_id=int(data['id']),
            alien_creator_id=int(data['owner']),
            alien_creator_name=data['name'],
        )
        print(f'Created object ({data["id"]}): ', _obj)
        self.save_metro(data, _obj)
        self.save_single_attrs(data, _obj)
        self.save_multiple_attrs(data, _obj)
        self.save_images(data, _obj)

    @staticmethod
    def save_metro(data, obj):
        print('Start to save metro stations...\n')
        metro_names = data['metro_cache'].split(',')
        for metro_name in metro_names:
            metro = MetroStation.objects.filter(name=metro_name).first()
            if not metro:
                print(f'Metro with name "{metro_name}" not found!')
                continue
            metro_obj = ObjectBeside.objects.create(
                metro=metro,
                model=obj,
                time=data['distance']
            )
            print('Added metro object: ', metro_obj)

    def save_single_attrs(self, data, obj):
        print('Start to save single attrs...')
        row_attr = SingleRowAttribute.objects.create(
            product=obj,
            attribute=self._rooms_attr,
            value=self._rooms
        )
        print('Saved attr "rooms": ', row_attr)

    def save_multiple_attrs(self, data, obj):
        print('Start to save multiple attrs...')
        for key in self.name_convert_list:
            row_attr = None
            attr = Attribute.objects.filter(slug=key).first()
            attr_values = attr.attr_values.all()

            for attr_name in self.name_convert_list[key]:

                for alien_attr_name in self.name_convert_list[key][attr_name]:
                    # print(f'data[{alien_attr_name}] "{data[alien_attr_name"')
                    if data[alien_attr_name] != '1':
                        # print('continue...')
                        continue

                    attr_value = attr_values.filter(
                        slug=attr_name
                    ).first()
                    print('attr_value: ', attr_value)

                    if not row_attr and attr_value:
                        row_attr = MultipleRowAttribute.objects.create(
                            product=obj,
                            attribute=attr,
                        )
                        print('Created multiple attr: ', row_attr)

                    if attr_value:
                        row_attr.values.add(attr_value)
                        # row_attr.save()
                        print(f'Added attr_value: ', attr_value)

    @staticmethod
    def save_images(data, obj):
        print('Start to save obj images...')
        img_urls = data['images']
        for img_url in img_urls:
            response = requests.get(img_url)
            if response.status_code == requests.codes.ok:
                _img = ContentFile(response.content)
                img_filename = os.path.basename(img_url)
                img_obj = SliderImage.objects.create(
                    model=obj
                )
                img_obj.image.save(img_filename, _img, save=True)
                print('Created image: ', img_obj)


class RealtyImportController:
    service_classes = [
        MegabazServiceImporter,
        SmartAgentImporter,
    ]

    def __init__(self):
        self.services = {
            service_class.service_name: service_class()
            for service_class in self.service_classes
        }

    def _get_service(self, service_name: str):
        if service_name in self.services.keys():
            return self.services[service_name]
        # raise KeyError(f'Service with name: "{service_name}" not found')
        return None

    def parse_all(self):
        for service_name in self.services:
            print(f'Executing: {service_name} parser\n')
            self.services[service_name].parse()

    def parse(self, service_name: str):
        service = self._get_service(service_name)
        if service:
            service.parse()

    def get_service_object_contact(self, service_name: str, contact_id: int):
        service = self._get_service(service_name)
        if service:
            return service.get_object_contact(contact_id)
        return None


realty_import_controller = RealtyImportController()