scraper actual

"""
File contains Scraper class with corresponding methods
"""
from typing import Union, List, Optional
from logging import Logger, getLogger
from datetime import datetime, tzinfo
from time import mktime
import asyncio
from pytz import timezone
import validators
import feedparser
from aiohttp import ClientSession, ClientResponseError


class Scraper:
    """
    Class containing scraping methods
    """

    _logger: Logger
    _default_url: str
    _default_response: dict
    _timezone: tzinfo

    def __init__(self):
        self._logger = getLogger(__name__)
        self._default_url = "http://www.nu.nl/rss/Algemeen"
        self._default_response = {
            "title": "Not Found",
            "description": "Bad or malformed url",
            "last_entry_update": datetime.utcnow(),
            "items": [],
        }
        self._timezone = timezone('Europe/Amsterdam')
        self._logger.debug("Scraping tool initialized")

    async def _scrape(self, session, url) -> Union[str, None]:
        """Internal method running sessions to scrape the supplied URL"""
        try:  # To scrape :)
            async with session.get(url, raise_for_status=True) as response:
                return await response.text()

        except ClientResponseError:  # When there is no website available
            self._logger.exception("Could not reach: %s", url)

    async def _parse(
            self,
            url: Optional[str] = None,
            newer_than: Optional[datetime] = None
    ) -> Union[List[dict], None]:
        """Internal method calling _scrape() and parsign it's contents"""
        url = url or self._default_url
        if not validators.url(url):
            return self._default_response

        async with ClientSession() as session:
            self._logger.debug("Attempting to scrape: '%s' ...'", url)
            content: str = await self._scrape(session, url)

            # Parse data & create a formatted payload to return
            data = feedparser.parse(content)
            last_update = datetime.fromtimestamp(
                mktime(data.feed.updated_parsed)
            ).astimezone(self._timezone)

            payload: dict = {
                "title": data.feed.title,
                "description": data.feed.description,
                "last_updated": last_update,
                "items": [],
            }

            for entry in data.entries:  # Apeend the data we want
                published = datetime.fromtimestamp(
                    mktime(entry.published_parsed)
                ).astimezone(self._timezone)
                if newer_than:
                    statement = published >= newer_than
                    if statement is False:
                        continue
                try:

                    payload["items"].append(
                        {
                            "id": entry.id,
                            "title": entry.title,
                            "summary": entry.summary,
                            "authors": entry.authors,
                            "links": entry.links,
                            "published": published,
                        }
                    )
                    self._logger.debug("Added entry with id: %s", entry.id)

                except AttributeError:
                    self._logger.error("Could not parse feed entry")

            # Return payload
            self._logger.debug("Scraping session completed.")
            return payload

    def scrape(
            self,
            url: Optional[str] = None,
            newer_than: Optional[datetime] = None
    ) -> dict:
        """
        Method scrapes the supplied url for information.

        Argument(s):
            url:        (str)       (optional) the url to scrape from
                                    (default)  class url
            newer_than  (datetime)  (optional) minimum item publish time

        Returns a list of feed items

        Format: {
            'id': entry.id,
            'title': entry.title,
            'summary': entry.summary,
            'authors': entry.authors,
            'links': entry.links
        }
        """
        loop = asyncio.get_event_loop()
        return loop.run_until_complete(
            self._parse(url=url, newer_than=newer_than)
        )

    def get_default_url(self) -> str:
        """Returns the default url"""
        return self._default_url

    def get_default_response(self) -> dict:
        """Returns the default response"""
        return self._default_response

    def get_class_logger(self) -> Logger:
        """Returns the class logger"""
        return self._logger

    def get_time_zone(self) -> tzinfo:
        """Returns the class logger"""
        return self._timezone