alexpdr

Scraper

Jun 14th, 2020
1,469
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 4.38 KB | None | 0 0
  1. """
  2. File contains test for the scraper tool
  3. """
  4. from unittest import TestCase
  5. from datetime import datetime, timedelta, tzinfo
  6. from logging import Logger
  7. from pytz import timezone
  8. from rss_tool.scraper import Scraper
  9.  
  10.  
  11. class TestScraper(TestCase):
  12.     """
  13.    Class containing all tests for the scraper tool
  14.    """
  15.  
  16.     base = {
  17.         "title": "Bacon ipsum dolor amet capicola",
  18.         "description": "alcatra venison hamburger shoulder filet mignon",
  19.         "items": ["https://baconipsum.com/"],
  20.         "last_updated": datetime(2020, 6, 10, 14, 52, 0),
  21.     }
  22.  
  23.     tz: tzinfo = timezone('Europe/Amsterdam')
  24.  
  25.     def test_class_initialized(self):
  26.         """Testing to see if class inits as expected"""
  27.         scraper = Scraper()
  28.  
  29.         self.assertIsInstance(scraper, Scraper)
  30.         self.assertIsNotNone(scraper.get_class_logger())
  31.         self.assertIsNotNone(scraper.get_default_url())
  32.         self.assertIsNotNone(scraper.get_default_response())
  33.         self.assertIsNotNone(scraper.get_time_zone())
  34.  
  35.     def test_scrape_default(self):
  36.         """Testing to see if default scrape works as expected"""
  37.         scraper = Scraper()
  38.         data = scraper.scrape()
  39.  
  40.         self.assertIsNotNone(data)
  41.         self.assertGreater(len(data.keys()), 0)
  42.         self.assertEqual(self.base.keys(), data.keys())
  43.  
  44.     def test_scrape_bad_input(self):
  45.         """Testing to see if bad scrape works as expected"""
  46.         scraper = Scraper()
  47.  
  48.         data = scraper.scrape("httttttp://www.nu.nl/rss/Algemeen")
  49.  
  50.         self.assertEqual(data["items"], [])
  51.  
  52.     def test_scrape_good_input(self):
  53.         """Testing to see if good scrape works as expected"""
  54.         scraper = Scraper()
  55.         result_one = scraper.scrape(url="http://www.nu.nl/rss/Algemeen")
  56.         result_two = scraper.scrape(
  57.             url="https://feeds.feedburner.com/tweakers/mixed"
  58.         )
  59.  
  60.         self.assertIsNotNone(result_one)
  61.         self.assertGreater(len(result_one.keys()), 0)
  62.         self.assertEqual(self.base.keys(), result_one.keys())
  63.  
  64.         self.assertIsNotNone(result_two)
  65.         self.assertGreater(len(result_two.keys()), 0)
  66.         self.assertEqual(self.base.keys(), result_two.keys())
  67.  
  68.     def test_scrape_with_datetime(self):
  69.         """Testing to see if scraper datetime parsing works as expected"""
  70.         scraper = Scraper()
  71.         right_now = datetime.now().astimezone(self.tz)
  72.         result_past = scraper.scrape(
  73.             newer_than=right_now - timedelta(hours=6)
  74.         )
  75.         result_present = scraper.scrape(newer_than=right_now)
  76.         result_future = scraper.scrape(
  77.             newer_than=(right_now + timedelta(hours=6))
  78.         )
  79.  
  80.         # Past
  81.         self.assertGreater(result_past["items"].__len__(), 0)
  82.         self.assertGreater(len(result_past.keys()), 0)
  83.         self.assertEqual(self.base.keys(), result_past.keys())
  84.         # Present
  85.         self.assertGreaterEqual(result_present["items"].__len__(), 0)
  86.         self.assertGreater(len(result_present.keys()), 0)
  87.         self.assertEqual(self.base.keys(), result_present.keys())
  88.         # Future
  89.         self.assertEqual(result_future["items"].__len__(), 0)
  90.         self.assertGreater(len(result_future.keys()), 0)
  91.         self.assertEqual(self.base.keys(), result_future.keys())
  92.  
  93.     def test_internal_parse(self):
  94.         """Testing to see if internal parsing works as expected"""
  95.         scraper = Scraper()
  96.  
  97.         result_one = scraper.scrape()  # Default url used
  98.         result_two = scraper.scrape(
  99.             url="https://feeds.feedburner.com/tweakers/mixed"
  100.         )
  101.  
  102.         self.assertIsNotNone(result_one)
  103.         self.assertGreater(len(result_one.keys()), 0)
  104.  
  105.         self.assertIsNotNone(result_two)
  106.         self.assertGreater(len(result_two.keys()), 0)
  107.  
  108.     def test_getters(self):
  109.         """Testing to see if getters works as expected"""
  110.         scraper = Scraper()
  111.  
  112.         result_logger = scraper.get_class_logger()
  113.         result_default_url = scraper.get_default_url()
  114.         result_default_respose = scraper.get_default_response()
  115.         result_time_zone = scraper.get_time_zone()
  116.  
  117.         self.assertIsInstance(result_logger, Logger)
  118.         self.assertIsInstance(result_default_url, str)
  119.         self.assertIsInstance(result_default_respose, dict)
  120.         self.assertIsInstance(result_time_zone, tzinfo)
  121.         self.assertEqual(result_time_zone, self.tz)
Advertisement
Add Comment
Please, Sign In to add comment