tumblr pic/vid scraper

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Github.com/Skarlett
__version__ = '1.6.12'
__author__ = 'Skarlett'
##################
# Tumblr APIv2 Basic Util Scraper
import pytumblr as api
import json
import os
import time
import requests
import hashlib
import pwd
import configparser
import multiprocessing # moar power

pprint = lambda obj: print(json.dumps(obj, separators=[',', ':'], indent=4, sort_keys=True))
getusername = lambda : pwd.getpwuid(os.getuid())[0]
DEBUGGING = False

##############
#  Exceptions
#####
class TooManyRequestsWarning(Warning):
  pass


class NoKeysInDatabase(ValueError):
  pass


class HourLimitCap(TooManyRequestsWarning):
  pass


class DayLimitCap(TooManyRequestsWarning):
  pass


class CONFIG_FILE:
    ''' Lets load up some configuration variables '''
    DIR = "/home/{}/.config/tumblr_scraper/".format(getusername())
    FP = os.path.join(DIR, "tumblr_scraper.conf")
    _CONFIG = configparser.ConfigParser()

    __base = '/home/{}/.config'.format(getusername())
    for d in [__base, __base+'/tumblr_scraper']:
      if not os.path.isdir(d):
        os.mkdir(d)
    del __base

    if not os.path.isfile(FP):
        _CONFIG['APP'] = {
            'jsondb': os.path.join(DIR, 'keys.json'),
            'reset': 24 * 60 * 60,
            'total_limit': 5000,
            'hour_limit': 1000,
            'save_every_x_requests': 50,
            'download_chunk_size': 4098,
            'download_threads': 4,
            'pass_tags_to_crawl_tags_if_crawl_tags_empty': True,
            'debug': False,
            'blacklist_fp': os.path.join(DIR, 'blacklist.lst')
        }
        _CONFIG['API'] = {
            'maxpost': 20,
            'default_img_cnt': 200,
            'default_vid_cnt': 40,
            'default_min_media': 20
        }
        with open(FP, 'w') as cf:
            _CONFIG.write(cf)

        print('configuration file wrote to {} - exitting, please restart.'.format(FP))
        exit(0)
    else:
      DB = _CONFIG.get('app', 'jsondb', fallback=os.path.join(DIR, 'keys.json'))
      BLACKLST = _CONFIG.get('app', 'blacklist_fp', fallback=os.path.join(DIR, 'blacklist.lst'))
      RESET = _CONFIG.get('app', 'reset', fallback=24*60*60)
      TLIMIT = _CONFIG.get('app', 'total_limit', fallback=5000)
      HLIMIT = _CONFIG.get('app', 'hour_limit', fallback=1000)
      DL_SIZE = _CONFIG.get('app', 'download_chunk_size', fallback=1028)
      DL_THREADS = _CONFIG.get('app', 'download_threads', fallback=2)
      SAVECNT = _CONFIG.get('app', 'save_every_x_requests', fallback=200)
      PASS_TAGS = _CONFIG.get('app','pass_tags_to_crawl_tags_if_crawl_tags_empty', fallback=True)
      DEBUG = _CONFIG.get('app','debug', fallback=False)

      POST_SIZE = _CONFIG.get('api', 'maxpost', fallback=20)
      IMG_CNT = _CONFIG.get('api', 'default_img_cnt', fallback=200)
      VID_CNT = _CONFIG.get('api', 'default_vid_cnt', fallback=40)
      MIN_CNT = _CONFIG.get('api', 'default_min_media', fallback=20)

      if DEBUG:
        global DEBUGGING
        DEBUGGING = True


class BlackList:
  ''' This is hackish - and i should probably find another way of implementing this, but fuck it'''
  _FP = CONFIG_FILE.BLACKLST
  ignore = set()

  if os.path.isfile(_FP):
    with open(_FP) as fd:
      for l in fd:
        data = l.strip()
        if data:
          ignore.add(data)
  else:
    with open(_FP, 'w') as fd:
      fd.write('')

  @classmethod
  def save(cls):
    with open(CONFIG_FILE.BLACKLST, 'w') as fd:
      fd.write('\n'.join(cls.ignore))

  @classmethod
  def add(cls, user):
    cls.ignore.add(user)

  @classmethod
  def update(cls, iterable):
    cls.ignore.update(iterable)

  @classmethod
  def remove(cls, user):
    cls.ignore.remove(user)\


class RotatedTumblrRestClient(api.TumblrRestClient):
    def __init__(self, db, consumer_key):
      self.db = db
      self.dlimit = CONFIG_FILE.TLIMIT
      self.reset = CONFIG_FILE.RESET
      self.hlimit = CONFIG_FILE.HLIMIT
      self.save_limit = CONFIG_FILE.SAVECNT

      self.consumer_key = consumer_key
      self.save_counter = 0
      api.TumblrRestClient.__init__(self, *self.db._data[self.consumer_key]['keys'])

    @property
    def dcnt(self):
      return self.db._data[self.consumer_key]['dcnt']

    @property
    def dts(self):
      return self.db._data[self.consumer_key]['dts']

    @property
    def hcnt(self):
      return self.db._data[self.consumer_key]['hcnt']

    @property
    def hts(self):
      return self.db._data[self.consumer_key]['hts']

    def is_ready(self):
      return self.hlimit > self.hcnt and self.dlimit > self.dcnt+self.hcnt

    def kill(self):
      self.db._data[self.consumer_key]['hcnt'] = self.hlimit
      self.db._data[self.consumer_key]['dcnt'] = self.dlimit

    def send_api_request(self, method, url, params={}, valid_parameters=[], needs_api_key=False):
        if self.save_counter >= self.save_limit:
          self.save_counter = 0
          self.db.save()
        else:
          self.save_counter += 1

        if self.hts+60*60 >= time.time():
          self.db._data[self.consumer_key]['dcnt'] += self.db._data[self.consumer_key]['hcnt']
          self.db._data[self.consumer_key]['hts'] = time.time()
          self.db._data[self.consumer_key]['hcnt'] = 0

        if self.dts+self.reset >= time.time():
          self.db._data[self.consumer_key]['dcnt'] = 0
          self.db._data[self.consumer_key]['dts'] = time.time()

        if not self.dcnt+self.hcnt > self.dlimit or not self.db._raise:
          if not self.hcnt > self.hlimit or not self.db._raise:
            if DEBUGGING:
              print(method, url, params, needs_api_key)

            resp = api.TumblrRestClient.send_api_request(
              self, method, url,
              params, valid_parameters, needs_api_key
            )

            if resp:
              self.db._data[self.consumer_key]['hcnt'] += 1
              return resp

          else:
            if self.db._raise:
              raise HourLimitCap('Reached hour limit')
        else:
          if self.db._raise:
            raise DayLimitCap('Reached day limit')


class JSONAPIrotator:
    '''
    Saves all API keys in a json loadable format to
    create multiple instances of clients with different api keys

    '''

    FP = CONFIG_FILE.DB

    def __init__(self, client_class=RotatedTumblrRestClient):
        self.clients = set()
        self.client_cls = client_class
        self._raise = True

        if not os.path.isfile(self.FP):
          with open(self.FP, 'w') as fd:
              json.dump({}, fd)
        else:
            with open(self.FP) as fd:
              self._data = json.load(fd)

        self.init_clients()

    def init_client(self, consumer_key):
      if not consumer_key in self._data:
        raise ValueError('Consumer Key not in database!')
      client = self.client_cls(self, consumer_key)
      self.clients.add(client)
      return client

    def init_clients(self):
      for key in self._data.keys():
        self.init_client(key)

    def save(self):
        with open(self.FP, 'w') as fd:
          json.dump(self._data, fd)

    def requests_used(self):
        t=0
        for _, v in self._data.items():
            t += v['dcnt'] + v['hcnt']
        return t

    def requests_left_today(self):
        t = 0
        for _, v in self._data.items():
            t += CONFIG_FILE.TLIMIT-v['dcnt']-v['hcnt']
        return t

    def requests_left_inhour(self):
        t = 0
        for _, v in self._data.items():
            t += CONFIG_FILE.HLIMIT-v['hcnt']
        return t

    def add_key(self, consumer_key, consumer_secret, oauth_token, oauth_secret):
        self._data[consumer_key] = {
            'dts': time.time(),
            'hts': time.time(),
            'dcnt': 0,
            'hcnt': 0,
            'keys': [consumer_key, consumer_secret, oauth_token, oauth_secret]
        }
        return "Updated key"

    def remove_key(self, consumer_key):
        self._data.pop(consumer_key)
        return "Removed key"

    def no_raise(self):
      self._raise = False
      return "Api limits turned off...", False

    def list_keys(self):
        data = '\n\t\t'.join(k for k in self._data.keys())
        return '''
        Listing Keys...
                {}

        Total Left per 24h: {}
        Total Left per 1h: {}
        '''.format(data, self.requests_left_today(), self.requests_left_inhour()), True

    def feed_client(self):
      for x in self.clients:
        if x.is_ready():
          return x
      self.save()

      if self._raise:
        raise NoKeysInDatabase('No keys found in database to return')
      else:
        return list(self.clients)[0]

class RestlessCrawler:
    #######
    # Manages the amount of pictures received from one place
    DEFAULT_POST_CNT = CONFIG_FILE.POST_SIZE
    DEFAULT_IMAGE_COUNT = CONFIG_FILE.IMG_CNT
    DEFAULT_VIDEO_COUNT = CONFIG_FILE.VID_CNT
    DEFAULT_TAGS = [""]
    DEFAULT_CRAWL_TAGS = [""]
    DEFAULT_MIN_MEDIA = CONFIG_FILE.MIN_CNT
    DEFAULT_IMAGE_OFFSET = 0
    DEFAULT_VIDEO_OFFSET = 0
    # Lots of samples yas
    ########

    def __init__(self, user, db, tags=None, crawl_tags=None,
                 max_imgs=None, max_vids=None, minimum_media=None,
                 ioffset=None, voffset=None, ignore=False, adult=False, nsfw=False):

        self.db = db
        self.client = db.feed_client()
        self.name = user
        self.shoutouts = set()
        self.media = set()
        self.ignore = ignore
        self.max_img = max_imgs or self.DEFAULT_IMAGE_COUNT
        self.max_vid = max_vids or self.DEFAULT_VIDEO_COUNT
        self.tags = tags or self.DEFAULT_TAGS
        self.crawl_tags = crawl_tags or self.DEFAULT_CRAWL_TAGS
        self.minimum_media = minimum_media or self.DEFAULT_MIN_MEDIA
        self.ioffset = ioffset or self.DEFAULT_IMAGE_OFFSET
        self.voffset = voffset or self.DEFAULT_VIDEO_OFFSET
        self.adult = adult
        self.nsfw = nsfw

    def __eq__(self, other):
        return isinstance(other, self.__class__) and self.name == other.name

    def __hash__(self):
        return hash(self.name)

    def _rotate_api(self):
      self.client = self.db.feed_client()

    def _rotate_posts(self, limit=None, start_at=0, push_limit=True, **kwargs):
        roll_over = False
        i = start_at
        if limit and start_at and push_limit:
          limit += start_at

        while limit and limit >= i:
            jpkg = self.client.posts(self.name, offset=i, **kwargs)
            pprint(jpkg)
            if jpkg:
                if 'errors' in jpkg and 'meta' in jpkg:
                  print(jpkg['meta']['msg'])
                  if jpkg['meta']['msg'].lower() == 'limit exceeded':
                    self.client._data['dcnt'] = self.client.dcnt-self.client.hcnt
                    self.client._data['hcnt'] = 0
                    self._rotate_api()
                    roll_over = True
                    break

                else:
                  _use = False
                  if self.adult or self.nsfw:
                    for _filter in ['is_adult', 'is_nsfw']:
                      if self.adult and _filter in jpkg['blog'] and jpkg['blog'][_filter]:
                        _use = True
                  else:
                    _use = True

                  if _use:
                    i += self.DEFAULT_POST_CNT
                    if 'posts' in jpkg:
                      if jpkg['posts']:
                        for post in jpkg['posts']:
                          yield jpkg, post
                      else:
                        break
                    else:
                      break
                  else:
                    continue
            else:
              break

        if roll_over:
          for x in self._rotate_posts(limit, start_at=i, push_limit=False, **kwargs):
            yield x

    def scrape(self, **kwargs):
      for deprecated in ('type', 'tag', 'offset'):
        if deprecated in kwargs:
          kwargs.pop(deprecated)

      for tag in self.tags:
        for catagory, limit in (('photo', self.max_img), ('video', self.max_vid)):
          if limit:
            for jpkg, post in self._rotate_posts(limit=limit, type=catagory,tag=tag, **kwargs):
              if DEBUGGING:
                pprint(jpkg)

              if 'source_url' in post:
                user = post['source_url'].split('//')[1].split('.')[0]
                if user != self.name and not user in BlackList.ignore:
                  self.shoutouts.add(self.__class__(
                      user, self.db, self.crawl_tags, self.crawl_tags,
                      self.max_img, self.max_vid, minimum_media=self.minimum_media, adult=self.adult, nsfw=self.nsfw
                  ))
                else:
                  self.scrape_post(post)
              else:
                self.scrape_post(post)

    def scrape_post(self, post):
        if post['type'] == 'video' and 'video_url' in post:
          self.media.add(post['video_url'])

        if post['type'] == 'photo':
          self.media.update(pic['original_size']['url'] for pic in post['photos'])

def download_media(pkg, chunksize=CONFIG_FILE.DL_SIZE):
    url, to = pkg
    if url:
      resp = requests.get(url)
      generator = resp.iter_content(chunksize)
      first_chunk = next(generator)
      file_hash = hashlib.md5(first_chunk).hexdigest()
      file_extension = url.split('.')[-1].lower()
      fp = os.path.join(to, '{}.{}'.format(file_hash, file_extension))

      if not os.path.isfile(fp):
        with open(fp, 'wb') as f:
          f.write(first_chunk)
          for chunk in generator:
            f.write(chunk)


def main(user, user_limit=0):
      assert isinstance(user, RestlessCrawler)

      root_user = user
      users = {root_user}
      directory = os.getcwd()
      remember = set()

      with multiprocessing.Pool(processes=CONFIG_FILE.DL_THREADS) as pool:
        closed = False
        try:
          while users:
            user = users.pop()
            remember.add(user)

            download_to = os.path.join(directory, user.name)
            user.scrape()

            for u in remember:
              if u in user.shoutouts:
                user.shoutouts.remove(u)
            users.update(user.shoutouts)

            if user.media and not user.ignore and len(user.media) >= user.minimum_media:
              if not os.path.isdir(download_to):
                os.mkdir(download_to)

            print('Scraping {} <{}>'.format(user.name, len(user.media)))
            pool.map(download_media, ((url, download_to) for url in user.media))

            if user_limit and len(remember) >= user_limit:
               break

        except KeyboardInterrupt:
          if not closed:
            pool.close()

        if not closed:
          pool.close()

        pool.join()

if __name__ == '__main__':
  import sys

  class CLI:
    ''' this thing is stiff as hell '''


    HELP = """
     _______
    ( Boobs )
     -------
           o   ^__^
            o  (xx)\_______
               (__)\       )\/\\
                U  ||----w |
                   ||     ||

    tumblr scraper tool for APIv2 - crawls profiles pulling pictures and videos while scanning for more users
    usage: {fp} <args> <blog_name>

    It should be noted this application is not retard-proof
    it probably wont work if you suffer from retardism,
    skiddism, aufucktism or all the above. P.S, your computer will blow up
    if you're a jew and use this application.

    DESC
    CMD <ARGUMENTS> | DEFAULT

      Use "None" if you'd like to query without comparing tags
      -t | --tags <tag1> <tag2> <tag3> | {tags_default}

      When a name gets found, use these tags
      -ct | --crawltags <tag1> <tag2> <tag3> | {tags_default}

      Limits the amount of users this application may crawl. 0 = Infinite.
      -ul | --user-limit <int> | 0

      Maximum Retrieval of media
      -i | --img <int> | {img_default}
      -v | --vid <int> | {vid_default}

      Forces it not to download ROOT BLOG CONTENTS but instead
      iterates through them for crawl tags
      -x | --ignore | False

      Add API keyset for the application to use.
      --db-add <consumer-key> <consumer-secret> <oauth_token> <oauth_secret>

      Remove an API keyset from the database
      --db-del <consumer-key>

      List database keys
      --keys

      Only captures blogs marked with ...
      -a | --adult | False
      -n | --nsfw | False

      Fine Tuning settings
      --min-media <int> | {min_data} =>
      --offset-images <int> | 0
      --offset-videos <int> | 0
      -il | --ignore-api-limitations | False

      Extra output
      --debug

      spawns help message.
      -h | --help

    """.format(
        fp=__file__ if __file__.count('/') < 2 else __file__.split('/')[-1],
        tags_default=RestlessCrawler.DEFAULT_TAGS,
        img_default=RestlessCrawler.DEFAULT_IMAGE_COUNT,
        vid_default=RestlessCrawler.DEFAULT_VIDEO_COUNT,
        min_data=RestlessCrawler.DEFAULT_MIN_MEDIA,
    )

    if len(sys.argv) == 1 or \
      '-h' in sys.argv or \
      '--help' in sys.argv:

      print(HELP)
      exit(2)

    GLOBAL_CMDS = (
        (('-ul', '--user-limit'), 'user_limit'),
    )

    DB_CMDS = (
        (('--db-add',), 'add_key'), (('--db-del',), 'remove_key'),
    )

    DB_FLAGS = (
        (('--keys',), 'list_keys'), (('--ignore-api-limitations', '-il'), 'no_raise')
    )

    CRAWL_CMDS = (
        (('-t', '--tags'), 'tags'), (('-ct', '--crawltags'), 'crawl_tags'),
        (('-i', '--img'), 'max_imgs'), (('-v', '--vid'), 'max_vids'),
        (('--min-media',), 'minimum_media'), (('--offset-images',), 'ioffset'),
        (('--offset-videos',), 'voffset')
    )
    CRAWL_FLAGS = (
        (('-x', '--ignore'), 'ignore'), (('-a', '--adult'), 'adult'), (('-ns', '--nsfw'), 'nsfw')
    )

    @classmethod
    def find_value(cls, key_tuple, kill_on_last_arg=True):
        ret = []
        for key in key_tuple:
          if key in sys.argv:
            collect = False
            for i, part in enumerate(sys.argv):
              if part == key:
                  collect = True
              else:
                  if part.startswith('-') or (kill_on_last_arg and i+1 == len(sys.argv)):
                      collect = False

                  elif collect:
                      if key in ('-ct', '-t', '--crawltags', '--tags'):
                          if part.lower() == "none":
                              part = ""
                          else:
                            part = part.replace('_', ' ')

                      ret.append(part)

            ret = [x for x in ret if x and not x == 0]

            if not key in ('-ct', '-t', '--crawltags', '--tags'):
              if len(ret) > 1:
                return ret

              elif len(ret) == 1:
                data = ret[0]
                if data.isdigit():
                    return int(data)
                return data

              elif len(ret) == 0:
                return None
            else:
              return ret

    @classmethod
    def interface(cls):
      global DEBUGGING
      # for keys, reference in cls.GLOBAL_CMDS:
      #   resp = cls.find_value(keys)
      db = JSONAPIrotator()

      g = {}  # global shit
      for keys, reference in cls.GLOBAL_CMDS:
        resp = cls.find_value(keys)
        if isinstance(resp, int):
          g[reference] = resp
        elif resp:
          g[reference] = resp

      for keys, reference in cls.DB_CMDS:
        resp = cls.find_value(keys, False)
        if resp:
          if hasattr(db, reference):
            ret = getattr(db, reference)(*tuple(resp))
            print(ret)
            db.save()
            exit(0)
          else:
            raise ValueError('{} had no method {}'.format(db, reference))

      for keys, reference in cls.DB_FLAGS:
        for k in keys:
          if k in sys.argv:
            if hasattr(db, reference):
              ret, status = getattr(db, reference)()
              print(ret)
              if status:
                exit(0)
            else:
              raise ValueError('{} had no method {}'.format(db, reference))

      kwargs = {}
      for keys, reference in cls.CRAWL_CMDS:
        resp = cls.find_value(keys)
        if isinstance(resp, int):
          kwargs[reference] = resp
        elif resp:
          kwargs[reference] = resp

      if 'tags' in kwargs and not 'crawl_tags' in kwargs and CONFIG_FILE.PASS_TAGS:
        kwargs['crawl_tags'] = kwargs['tags']

      for keys, reference in cls.CRAWL_FLAGS:
        for k in keys:
          if k in sys.argv:
            kwargs[reference] = True
            break

      if '--debug' in sys.argv:
        DEBUGGING = True

      user = RestlessCrawler(sys.argv[-1], db, **kwargs)
      main(user, **g)
      db.save()
  #try:
  CLI.interface()
  # except (KeyboardInterrupt, Exception) as e:
  #   logging.exception(str(e))
  #   print("exitting...")
  #   exit(127)