#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# tweetstream to redis motherlode
#
import tweetstream_gzip
import logging
from tornado import escape
import urllib
import cjson, zlib, sys, time, os
import struct
from time import sleep

from tornado import ioloop
from tornado import iostream
import socket, ssl, sys, re
from urlparse import urlparse
from bs4 import BeautifulSoup
import opengraph
from lxml.html import document_fromstring
from lxml.html import fragment_fromstring
from lxml.html import fromstring

from lxml import etree
import lxml

# redis connection to motherlode
import redis
r = redis.StrictRedis(host='REDACTED', port=9999, db=0, password='REDACTED')
ps = r.pubsub()




# global debug vars
global n, mm, f, l, intime, sockcount, pcount
pcount = 0
sockcount = 0
n = 0
mm = 10000
intime = time.time()
##print time.time()




###################### 
#  OPEN GRAPH PARSER #
######################

size = 5000





class FastScraper(object):
	""" Scrape and find Open Graph metadata in urls """
	def __init__(self):
		self.io = ioloop.IOLoop.instance()
		self.redirect = False
		self.results = []
		self.redirect_url = ''


	def finish_up(self):
		""" Send json (self.rj) to motherlode """
		
		try:
			message = cjson.encode(self.rj)
			zm = zlib.compress(message)	# saving bandwidth
			r.publish('tweet', zm)		# send to motherlode
		
			self.rj = ''
			
		except Exception:
			print 'finish_up error: ', sys.exc_info()[1:3]
			pass



	def run(self, rjson):
		""" Run scraper """
		try:	
			# set json
			self.rj = {}
			self.rj = rjson
	
			# if no url, finish early
			if not self.rj['urls']:
				self.finish_up()
				return
	
			# set url and scrape
			self.longurl = self.rj['urls'][0]['expanded_url'].replace('\/', '/')
			self.scrape()
		except Exception:
			print 'run error: ', sys.exc_info()[1:3]
			pass

	


	
	def scrape(self):
	
		# make connection
		try:
			s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
			s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
			s.settimeout(2)	# 2 seconds timeout
			
			# streamloop.add_callback(self.connect, s)
			l_onoff = 1
			l_linger = 0
			s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', l_onoff, l_linger))
			streamloop.add_callback(self.connect, s)	# exp. tried putting this after linger settings, no apparent effect 
			
		except Exception:
			print 'socket error', sys.exc_info()[1:3]
			pass
	
	
	def prepare_url(self):
		try:
			# add http if missing in url
			if not 'http' in self.longurl:
				url = 'http://' + self.longurl
			
			# parse url 
			url = urlparse(self.longurl)
			self.url = url.netloc	# eg. www.vg.no
			self.path = url.path	# eg. /artikkel.php?artid=10119923
	
			# catch empty url
			if self.url == '' or self.url == None:
				return False
				
			# determine path
			self.dieurl = self.longurl.split(self.url)
			self.dieurl.reverse()
			self.path = self.dieurl[0]
			if self.path == '':
				self.path = '/'
			
			# set get size in bytes
			self.size = self.setsize(url.netloc)
			
			# determine SSL
			if url.scheme == 'https':
				self.SSL = True
			else:
				self.SSL = False
		
			return True
		except Exception:
			print 'prepare_url error: ', sys.exc_info()[1:3]
			print 'url: ', url
			return False
			
	

	
	
	def connect(self, s):
		# connect		
		
		try:
			prep = self.prepare_url()
	
			if not prep:
				self.stream.close()	
				return
			
			if self.SSL:
				ss = ssl.wrap_socket(s, do_handshake_on_connect=False)
				self.stream = iostream.SSLIOStream(ss)
				self.stream.connect((self.url, 443), self.send_ssl_request)
			else:
				self.stream = iostream.IOStream(s)
				self.stream.connect((self.url, 80), self.send_request)
			
			
			global sockcount
			sockcount += 1
# 			print 'currently open sockets: ', sockcount
		except Exception:
			print 'connect error:'
			print sys.exc_info()[1:3]
			print 'url: ', self.url
			print len(self.url)
			print self.path
			self.stream.close()	# maybe not needed
			pass
	
	def reconnect(self):
		print 'reconnect!reconnect!reconnect!reconnect!'
		pass
	
	
	
	
	def send_request(self):
		
		""" Send non-SSL request """
		self.stream.write("GET %s HTTP/1.0\r\nHost: %s\r\n\r\n" % (self.path, self.url))
		self.stream.read_until("\r\n\r\n", self.on_headers)	# read headers
		
		

	def send_ssl_request(self):
		""" Send SSL request """
		self.stream.write("GET %s HTTP/1.0\r\nHost: %s\r\n\r\n" % (self.path, self.url))
		self.stream.read_until("\r\n\r\n", self.on_headers)	# read headers
		


	def setsize(self, neturl):
		""" Custom scrape sizes in bytes """
		if 'youtu' in neturl:
			return 5000
		if 'aje.me' in neturl:
			return 10000
		if 'aljazeera' in neturl:
			return 10000
		else:
			# default size
			return size	

	


	
			
	def on_headers(self, data):
		""" Callback for headers """
		self.redirect = False
		global sockcount
		try:
			headers = {}
			for line in data.split("\r\n"):
				parts = line.split(": ")
				if len(parts) == 2:
					headers[parts[0].strip()] = parts[1].strip()
				
				if 'HTTP' in line:
					code = line.split(' ')
					if code[1] == '301' or code[1] == '302' or code[1] == '303':
						# set redirect
						self.redirect = True
	
			if self.redirect:
				if 'Location' in headers:
					self.longurl = headers['Location']
				if 'location' in headers:
					self.longurl = headers['location']
				
				self.stream.close()					# close				
				self.io.add_callback(self.scrape)	# restart

				sockcount -= 1
				return
				
			
			
			if 'Content-Length' in headers:
				if size > int(headers['Content-Length']):
					# make sure we're not scraping more than length of document
					self.size = int(headers['Content-Length'])

			try:
				# go ahead and scrape the body
				self.stream.read_bytes(self.size, self.on_body)
			except Exception:
				print 'on headers ==> body error:'
				print sys.exc_info()[1:3]
				print headers
				print data
				print self.size
				print self.url
				print self.path
				sockcount -= 1
				self.stream.close()	
				pass

		except Exception:
			print 'on headers error:'
			print sys.exc_info()[1:3]
			print headers
			print data
			sockcount -= 1
			self.stream.close()	
			pass





			
						
	
	def on_body(self, data):
		""" Callback after scraping body """
		
		# parse the og:tags
		self.parse_og(data)		
		
		# finish up
		self.finish_up()

		# clean up
		self.stream.close()
		global sockcount
		sockcount -= 1



	
	
	
	
	def parse_og(self, data):
		""" lxml parsing to the bone! """
		
# 		return	# debug, still all sockets but no lxml	// runs fine, aka it's lxml, not amazon aws blocking socket use for example...
		
		try:
			tree = etree.HTML( data )
			m = tree.xpath("//meta[@property]")
			
# 			for i in m:
# # 				print (i.attrib['property'], i.attrib['content'])		
# 				y = i.attrib['property']
# 				x = i.attrib['content']
# 				self.rj[y] = x
		
		
			tree = ''
			m = ''
			i = ''
			y = ''
			x = ''
			
			del tree
			del m
			del i
			del y
			del x

			
		
		except Exception:
			print 'lxml error: ', sys.exc_info()[1:3]
			print len(data)
			data = ''
			tree = ''
			m = ''
			i = ''
			x = ''
			y = ''
			pass
	
	







####################
#  RSJON PARSER	   #
####################


# todo: convert facebook, instagram, bambuser, etc. to rawger json


def t2r(tdoc):
	""" Convert Twitter JSON to Rawger JSON. """
	
	try:
		

		# if rate limit

		rjson = {} 
		
		# general
		rjson['source'] = 'twitter'
		rjson['device'] = tdoc['source']	# todo
	
		# user
		rjson['userid'] = tdoc['user']['id']
		rjson['fullname'] = tdoc['user']['name']
		rjson['screenname'] = tdoc['user']['screen_name']
		rjson['avatar'] = tdoc['user']['profile_image_url_https'].replace('\/', '/')
		rjson['description'] = tdoc['user']['description']
		rjson['followers'] = tdoc['user']['followers_count']
		rjson['friends'] = tdoc['user']['friends_count']
		rjson['following'] = tdoc['user']['friends_count']
		rjson['userlocation'] = tdoc['user']['location']
		rjson['statuscount'] = tdoc['user']['statuses_count']
		rjson['hashtags'] = tdoc['entities']['hashtags']
		rjson['mentions'] = tdoc['entities']['user_mentions']
		
		# urls
		rjson['urls'] = tdoc['entities']['urls']

		# tweet
		rjson['id'] = tdoc['id']
		if 'lang' in tdoc: 
			rjson['lang'] = tdoc['lang'] 
		else: 
			rjson['lang'] = None
		
		rjson['rt'] = tdoc['retweet_count']
		
		try:
			rjson['text'] = escape.linkify(tdoc['text'].replace('\/', '/'), True, extra_params='target="_blank"')	# todo: get rid of &amps
		except Exception:
			print 't2r escape error!', sys.exc_info()[1:3]
			pass


		# different types of geo
		# only [geo] atm, twitter places too general
		rjson['geo'] = tdoc['geo']

		# if retweet, get original timestamp instead
		if 'retweeted_status' in tdoc.keys():
			rjson['timestamp'] = tdoc['retweeted_status']['created_at']
		else:
			rjson['timestamp'] = tdoc['created_at']


		# if twitter embedded image
		if 'media' in tdoc['entities'].keys():
			for m in tdoc['entities']['media']:
				media = m['media_url_https'].replace('\/', '/')
			rjson['photo'] = media
		else:
			rjson['photo'] = None
			
			
		# if other images
		if not rjson['photo']:
			if rjson['urls']:
			
			
				for urls in rjson['urls']:
					url = urls['expanded_url'].replace('\/', '/')

					#youtube
					if ('youtu' in url):
						rjson['video'] = url
						#vid = video_id(self, youtube_url)
	
					#yfrog
					if ('yfrog' in url):
						rjson['photo'] = url
						
					#instagram
					if ('instagr' in url):
						rjson['photo'] = url
					
					#twitpic
					if ('twitpic' in url):
						arr = url.split('/')
						rjson['photo'] = 'https://twitpic.com/show/large/'
						rjson['photo'] += '%s' % arr[3].encode('ascii')
					
					#tumblr
					if ('media.tumblr' in url):
						if ('.jpg' in url):
							rjson['photo'] = '%s500.jpg' % url[:-7]
				
		# done!
		return rjson


	except Exception:
		print 't2r general error!'
		print sys.exc_info()[1:3]
		print tdoc
		pass



















def callback(message):
	""" Callback for tweetstream_gzip """
	try:
		global n, mm, f, l, intime
		n += 1
		if n == 1:
			doc = cjson.decode(message)
			f = doc['id']
			##print '@%s: %s' % (doc['user']['screen_name'], doc['text'])
	
		if n == mm:
			mm += 1000
			n += 1
			print '%i tweets per second.' % ( n / ( time.time() - intime ))			
	
	
	
		if len(message) > 1:
			# process from json to rawger json + opengraph
			m = cjson.decode(message)
			
			if 'limit' in m:
				print 'limit!'
				print m
				return

			# convert to rawger json
			rj = t2r(m)
			
			# add scrape to ioloop
			streamloop.add_callback(fast, rj)
	
	except Exception:
		print 'stream.py callback error: ', sys.exc_info()
		pass




def fast(rj):
	""" IOLoop add_callback for FastScraper """
	try:
		fs = FastScraper()
		fs.run(rj)		
		global sockcount

	except Exception:
		print 'fast error:', sys.exc_info()[1:3]
		pass
	
	
	









if __name__ == '__main__':

	



	#config 0 
	configuration_0 = {
		"twitter_consumer_key": "sorry",
		"twitter_consumer_secret": "sorry",
		"twitter_access_token": "sorry",
		"twitter_access_token_secret": "sorry"
	}

	keywords_0 = [
 		'love',
 		'RT',
		'you',
		
	]

	tracks_0 = ','.join(keywords_0)
	stream_0 = tweetstream_gzip.TweetStream(configuration_0, gzip=True)
	url_0 = "/1.1/statuses/filter.json?track=%s" % urllib.quote(tracks_0).encode('utf-8')

	stream_0.fetch(url_0, callback=callback)


	print 'Stream is up.'
	


	from tornado.ioloop import IOLoop
	streamloop = IOLoop.instance()
	streamloop.start()