twitter.py

import json
import os
import signal
import time
import traceback
from contextlib import closing
from multiprocessing import Manager, Process, Queue
from multiprocessing.managers import SyncManager



def training_crawler(queue, track, lang='en', limit=-1):
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    while True:
        time.sleep(2)
        tweet = {'text': 'blaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'}
        try:
            queue.put(tweet)
        except IOError:
            return

def logger(queue):
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    for tweet in iter(queue.get, 'STOP'):
        print tweet['text']


def start(lang, fields=('pos', 'neg'), track=':(,:)'):
    manager = SyncManager()
    manager.start(signal.signal, [signal.SIGINT, signal.SIG_IGN])

    queue = manager.Queue(5000)

    crawler = Process(target=training_crawler, args=(queue, track, lang))
    worker = Process(target=logger, args=(queue,))

    crawler.start()
    worker.start()


    def handler(sig, frame):
        print 'Received Ctrl+C terminating..'

        queue.put('STOP')
        time.sleep(3)
        if worker.is_alive():
            worker.terminate()
            worker.join()
        crawler.terminate()
        crawler.join()
        manager.shutdown()

    signal.signal(signal.SIGINT, handler)
    return manager,queue,crawler,worker