collect.py

'''
http://pastebin.com/D1JUm1nk
5902a57cf13f6280665357eaceec1cabe9129f91369475697abe85be52278ea8


In this step we'll be creating a distributed index. This script is comprised of
a scanner called the Carpenter that will spawn several threads, each sending out
feelers looking for web servers out there that are willing to respond. Responses
are kept in a sqlite database named walrus.db

walrus.db will contain a table, Walrus, with the following columns:
    host -- The host IP address of the discovered server
    key  -- A header key from the server response
    val  -- The associated value to the header key

This will allow us to later create a massively parallel database of all servers
that interest us. But for now, let's just do some collecting.

Save this file as collect.py and execute it using
python (https://www.python.org/downloads/) with the following command:

python collect.py

The longer you run this the more you will collect. You can quit and resume at
any time.

Feel free to query this table directly should you be interested in this
intermediate state. But whatever you do, keep your walrus.db file safe because
we'll be using this in our next step, which will be published with the following
key: 38a11b99402ab435eb0bc53419bf278e582cb3c3361da5be1b8f69d3bf422470

Thank you for your support. Together we can do this!

DISCLAIMER: Your ISP (or local law) may prohibit scanning of this nature. Hey,
let's be careful out there.
'''

try:
    from Queue import Queue
except:
    from queue import Queue
from random import randrange as rand
try:
    from requests import get
except:
    print('requests module required:\npip install requests')
    exit()
from sqlite3 import connect
from threading import Thread, active_count
from time import sleep

# whitelist of header keys
KEYS = [
    'connection',
    'content-encoding',
    'content-length',
    'content-type',
    'etag',
    'server',
    'x-frame-options',
    'x-powered-by'
]

class Carpenter:

    def __init__(self, threadcount=100):
        self.threadcount = threadcount

    def run(self):
        '''
        Let's get this party started!
        '''
        queue = Queue()
        threads = []
        for i in range(self.threadcount):
            self.walk(self.worker, queue)
        self.walk(self.tweedle, queue)
        while active_count() > 0:
            sleep(0.25)

    def walk(self, fn, queue):
        '''
        Spawn a daemon thread. 2spooky4me.
        '''
        thread = Thread(target=fn, args=(queue,))
        thread.daemon = True
        thread.start()

    def tweedle(self, queue):
        '''
        Process the queue in one thread (because sqlite).
        '''
        dee = connect('walrus.db')
        dum = dee.cursor()
        dum.execute('''
            create table if not exists Walrus (host text, key text, val text)
        ''')
        dum.execute('create index if not exists idx_host on Walrus (host)')
        dee.commit()
        while True:
            host, headers = queue.get()
            if headers:
                print(host)
                values = []
                for key, val in headers.items():
                    print('   ', key, val)
                    values.append((host, key, val))
                dum.executemany('''
                    insert into Walrus (host, key, val) values (?, ?, ?)
                ''', values)
                dee.commit()

    def work(self):
        '''
        Do work on one random host.
        '''
        host = '{}.{}.{}.{}'.format(rand(256), rand(256), rand(256), rand(256))
        req = get('http://' + host, timeout=5, headers={
            'User-Agent': 'Do you admire the view?',
        })
        headers = {}
        for key, val in req.headers.items():
            # Restrict the value length... There are weirdos ones out there...
            if key.lower() in KEYS and len(val) < 256:
                headers[key] = val
        headers['status'] = req.status_code
        return [host, headers]

    def worker(self, queue):
        '''
        Work forever... :|
        Don't worry, computers are into that kind of thing.
        '''
        while True:
            try:
                result = self.work()
            except:
                continue
            queue.put(result)


if __name__ == '__main__':
    Carpenter().run()