WebBot source

#-*-coding:u8-*-
__author__ = u'Сергей Снегирев <tz4678@gmail.com>'

import urllib
import urllib2
import cookielib
import re
import os
import mimetypes
import sys
import random

def normalize_headers(headers):
    ret = {}
    for k, v in headers.items():
        ret[k.lower().strip()] = v
    return ret

def get_boundary(length=30):
    chars = 'QWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm1234567890'
    ret = ''
    while len(ret) < length:
        ret += random.choice(chars)
    return ret

class WebBot(object):
    def __init__(self):
        self.cookie_jar = cookielib.CookieJar()
        self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookie_jar))
        self.default_headers = {'User-Agent': 'Mozilla/5.0'}

    def request(self, url, data=None, headers={}):
        _headers = normalize_headers(self.default_headers)
        _headers.update(normalize_headers(headers))
        req = urllib2.Request(url, data, _headers)
        res = self.opener.open(req)
        return res

    def download(self, url, dst):
        r = self.request(url)
        fp = open(dst, 'wb')
        while 1:
            chunk = r.read(4096)
            if not chunk:
                fp.close()
                break
            fp.write(chunk)

    def fetch(self, url, data=None, headers={}):
        try:
            h = self.request(url, data, headers)
        except urllib2.HTTPError as h:
            pass
        content = h.read()
        ct = h.info().getheader('content-type')
        try:
            m = re.search(r'(?i)\bcharset\s*=\s*([^;]*)', ct)
            content = content.decode(m.group(1))
        except:
            pass
        return content

    def get(self, url, headers={}):
        return self.fetch(url, None, headers)

    def post(self, url, parameters, headers={}):
        data = urllib.urlencode(parameters)
        return self.fetch(url, data, headers)

    def send_multipart(self, url, fields={}, files={}, headers={}):
        lines = []
        boundary = get_boundary()
        headers = dict(headers)
        headers['content-type'] = 'multipart/form-data; boundary=' + boundary
        content_type = 'application/unknown'
        for k, v in fields.items():
            lines.extend([
                '--' + boundary,
                'Content-Disposition: form-data; name="' + k + '"',
                '',
                str(v)
            ])
        for k, v in files.items():
            if re.match('(?i)https?://', v, re.I):
                v = str(v)
                r = self.request(v)
                content_type = r.info().get('content-type', content_type)
                data = r.read()
            else:
                f = open(v, 'rb')
                data = f.read()
                f.close()
                ending = os.path.splitext(v)[1]
                content_type = mimetypes.types_map.get(ending, content_type)
                v = v.encode(sys.getfilesystemencoding())
            lines.extend([
                '--' + boundary,
                'Content-Disposition: form-data; name="' + k +'"; filename="' + os.path.basename(v) + '"',
                'Content-Type: ' + content_type,
                '',
                data
            ])
        lines.extend([
            '--' + boundary + '--',
            ''
        ])
        payload = '\r\n'.join(lines)
        return self.fetch(url, payload, headers)

    def del_cookie(self, domain=None, path=None, name=None):
        self.cookie_jar.clear(domain, path, name)

if __name__ == '__main__':
    wb = WebBot()
    # do smth