immoscrap.py

#!/usr/local/bin/python

# To compile this script, you will need Python 2.4, and the following packages:
#
# BeautifulSoup 3.0.3, to parse the HTML code
#      http://www.crummy.com/software/BeautifulSoup/#Download
#
# PyWin32 for the optional Excel support
#      http://www.python.net/crew/mhammond/win32

import sys
import os.path
import re
import urllib2
import threading
import time
import Queue
from BeautifulSoup import BeautifulSoup, SoupStrainer

createCache = False

m = re.compile(r'\b(\d+)\b')
def GetInt(p):
    l = m.findall(p)
    if len(l):
        return l[0]
    else:
        return ""


def GetInt2(p): return "".join(m.findall(p))


class DownloadWorker(threading.Thread):
    def __init__(self, codePostal):
        threading.Thread.__init__(self)
        self.__codePostal = codePostal

    def run(self):
        liste = SeLogerScrapper(self.__codePostal).GetDepartement()
        exportQueue.put([self.__codePostal, liste])


class ExcelExporter:
    def __init__(self):
        try:   # Import COM if available
            from win32com.client import Dispatch
            self.__xlApp = Dispatch("Excel.Application")
            self.__xlApp.Workbooks.Open(os.path.abspath("immoscrap.xls"))
        except ImportError:
            self.__xlApp = None


    def Export(self, codePostal, liste):
        if self.__xlApp is not None:
            self.__xlApp.Run('SetupPage', codePostal)
            for v in liste:
                self.__xlApp.Run('AddLine', v[0], v[1], v[2] )
        else:
            output = open ("prix_" + str(dep) + ".csv", "wb")
            for t in liste:
                line = t[0] + ", " + t[1] + ", " + t[2] + '\n'
                output.write(line)
            output.close

    def End(self):
            self.__xlApp.Visible = 1
            del self.__xlApp


class SeLogerScrapper:
    def __init__(self, codePostal):
        self.__codePostal = codePostal
        self.ads = SoupStrainer('table', attrs={'class' : re.compile('addchapeau|addpages')} )

    def GetNextPageNumber(self, soup):
        try:
            addPages = soup.find(attrs={'class' : 'addpages'} )
            nextLink = addPages.find('td', align='right').a['href']
            return GetInt2(nextLink)
        except:
            return 0


    def DownloadPage(self, startPrice, pageNumber):
        pageCacheName = "cache_" + str(self.__codePostal) +"_" + str(startPrice) +"_" + str(pageNumber) + ".html"
        if not os.path.exists(pageCacheName):
            print "* Download", self.__codePostal, " Prix :", startPrice, ", Page:" , pageNumber

            pageName = "http://www.seloger.com/recherche.htm?euro=1&idtt=2&rnd=121&cp="
            url= pageName + str(self.__codePostal)

            if False: #appartement
                url += "&idtypebien=1"
            else:
                url += "&idtypebien=2"

            if startPrice:
                url+="&pxmin=" + str(startPrice)

            if pageNumber:
                url+="&immobw_ann_qrypg=" + str(pageNumber)

            if 0: #if you have a proxy, set it here
                proxy_handler = urllib2.ProxyHandler({'http': 'http://xx.xx.xx.xx:80'} )
                opener = urllib2.build_opener(proxy_handler)
                urllib2.install_opener(opener)

            target = urllib2.urlopen(url).read()

            if createCache :
                output = open (pageCacheName, "wb")
                output.write(target)
                output.close

        else:
            print "* Cache(", self.__codePostal, "). Prix :", startPrice, ", Page:" , pageNumber
            target = open (pageCacheName, "rb")

        soup = BeautifulSoup(target, fromEncoding="utf-8", parseOnlyThese=self.ads)

        return soup


    def ExtractAds(self, soup, liste):
        annonceTags = soup.findAll(attrs={'class' : 'addchapeau'} )
        for annonce in annonceTags:
            id = annonce.tr.td.span.a['href']
            price = annonce.find(attrs={'class':'addprixeu'}).contents[0]
            surface = annonce.tr.td.contents[2]
            liste.append ([id, GetInt(surface), GetInt2(price) ])
        return liste


    def GetDepartement(self):
        liste = []
        startPrice = 0
        departmentDone = False
        while not departmentDone:
            pageNumber = nextPage = 1
            pagesToDo=True

            while pagesToDo:
                soup = self.DownloadPage(startPrice, pageNumber )
                self.ExtractAds(soup, liste)

                nextPage = self.GetNextPageNumber(soup)
                if (nextPage==0):
                    pagesToDo=False
                else:
                    pageNumber = nextPage

            if str(pageNumber)!="29":
                departmentDone = True
            else:
                lastPrice = startPrice
                startPrice = liste[-1][2]
                if (startPrice==lastPrice):
                    departmentDone = True

        return liste


exportQueue = Queue.Queue(0)

print "Entrez la liste des codes postaux (separes par espace) : "
deps = sys.stdin.readline().split()

xl = ExcelExporter()

for departement in deps:
    DownloadWorker(departement).start()

for departement in deps:
    item = exportQueue.get()
    if len(item[1]):
        xl.Export(*item)
    else:
        print "Le code postal", item[0], "n'a pas produit de resultats."
xl.End()