Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- var express = require('express');
- var fs = require('fs');
- var request = require('request');
- var cheerio = require('cheerio');
- var async = require('async');
- var csvjson = require('csvjson');
- var diacritics = require('diacritic');
- var curl = require('node-curl');
- var stringify = require('querystring').stringify;
- var url = "http://www.meilleursagents.com";
- var app = express();
- //Cookies are disabled by default
- //request = request.defaults({jar: true});
- var form = {
- url: '',
- action:'signin',
- user_username: "bob@ledur.fr",
- user_password: "ledur"
- };
- var conf = {
- 'url': url,
- 'jar': request.jar()
- };
- var options = {
- 'uri': url + '/signin?show=signin',
- 'jar': conf.jar,
- 'form': form
- };
- // 3 - callback pour confirmer que je suis toujours login
- function verifLogin(err, res, body) {
- if (err) return console.log("ERROR:", err);
- var $ = cheerio.load(body);
- console.log($('a[href="/my/"]').text());
- start();
- }
- var base;
- //start();
- connect();
- function connect() {
- request.post(options, function (err, res, body) {
- if (err) {
- console
- .log(err);
- } else {
- request.get(conf, verifLogin);
- }
- });
- }
- function start() {
- csvjson.toSchemaObject('./fake_ads_samples.csv').save('./fake_ads_samples.json');
- base = JSON.parse(fs.readFileSync('./fake_ads_samples.json', 'utf8'));
- getSqMeterPrices(base);
- // console.log(base);
- }
- function getSqMeterPrices(base) {
- async.eachSeries(base, function (item, callback) {
- if (item.chk == "true") {
- return callback();
- }
- var u = url + "/prix-immobilier/" + diacritics.clean(item.city.name.toLowerCase().replace(/\s/g, '-')) + '-' + item.city.postalCode + '/';
- console.log(u);
- request.get({url: u, jar: conf.jar}, function (err, res, body) {
- var $ = cheerio.load(body);
- console.log($('a[href="/my/"]').text());
- console.log($('title').text());
- if (err || res.statusCode != 200) {
- return console.log(err, res.statusCode);
- } else {
- // console.log(res.headers);
- // console.log($('title').text());
- // console.log(body);
- $('li.price_house').each(function (i, element) {
- var low = $(this).next();
- var avg = $(this).next().next();
- var high = $(this).next().next().next();
- console.log(item.city.name + ',' + low.text().clear() + ',' + avg.text().clear() + ',' + high.text().clear());
- });
- }
- callback();
- });
- }, function (err) {
- if (err) {
- console.log(err);
- } else {
- console.log('Grab Done!');
- }
- });
- }
- String.prototype.clear = function () {
- return this.replace(/[\sā¬]/g, '');
- };
- module.exports = app;
- var server = app.listen(3333, function() {
- console.log('Express server listening on port ' + server.address().port);
- });
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement