Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- var request = require("request");
- var tough = require('tough-cookie');
- var Cookie = tough.Cookie;
- var FileCookieStore = require("tough-cookie-store");
- var fs = require("fs");
- var cheerio = require("cheerio");
- var mongojs = require("mongojs");
- var db = mongojs("reddit");
- function Reddit(user, pass) {
- var me = this;
- this.user = user;
- this.pass = pass;
- if (!this.user) this.user = "none";
- me.cookieFile = new FileCookieStore("./cookies/" + me.user + ".json", {encrypt: false});
- var jar = request.jar(me.cookieFile);
- me.jar = jar;
- this.request = request.defaults({
- jar : me.jar,
- headers: {
- "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
- }
- });
- }
- Reddit.prototype.login = function(cb) {
- var me = this;
- this.request.post({
- url: "https://www.reddit.com/api/login/" + me.user,
- headers: {
- "x-requested-with": "XMLHttpRequest",
- "referer": "https://www.reddit.com/"
- },
- form: {
- op: "login-main",
- user: me.user,
- passwd: me.pass,
- rem: "on",
- api_type: "json"
- }
- }, function(err, res, body) {
- me.cookieFile.flush();
- cb();
- });
- };
- Reddit.prototype.getHome = function(cb) {
- var me = this;
- this.request("https://www.reddit.com/", function(err, res, body) {
- fs.writeFileSync("scratch.html", body);
- });
- }
- Reddit.prototype.getSubReddits = function(pages, cb) {
- var me = this;
- var subs = [];
- function getPage(link) {
- me.request(link, function(err, res, body) {
- var $ = cheerio.load(body);
- $("div.subreddit").each(function(i, el) {
- var sub = {
- _id: $(el).find(".titlerow a").text().split(":")[0],
- subscribed: null,
- title: $(el).find(".titlerow a").text(),
- url: $(el).find(".titlerow a").attr("href"),
- text: $(el).find(".usertext-body").text().trim(),
- subs: Number($(el).find(".score.unvoted .number").eq(0).text().replace(/,/g, ""))
- };
- if ($(el).find("a").eq(0).text() === "subscribe") {
- sub.subscribed = false;
- } else {
- sub.subscribed = true;
- }
- db.collection("subreddits").update({_id: sub._id}, {$set: sub}, {upsert: true});
- subs.push(sub);
- });
- pages--;
- if (pages === 0) {
- cb(subs);
- } else {
- var next = $(".next-button a").attr("href");
- getPage(next);
- }
- });
- }
- getPage("https://www.reddit.com/subreddits/");
- };
- Reddit.prototype.getSubReddit = function(id, cb) {
- var ret = new SubReddit(id);
- ret.request = this.request;
- return ret;
- }
- function SubReddit(id) {
- var me = this;
- me._id = id;
- me.url = "https://www.reddit.com/" + me._id;
- me.nextUrl = "";
- me.posts = [];
- }
- SubReddit.prototype.getTopYear = function(pages, cb) {
- var me = this;
- me.getPosts(me.url + "/top/?sort=top&t=year", pages, cb);
- };
- SubReddit.prototype.getPosts = function(url, pages, cb) {
- var me = this;
- me.request(url, function(err, res, body) {
- var $ = cheerio.load(body);
- $("div.thing.link").each(function(i, el) {
- var post = {
- _id: $(el).attr("data-fullname"),
- sub: me._id,
- title: $(el).find("a.title").text().trim(),
- link: $(el).find("a.title").attr("href"),
- score: Number($(el).find(".score.likes").text()),
- comments: Number($(el).find(".bylink.comments").text().split(" ")[0]),
- thumbnail: $(el).find("a.thumbnail img").attr("src"),
- posted: new Date($(el).find("time").attr("title"))
- };
- me.posts.push(post);
- });
- pages--;
- if (pages === 0) {
- cb();
- } else {
- var nextUrl = $(".next-button a").attr("href");
- me.getPosts(nextUrl, pages, cb);
- }
- });
- };
- function getPosts(req, url, cb) {
- var ret = {
- nextUrl: null,
- posts: []
- };
- req(url, function(err, res, body) {
- var $ = cheerio.load(body);
- $("div.thing.link").each(function(i, el) {
- var post = {
- title: $(el).find("a.title").text().trim(),
- link: $(el).find("a.title").attr("href"),
- score: Number($(el).find(".score.likes").text()),
- comments: Number($(el).find(".bylink.comments").text().split(" ")[0]),
- thumbnail: $(el).find("a.thumbnail img").attr("src"),
- posted: new Date($(el).find("time").attr("title"))
- };
- ret.posts.push(post);
- });
- ret.nextUrl = $(".next-button a").attr("href");
- cb(ret);
- });
- }
- module.exports = function(user, pass) {
- return new Reddit(user, pass);
- };
Add Comment
Please, Sign In to add comment