Advertisement
Guest User

Untitled

a guest
Jul 23rd, 2016
51
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.00 KB | None | 0 0
  1. public static void browsingSpider(int page, int max_pages) throws Exception {
  2. while (page <= max_pages) {
  3. String url = base_url + String.valueOf(page) + "/" + search;
  4. final Document document = Jsoup.connect(url)
  5. .userAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36")
  6. .referrer("https://google.com")
  7. .get();
  8. //System.out.println(url);
  9. //System.out.println(document.outerHtml());
  10. for(Element link: document.select("a")) {
  11. final String get_url = link.attr("abs:href");
  12. final String relHref = link.attr("href");
  13. System.out.println(get_url);
  14. System.out.println(relHref);
  15. String href = "https://e621.net" + get_url;
  16. System.out.println(href);
  17. }
  18. page = page + 1;
  19. }
  20. }
  21. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement