Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- public static void browsingSpider(int page, int max_pages) throws Exception {
- while (page <= max_pages) {
- String url = base_url + String.valueOf(page) + "/" + search;
- final Document document = Jsoup.connect(url)
- .userAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36")
- .referrer("https://google.com")
- .get();
- //System.out.println(url);
- //System.out.println(document.outerHtml());
- for(Element link: document.select("a")) {
- final String get_url = link.attr("abs:href");
- final String relHref = link.attr("href");
- System.out.println(get_url);
- System.out.println(relHref);
- String href = "https://e621.net" + get_url;
- System.out.println(href);
- }
- page = page + 1;
- }
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement