Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- use scraper::{Html, Selector};
- #[tokio::main]
- pub async fn main() -> Result<(), Box<dyn std::error::Error>> {
- // the starting place for our scraper
- let news_homepage = "https://news.ycombinator.com/news";
- let response = reqwest::get(news_homepage)
- .await?;
- let news_homepage_html = Html::parse_document(&response.text().await?);
- let link_selector = Selector::parse("a[href]").unwrap();
- let img_selector = Selector::parse("img[src]").unwrap();
- for link_element in news_homepage_html.select(&link_selector) {
- // follow each link
- let next_page = link_element.value().attr("href").unwrap();
- println!("INFO: Visiting link {:}", next_page);
- let next_page_response = reqwest::get(next_page).await;
- let next_page_response = match next_page_response {
- Ok(response) => response,
- // Err(error) => println!("WARN: Problem getting the url '{}'. \
- // The error was {:?}", next_page, error),
- Err(error) => panic!("WARN: Problem getting the url '{}'. \
- The error was {:?}", next_page, error),
- };
- }
- Ok(())
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement