Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- start = 0
- keep_going = true
- while keep_going
- # use threads to contain memory consumption
- thread = Thread.new do |t|
- @search_result_page_num = start/10
- puts "google page #{start/10}"
- page = a.get("http://www.google.com/search?safe=off&start=#{start}&sa=N&q=#{search_str}")
- page.links.each do |link|
- #do some stuff
- end
- keep_going = false if page.body.include?('we have omitted some') || page.body.include?('did not match any documents')
- t.join
- Thread.exit
- end
- # wait for thread
- while thread.alive?
- sleep 1
- end
- start += 10
- sleep 5
- end
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement