SHARE
TWEET

Untitled

a guest Sep 30th, 2012 190 Never
  1. "http://www.facebook.com/"
  2. ==========
  3. {:homepage=>{:url=>"http://www.facebook.com/", :title=>"Welcome to Facebook - Log In, Sign Up or Learn More", :meta_tags=>"noodp, noydir"}, :robots_file=>"# Notice: if you would like to crawl Facebook you can\n# contact us here: http://www.facebook.com/apps/site_scraping_tos.php\n# to apply for white listing. Our general terms are available\n# at http://www.facebook.com/apps/site_scraping_tos_terms.php\n\nUser-agent: baiduspider\nDisallow: /ac.php\nDisallow: /ae.php\nDisallow: /album.php\nDisallow: /ap.php\nDisallow: /autologin.php\nDisallow: /checkpoint/\nDisallow: /feeds/\nDisallow: /l.php\nDisallow: /o.php\nDisallow: /p.php\nDisallow: /photo.php\nDisallow: /photo_comments.php\nDisallow: /photo_search.php\nDisallow: /photos.php\n\nUser-agent: Googlebot\nDisallow: /ac.php\nDisallow: /ae.php\nDisallow: /album.php\nDisallow: /ap.php\nDisallow: /autologin.php\nDisallow: /checkpoint/\nDisallow: /feeds/\nDisallow: /l.php\nDisallow: /o.php\nDisallow: /p.php\nDisallow: /photo.php\nDisallow: /photo_comments.php\nDisallow: /photo_search.php\nDisallow: /photos.php\n\nUser-agent: msnbot\nDisallow: /ac.php\nDisallow: /ae.php\nDisallow: /album.php\nDisallow: /ap.php\nDisallow: /autologin.php\nDisallow: /checkpoint/\nDisallow: /feeds/\nDisallow: /l.php\nDisallow: /o.php\nDisallow: /p.php\nDisallow: /photo.php\nDisallow: /photo_comments.php\nDisallow: /photo_search.php\nDisallow: /photos.php\n\nUser-agent: naverbot\nDisallow: /ac.php\nDisallow: /ae.php\nDisallow: /album.php\nDisallow: /ap.php\nDisallow: /autologin.php\nDisallow: /checkpoint/\nDisallow: /feeds/\nDisallow: /l.php\nDisallow: /o.php\nDisallow: /p.php\nDisallow: /photo.php\nDisallow: /photo_comments.php\nDisallow: /photo_search.php\nDisallow: /photos.php\n\nUser-agent: seznambot\nDisallow: /ac.php\nDisallow: /ae.php\nDisallow: /album.php\nDisallow: /ap.php\nDisallow: /autologin.php\nDisallow: /checkpoint/\nDisallow: /feeds/\nDisallow: /l.php\nDisallow: /o.php\nDisallow: /p.php\nDisallow: /photo.php\nDisallow: /photo_comments.php\nDisallow: /photo_search.php\nDisallow: /photos.php\n\nUser-agent: Slurp\nDisallow: /ac.php\nDisallow: /ae.php\nDisallow: /album.php\nDisallow: /ap.php\nDisallow: /autologin.php\nDisallow: /checkpoint/\nDisallow: /feeds/\nDisallow: /l.php\nDisallow: /o.php\nDisallow: /p.php\nDisallow: /photo.php\nDisallow: /photo_comments.php\nDisallow: /photo_search.php\nDisallow: /photos.php\n\nUser-agent: teoma\nDisallow: /ac.php\nDisallow: /ae.php\nDisallow: /album.php\nDisallow: /ap.php\nDisallow: /autologin.php\nDisallow: /checkpoint/\nDisallow: /feeds/\nDisallow: /l.php\nDisallow: /o.php\nDisallow: /p.php\nDisallow: /photo.php\nDisallow: /photo_comments.php\nDisallow: /photo_search.php\nDisallow: /photos.php\n\nUser-agent: Yandex\nDisallow: /ac.php\nDisallow: /ae.php\nDisallow: /album.php\nDisallow: /ap.php\nDisallow: /autologin.php\nDisallow: /checkpoint/\nDisallow: /feeds/\nDisallow: /l.php\nDisallow: /o.php\nDisallow: /p.php\nDisallow: /photo.php\nDisallow: /photo_comments.php\nDisallow: /photo_search.php\nDisallow: /photos.php\n\nUser-agent: *\nDisallow: /\n\n# E-mail sitemaps@lists.facebook.com if you are authorized to access these and are getting denied.\nSitemap: http://www.facebook.com/sitemap.php\n\n\n"}
  4. ==========
  5. finished call to redis
RAW Paste Data
Top