Guest User

Scraping Google Sketchup, pt.1 by @mtrc

a guest
Jul 5th, 2013
132
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. private static IEnumerator aSyncSearchForModel(string searchterm, MineMain root){
  2. WebClient w = new WebClient();
  3. string htmlstring = w.DownloadString("http://sketchup.google.com/3dwarehouse/doadvsearch?title="+searchterm
  4. +"&scoring=d&btnG=Search+3D+Warehouse&dscr=&tags=&styp=m&complexity=any_value&file="
  5. +"zip"+"&stars=any_value&nickname=&createtime=any_value&modtime=any_value&isgeo=any_value&addr=&clid=");
  6. HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
  7. doc.LoadHtml(htmlstring);
  8.  
  9. //Grab a result from the search page
  10. HtmlAgilityPack.HtmlNodeCollection results = doc.DocumentNode.SelectNodes("//div[@class='searchresult']");
  11. string model_page_url = "http://sketchup.google.com"+results[0].SelectSingleNode(".//a").Attributes["href"].Value;
  12.  
  13. htmlstring = w.DownloadString(model_page_url);
  14. doc.LoadHtml(htmlstring);
  15.  
  16. //Find the downloadable zip file
  17. results = doc.DocumentNode.SelectNodes("//a[contains(@href,'rtyp=zip')]");
  18.  
  19. string url = "http://sketchup.google.com"+WWW.UnEscapeURL(results[0].Attributes["href"].Value);
  20. url = url.Replace("&", "&");
  21. UnityEngine.Debug.Log (WWW.EscapeURL(results[0].Attributes["href"].Value));
  22. UnityEngine.Debug.Log (url);
  23.  
  24. WWW model_zip = new WWW(url);
  25. yield return model_zip;
  26.  
  27. string file_loc = "path/to/downloads/"+searchterm+".zip";
  28. System.IO.FileStream filestream = new System.IO.FileStream(file_loc, System.IO.FileMode.Create, System.IO.FileAccess.Write); //Assets/Resources/sfx/
  29. filestream.Write (model_zip.bytes, 0, model_zip.bytes.Length);
  30. filestream.Close();
  31.  
  32. }
RAW Paste Data