Guest User

Untitled

a guest
Apr 24th, 2018
73
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.72 KB | None | 0 0
  1. public static class DownloadUtilities
  2. {
  3. public static void DownloadLinks(Dictionary<string, string> files)
  4. {
  5. Parallel.ForEach(
  6. files,
  7. new ParallelOptions { MaxDegreeOfParallelism = 20 },
  8. DownloadLink);
  9. }
  10.  
  11. private static void DownloadLink(KeyValuePair<string, string> link, bool retrying = false)
  12. {
  13. try
  14. {
  15. using (var webClient = new WebClient())
  16. {
  17. webClient.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0)");
  18. webClient.DownloadFile(new Uri(link.Key), link.Value);
  19. }
  20. }
  21. catch (WebException e)
  22. {
  23. if (retrying) { return; } // Silently exit, we're retrying.
  24.  
  25. if (e.Status != WebExceptionStatus.ProtocolError)
  26. {
  27. throw;
  28. }
  29.  
  30. if (e.Message.Contains("(504) Gateway Timeout") || e.Message.Contains("(403) Forbidden"))
  31. {
  32. if (!RetryFailedDownload(link))
  33. {
  34. Program.FailedDownloads.Add(link.Key); // Lets settle for the fact it can't download, and add it to the failed list.
  35. }
  36. }
  37. else
  38. {
  39. Logger.Error("Failed to download: " + link.Key);
  40. Logger.Error(e.Message);
  41. }
  42. }
  43. }
  44.  
  45. private static bool RetryFailedDownload(KeyValuePair<string, string> link)
  46. {
  47. for (var i = 0; i < 4; i++) // Retry mechanism for 4 trys?
  48. {
  49. DownloadLink(link, true);
  50.  
  51. if (File.Exists(link.Value)) // It finally managed to download?
  52. {
  53. return true;
  54. }
  55. }
  56.  
  57. return false;
  58. }
  59. }
Add Comment
Please, Sign In to add comment