Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- public static class DownloadUtilities
- {
- public static void DownloadLinks(Dictionary<string, string> files)
- {
- Parallel.ForEach(
- files,
- new ParallelOptions { MaxDegreeOfParallelism = 20 },
- DownloadLink);
- }
- private static void DownloadLink(KeyValuePair<string, string> link, bool retrying = false)
- {
- try
- {
- using (var webClient = new WebClient())
- {
- webClient.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0)");
- webClient.DownloadFile(new Uri(link.Key), link.Value);
- }
- }
- catch (WebException e)
- {
- if (retrying) { return; } // Silently exit, we're retrying.
- if (e.Status != WebExceptionStatus.ProtocolError)
- {
- throw;
- }
- if (e.Message.Contains("(504) Gateway Timeout") || e.Message.Contains("(403) Forbidden"))
- {
- if (!RetryFailedDownload(link))
- {
- Program.FailedDownloads.Add(link.Key); // Lets settle for the fact it can't download, and add it to the failed list.
- }
- }
- else
- {
- Logger.Error("Failed to download: " + link.Key);
- Logger.Error(e.Message);
- }
- }
- }
- private static bool RetryFailedDownload(KeyValuePair<string, string> link)
- {
- for (var i = 0; i < 4; i++) // Retry mechanism for 4 trys?
- {
- DownloadLink(link, true);
- if (File.Exists(link.Value)) // It finally managed to download?
- {
- return true;
- }
- }
- return false;
- }
- }
Add Comment
Please, Sign In to add comment