Advertisement
ejectedmatrix

Untitled

Nov 3rd, 2018
160
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 4.11 KB | None | 0 0
  1. using System;
  2. using System.Collections.Generic;
  3. using System.IO;
  4. using System.Linq;
  5. using System.Net.Http;
  6. using System.Threading.Tasks;
  7.  
  8. namespace Num
  9. {
  10. public class Program
  11. {
  12. static Program()
  13. {
  14. Task.Run(() =>
  15. {
  16. while(true)
  17. {
  18. if(_stack.Count > 0)
  19. {
  20. Console.WriteLine(_stack.Pop());
  21. }
  22. }
  23. });
  24. }
  25.  
  26. private static HttpClient _client = new HttpClient();
  27. private static readonly Stack<string> _stack = new Stack<string>();
  28.  
  29. public static async Task Main(string[] args)
  30. {
  31. int threadSize = 30;
  32. if (args.Length > 0)
  33. {
  34. threadSize = int.Parse(args[0]);
  35. }
  36.  
  37. string rawFiles = await _client.GetStringAsync("https://reiyo.kibii.org/~/dump/reiyo-full-dump.txt");
  38. string[] urls = rawFiles.Split('\n').Where(f => !f.StartsWith("#") && !string.IsNullOrWhiteSpace(f)).Select(c => c.Trim()).ToArray();
  39.  
  40. if (!Directory.Exists("out/"))
  41. {
  42. Directory.CreateDirectory("out/");
  43. }
  44.  
  45. IList<Task> tasks = new List<Task>();
  46. string[][] uriLists = urls.SliceArray(threadSize).Select(u => u.ToArray()).ToArray();
  47.  
  48. for (int i = 0; i < uriLists.Length; i++)
  49. {
  50. Task localTask = await Task.Factory.StartNew(async () => await DownloadAndSaveFiles(uriLists[i], i));
  51. tasks.Add(localTask);
  52. }
  53.  
  54. Task.WaitAll(tasks.ToArray());
  55. Console.WriteLine("All images have been downloaded.");
  56. Console.ReadKey();
  57. }
  58.  
  59. public static async Task DownloadAndSaveFiles(string[] urls, int batchId)
  60. {
  61. for (int i = 0; i < urls.Length; i++)
  62. {
  63. await Task.Run(async () =>
  64. {
  65. try
  66. {
  67. Uri uri = new Uri(urls[i]);
  68. string originalFileName = uri.AbsolutePath.Split('/')?.Last().Trim();
  69. if (string.IsNullOrWhiteSpace(originalFileName) || Guid.TryParse(originalFileName, out Guid res1))
  70. originalFileName = "no-file-name-available";
  71.  
  72. string fileId = uri.AbsolutePath.Split('/').FirstOrDefault(u => Guid.TryParse(u, out Guid res2)) ?? Guid.NewGuid().ToString("N");
  73.  
  74. HttpResponseMessage res = await _client.GetAsync(uri);
  75. string newFileName = originalFileName + "_" + fileId + "." + res.Content.Headers.ContentType.MediaType.Split('/').Last();
  76. string filePath = Path.Combine(Path.Combine(Directory.GetCurrentDirectory(), "out/"), newFileName);
  77.  
  78. using (Stream outStr = File.Create(filePath))
  79. {
  80. Stream inStr = await res.Content.ReadAsStreamAsync();
  81. await inStr.CopyToAsync(outStr);
  82. }
  83.  
  84. QueueOut("[B" + batchId + "] File DL'd (" + i.ToString() + "/" + urls.Length + "): '" + newFileName + "'");
  85. }
  86. catch
  87. {
  88. QueueOut("[B" + batchId + "] ERR (" + i.ToString() + "/" + urls.Length + ").");
  89. }
  90. });
  91. }
  92. }
  93.  
  94. private static void QueueOut(string message)
  95. {
  96. Task.Run(() => _stack.Push(message));
  97. }
  98. }
  99.  
  100. public static class EnumerbleExtensions
  101. {
  102. public static IEnumerable<IEnumerable<T>> SliceArray<T>(this T[] array, int groupSize)
  103. {
  104. List<T> locations = array.ToList();
  105. int nSize = array.Length / groupSize;
  106.  
  107. for (int i = 0; i < locations.Count; i += nSize)
  108. {
  109. yield return locations.GetRange(i, Math.Min(nSize, locations.Count - i));
  110. }
  111. }
  112. }
  113. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement