Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- using System;
- using System.Collections.Generic;
- using System.IO;
- using System.Linq;
- using System.Net.Http;
- using System.Threading.Tasks;
- namespace Num
- {
- public class Program
- {
- static Program()
- {
- Task.Run(() =>
- {
- while(true)
- {
- if(_stack.Count > 0)
- {
- Console.WriteLine(_stack.Pop());
- }
- }
- });
- }
- private static HttpClient _client = new HttpClient();
- private static readonly Stack<string> _stack = new Stack<string>();
- public static async Task Main(string[] args)
- {
- int threadSize = 30;
- if (args.Length > 0)
- {
- threadSize = int.Parse(args[0]);
- }
- string rawFiles = await _client.GetStringAsync("https://reiyo.kibii.org/~/dump/reiyo-full-dump.txt");
- string[] urls = rawFiles.Split('\n').Where(f => !f.StartsWith("#") && !string.IsNullOrWhiteSpace(f)).Select(c => c.Trim()).ToArray();
- if (!Directory.Exists("out/"))
- {
- Directory.CreateDirectory("out/");
- }
- IList<Task> tasks = new List<Task>();
- string[][] uriLists = urls.SliceArray(threadSize).Select(u => u.ToArray()).ToArray();
- for (int i = 0; i < uriLists.Length; i++)
- {
- Task localTask = await Task.Factory.StartNew(async () => await DownloadAndSaveFiles(uriLists[i], i));
- tasks.Add(localTask);
- }
- Task.WaitAll(tasks.ToArray());
- Console.WriteLine("All images have been downloaded.");
- Console.ReadKey();
- }
- public static async Task DownloadAndSaveFiles(string[] urls, int batchId)
- {
- for (int i = 0; i < urls.Length; i++)
- {
- await Task.Run(async () =>
- {
- try
- {
- Uri uri = new Uri(urls[i]);
- string originalFileName = uri.AbsolutePath.Split('/')?.Last().Trim();
- if (string.IsNullOrWhiteSpace(originalFileName) || Guid.TryParse(originalFileName, out Guid res1))
- originalFileName = "no-file-name-available";
- string fileId = uri.AbsolutePath.Split('/').FirstOrDefault(u => Guid.TryParse(u, out Guid res2)) ?? Guid.NewGuid().ToString("N");
- HttpResponseMessage res = await _client.GetAsync(uri);
- string newFileName = originalFileName + "_" + fileId + "." + res.Content.Headers.ContentType.MediaType.Split('/').Last();
- string filePath = Path.Combine(Path.Combine(Directory.GetCurrentDirectory(), "out/"), newFileName);
- using (Stream outStr = File.Create(filePath))
- {
- Stream inStr = await res.Content.ReadAsStreamAsync();
- await inStr.CopyToAsync(outStr);
- }
- QueueOut("[B" + batchId + "] File DL'd (" + i.ToString() + "/" + urls.Length + "): '" + newFileName + "'");
- }
- catch
- {
- QueueOut("[B" + batchId + "] ERR (" + i.ToString() + "/" + urls.Length + ").");
- }
- });
- }
- }
- private static void QueueOut(string message)
- {
- Task.Run(() => _stack.Push(message));
- }
- }
- public static class EnumerbleExtensions
- {
- public static IEnumerable<IEnumerable<T>> SliceArray<T>(this T[] array, int groupSize)
- {
- List<T> locations = array.ToList();
- int nSize = array.Length / groupSize;
- for (int i = 0; i < locations.Count; i += nSize)
- {
- yield return locations.GetRange(i, Math.Min(nSize, locations.Count - i));
- }
- }
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement