Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- using stigzler.Screenscraper.Data.Models;
- using stigzler.Screenscraper.Enums;
- using stigzler.Screenscraper.Helpers;
- using stigzler.Screenscraper.Services;
- using System;
- using System.Collections.Generic;
- using System.Diagnostics;
- using System.Linq;
- using System.Net;
- using System.Net.Http;
- using System.Threading;
- using System.Threading.Tasks;
- namespace stigzler.Screenscraper
- {
- public class GetData
- {
- #region Properties
- // Public Members
- public Credentials Credentials { get; set; }
- public ApiServerParameters ApiParameters { get; set; }
- public HttpClient HttpClient { get; set; }
- public MetadataOutput MetadataOutputFormat
- {
- get { return metadataOutputFormat; }
- set
- {
- metadataOutputFormat = value;
- urlBuilder.MetadataOutputType = metadataOutputFormat;
- }
- }
- public int UserThreads
- {
- get { return userThreads; }
- set
- {
- if (value > Data.Constants.MaxApiThreads)
- { throw new ArgumentException("UserThreads cannot exceed the maximum for the Screenscraper API server: " + Data.Constants.MaxApiThreads); }
- else
- {
- userThreads = value;
- SetNumberApiThreads();
- }
- }
- }
- // Any Property private members
- private int userThreads = 1;
- private MetadataOutput metadataOutputFormat = MetadataOutput.xml;
- #endregion
- // Class level private vars
- private ApiUrlBuilder urlBuilder;
- private GetDataService getDataService;
- public GetData(Credentials credentials, ApiServerParameters apiServerParameters, HttpClient httpClient, int userThreads)
- {
- // Set Public Properties
- Credentials = credentials;
- ApiParameters = apiServerParameters;
- HttpClient = httpClient;
- this.userThreads = userThreads;
- // Setup getDataService
- getDataService = new GetDataService(apiServerParameters, UserThreads, HttpClient);
- urlBuilder = new ApiUrlBuilder(Credentials, ApiParameters);
- }
- private void SetNumberApiThreads()
- {
- ServicePointManager.FindServicePoint(new Uri(ApiParameters.HostAddress)).ConnectionLimit = userThreads;
- }
- public async Task<ApiGetOutcome> GetApiServerInfo()
- {
- List<string> urlList = new List<string> { urlBuilder.Build(ApiQueryType.ServerInfo) };
- var result = await Task.Run(() => getDataService.GetUrlListData(urlList));
- return result.First(); // First, because this operation only has one entry
- }
- public async Task<ApiGetOutcome> GetUserInfo()
- {
- List<string> urlList = new List<string> { urlBuilder.Build(ApiQueryType.UserInfo) };
- var result = await Task.Run(() => getDataService.GetUrlListData(urlList));
- return result.First(); // First, because this operation only has one entry
- }
- public async Task<List<ApiGetOutcome>> GetGameInfo(int systemID, List<string> romNames)
- {
- List<QueryParameter> parameters = new List<QueryParameter>();
- List<string> urlList = new List<string>();
- foreach (string romname in romNames)
- {
- parameters.Clear();
- parameters.Add(new QueryParameter() { Parameter = ApiQueryParameter.RomFilename, Value = romname });
- parameters.Add(new QueryParameter() { Parameter = ApiQueryParameter.SystemID, Value = systemID.ToString() });
- urlList.Add(urlBuilder.Build(ApiQueryType.GameInfo, parameters));
- }
- List<ApiGetOutcome> apiGetOutcomes = new List<ApiGetOutcome>();
- ServicePointManager.DefaultConnectionLimit = userThreads;
- ServicePointManager.FindServicePoint(new Uri(ApiParameters.HostAddress)).ConnectionLimit = userThreads;
- Stopwatch SW = Stopwatch.StartNew();
- ThreadPool.SetMinThreads(100, 100);
- // APPROACH 1: Parallel.ForEach
- //Parallel.ForEach(urlList, new ParallelOptions() { MaxDegreeOfParallelism = userThreads }, url =>
- //{
- // var getResult = HttpClient.GetStringAsync(url);
- // string json = getResult.Result;
- // apiGetOutcomes.Add(new ApiGetOutcome() { Data = json, Url = url });
- // Debug.WriteLine("Rate: " + apiGetOutcomes.Count / SW.Elapsed.TotalSeconds);
- // Debug.WriteLine(apiGetOutcomes.Count);
- //});
- // OUTCOME 1: steady stream of 1/s
- // APPROACH 2: Task/Semaphore mix
- //var tasks = new List<Task>();
- //foreach (string url in urlList)
- //{
- // tasks.Add(PerformApiGet(url, HttpClient));
- //}
- //await Task.WhenAll(tasks);
- // OUTCOME 2: steady stream of 1/s
- // APPROACH 3: Parallel.ForEach using WebClient
- //Parallel.ForEach(urlList, new ParallelOptions() { MaxDegreeOfParallelism = userThreads }, url =>
- //{
- // using (WebClient wc = new WebClient())
- // {
- // var result = wc.DownloadString(new Uri(url));
- // apiGetOutcomes.Add(new ApiGetOutcome() { Data = result, Url = url });
- // Debug.WriteLine("Rate: " + apiGetOutcomes.Count / SW.Elapsed.TotalSeconds);
- // Debug.WriteLine(apiGetOutcomes.Count);
- // }
- //});
- // OUTCOME 3: steady stream of 5/s
- // APPROACH 4: Charlieface's approach
- var DlTasks = new List<Task>();
- foreach (string url in urlList)
- {
- if (DlTasks.Count >= UserThreads)
- {
- var finished = await Task.WhenAny(DlTasks);
- DlTasks.Remove(finished);
- }
- DlTasks.Add(PerformApiGet(url, HttpClient));
- }
- await Task.WhenAll(DlTasks);
- return apiGetOutcomes;
- }
- private static readonly SemaphoreSlim semaphore = new SemaphoreSlim(7);
- private async Task PerformApiGet(string url, HttpClient httpClient)
- {
- await semaphore.WaitAsync();
- try
- {
- var response = await httpClient.GetStringAsync(url);
- Debug.WriteLine("Downloaded: " + response.Length);
- }
- finally
- {
- semaphore.Release();
- }
- }
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement