Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- using System;
- using System.Collections.Concurrent;
- using System.Collections.Generic;
- using System.Linq;
- using System.Threading.Tasks;
- using Newtonsoft.Json;
- using SchoolInfoApp.Domain;
- using SchoolInfoApp.Domain.Models;
- using SchoolInfoApp.Framework.Translation;
- using SchoolInfoApp.Infrastructure.NHibernate.Interface;
- using SIA.BL.Modules.Translation.Commands;
- using SIA.Core;
- using SIA.Core.Cqrs.Interface;
- using SIA.Core.Translation;
- using SIA.Core.Utilities.Extensions;
- namespace SIA.API.Services.Translation
- {
- public class TranslationExecutor : ITranslationExecutor
- {
- private readonly ILogService _logService;
- private readonly ISessionSource _sessionSource;
- private readonly ICommandDispatcher _commandDispatcher;
- private readonly IGoogleTranslateService _googleTranslateService;
- private readonly IGoogleTranslationRepository _googleTranslationRepository;
- private const int WaitThreshold = 200; //ms
- private readonly object _taskLock = new object();
- private readonly object _saveCacheLock = new object();
- private readonly ConcurrentQueue<TranslationTextRequest> _pendingTranslations = new ConcurrentQueue<TranslationTextRequest>();
- private Task<Dictionary<string, IDictionary<string, string>>> _translationTask;
- public TranslationExecutor(
- ILogService logService,
- ISessionSource sessionSource,
- ICommandDispatcher commandDispatcher,
- IGoogleTranslateService googleTranslateService,
- IGoogleTranslationRepository googleTranslationRepository)
- {
- _logService = logService;
- _sessionSource = sessionSource;
- _commandDispatcher = commandDispatcher;
- _googleTranslateService = googleTranslateService;
- _googleTranslationRepository = googleTranslationRepository;
- }
- public IDictionary<string, string> GetTranslations(OrgTranslationRequest request)
- {
- _logService.Info("Translation Executor called");
- var trans = request.TextsToTranslate.Select(x => new TranslationTextRequest(request.Language, x));
- foreach (var tran in trans)
- {
- _pendingTranslations.Enqueue(tran);
- }
- lock (_taskLock)
- {
- StartTranslationTask();
- }
- IDictionary<string, IDictionary<string, string>> allTranslationsDictionary = null;
- _translationTask.ContinueWith(x =>
- {
- allTranslationsDictionary = x.Result;
- _logService.Info("Translation ended. Result: " + JsonConvert.SerializeObject(allTranslationsDictionary));
- lock (_taskLock)
- {
- ClearTranslationTask();
- }
- }).Wait();
- var requestedTranslations = request.TextsToTranslate.ToDictionary(x => x, x => x);
- var allTranslationKeys = requestedTranslations.Keys.ToArray();
- if (allTranslationsDictionary == null)
- {
- return requestedTranslations;
- }
- if (allTranslationsDictionary.ContainsKey(request.Language))
- {
- var translations = allTranslationsDictionary[request.Language];
- var count = 0;
- var caches = new HashSet<GoogleTranslationCache>();
- foreach (var key in allTranslationKeys)
- {
- if (translations.ContainsKey(key))
- {
- var translation = translations[key];
- requestedTranslations[key] = translation;
- count += translation.Length;
- caches.Add(new GoogleTranslationCache
- {
- Id = Guid.NewGuid(),
- Hash = key.GetSha1Hash(),
- Language = request.Language,
- Translation = translation
- });
- }
- }
- SaveTranslationCaches(caches);
- if (count > 0)
- {
- LogCharacterCount(request.OrgId, request.Language, count);
- }
- }
- _logService.Info("Done: requestedTranslations: " + JsonConvert.SerializeObject(requestedTranslations));
- return requestedTranslations;
- }
- private void ClearTranslationTask()
- {
- _translationTask = null;
- }
- private void StartTranslationTask()
- {
- if (_translationTask != null) return;
- _translationTask = Task.Run(() =>
- {
- var allTranslationsDictionary = new Dictionary<string, IDictionary<string, string>>();
- var translationsDictionary = new Dictionary<string, HashSet<string>>();
- TranslationTextRequest request;
- while (_pendingTranslations.TryDequeue(out request))
- {
- _logService.Info("Dequeue request: " + _pendingTranslations.Count);
- if (!translationsDictionary.ContainsKey(request.Language))
- {
- translationsDictionary.Add(request.Language, new HashSet<string>());
- }
- translationsDictionary[request.Language].Add(request.TextToTranslate);
- if (_pendingTranslations.Count == 0)
- {
- _logService.Info($"Queue is empty. Waitnig for {WaitThreshold}ms for more incoming requests");
- Task.Delay(WaitThreshold).Wait();
- }
- }
- foreach (var language in translationsDictionary.Keys)
- {
- var sourceTexts = translationsDictionary[language].ToArray();
- var translations = sourceTexts.ToDictionary(x => x, x => x);
- string[] googleTranslations;
- try
- {
- _logService.Info("Calling Google Translate API");
- googleTranslations = _googleTranslateService.GetTranslations(sourceTexts, language);
- }
- catch (Exception e)
- {
- _logService.Error("Error during calling Google Translator Service: " + e);
- continue;
- }
- for (var i = 0; i < googleTranslations.Length; i++)
- {
- var original = sourceTexts[i];
- var translation = googleTranslations[i];
- translations[original] = translation;
- }
- allTranslationsDictionary.Add(language, translations);
- }
- return allTranslationsDictionary;
- });
- }
- private void LogCharacterCount(int orgId, string language, int count)
- {
- var org = _sessionSource.GetContextualSession().Load<Organization>(orgId);
- using (var session = _sessionSource.CreateStatelessSession())
- {
- session.SetBatchSize(1000);
- using (var tran = session.BeginTransaction())
- {
- var charCount = new GoogleTranslationCharacterCount
- {
- Organization = org,
- CharacterCount = count,
- CreatedAt = SystemTime.Now(),
- LanguageCode = language
- };
- session.Insert(charCount);
- tran.Commit();
- }
- }
- }
- private void SaveTranslationCaches(ICollection<GoogleTranslationCache> caches)
- {
- lock (_saveCacheLock)
- {
- var unsavedCaches = caches.Where(x => !_googleTranslationRepository.ContainsItemInCache(x.Hash, x.Language)).ToList();
- _commandDispatcher.Dispatch(new BulkInsertGoogleTranslationCaches(unsavedCaches));
- _googleTranslationRepository.AddItems(caches, false);
- }
- }
- private void SaveTranslationCache(string language, string original, string translation)
- {
- lock (_saveCacheLock)
- {
- var hash = original.GetSha1Hash();
- if (_googleTranslationRepository.ContainsItemInCache(hash, language)) return;
- var translationCache = new GoogleTranslationCache
- {
- Id = Guid.NewGuid(),
- Language = language,
- Hash = hash,
- Translation = translation
- };
- _googleTranslationRepository.AddTranslation(translationCache);
- }
- }
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement