from translate import Translator
import re
import codecs
import glob
import time
import os
import os.path
#from nltk.tokenize import sent_tokenize
#import nltk

result = []

def process_one_file(filename):
    f = open("dialog/"+filename+"",'r')
    #lines = f.readlines()
    data0 = f.read()
    data = data0

    f = re.findall(r'{\d+}{.*?}{([\w\W]*?)}', data,re.MULTILINE)
    all_lines = set()
    for txt in f:
        all_lines.add(txt)

    all_lines = list(all_lines)
    all_lines = sorted(all_lines, key=len, reverse=True)

    for txt in all_lines:
        if len(txt)>500:
            continue
        translation = ""
        while True:
            translator = Translator(to_lang="ru")
            try:
                translation = translator.translate(txt)
            except:
                print("network error!")
                time.sleep(3)
                continue

            if "MYMEMORY WARNING:" in translation:
                print("restart tor!")
                time.sleep(3)
                continue

            print(translation)
            break

        data = data.replace(txt,translation)

    f = open("dialog_new/"+filename+"",'wb')
    data2 = bytes(data, "cp866", errors='ignore')

    f.write(data2)

files = glob.glob("dialog/*.msg")

for filename in files:
    filename  = filename.replace("\\","/")
    filename = filename.split("/")[1]
    print(filename)
    print("=============================================")
    if os.path.exists("dialog_new/"+filename):
        continue
    process_one_file(filename)