train_2

import re
import collections
import pickle
import argparse
import os


def clear_the_string(source_str):
    spaces_to_zero = re.compile('\s')
    zero_to_spaces = re.compile('[0]')
    ignored_pro = re.compile('\W')
    without_nums = re.compile('\d')
    source_str = spaces_to_zero.sub("0", source_str)
    source_str = ignored_pro.sub("", source_str)
    source_str = zero_to_spaces.sub(" ", source_str)
    source_str = without_nums.sub("", source_str)
    return source_str


def stdin_to_file():
    filename = 'stdin_input.txt'
    open(filename, 'a').close()
    with open(filename, 'w') as f:
        while True:
            new_line = input()
            if (new_line == ""):
                break
            f.write(new_line + '\n')


parser = argparse.ArgumentParser()
parser.add_argument('way_to_training_files', '--input-dir', type=str,
                    help='The way to directory with training files')
parser.add_argument('way_to_model_file', "--model", type=str,
                    help='The way to model file')
parser.add_argument('lowercase', '--lc', action='store_true',
                    help='To lead to lowercase')
parser.add_argument('Help', '--help', action='store_true',
                    help='Show the utility reference')
args = parser.parse_args()


def make_model(way_to_train_files, way_to_model_file, lowercase):
    ignored = re.compile('\w+')

    filenames_list = os.listdir(args.way_to_train_files)
    dictionary = collections.OrderedDict()

    for filename in filenames_list:
        with open(filename, 'r', encoding="utf-8") as f:
            new_string = f.readline()
            if (lowercase):
                new_string = new_string.lower()
            new_string = clear_the_string(new_string)
            new_list = ignored.findall(new_string)
            while True:
                next_string = f.readline()
                if (next_string == ''):
                    break
                if (lowercase):
                    next_string = new_string.lower()
                next_string = clear_the_string(next_string)
                next_list = ignored.findall(next_string)
                while (len(next_list) == 0):
                    next_string = f.readline()
                    next_string = clear_the_string(next_string)
                    next_list = ignored.findall(next_string)
                new_list.append(next_list[0])
                counter = 1
                for k in range(len(new_list) - 1):
                    if new_list[k] not in dictionary:
                        dictionary.setdefault(new_list[k],
                                              collections.OrderedDict())
                    if new_list[counter] in dictionary[new_list[k]]:
                        dictionary[new_list[k]][new_list[counter]] += 1
                    else:
                        dictionary[new_list[k]].setdefault(new_list[counter], 1)
                    counter += 1
                new_list = next_list
    open('{}/model.pkl'.format(way_to_model_file), 'a').close()
    with open('{}/model.pkl'.format(way_to_model_file), 'wb') as pickle_file:
        pickle.dump(dictionary, pickle_file, 2)


if (args.Help):
    print('''Help :
   It is the utility for creation of model of the text.
   You can specify the directory with source files or use stdin:
   --input-dir <dir_name> . stdin use by default.
   Also you should specify a directory in which it
   is required to keep model:
   --model <dir_name> .
   If you want to get model in lowercase, you should use --lc.
   SUDDENLY use --help to see this text.''')
if (args.way_to_training_files is None):
    stdin_to_file()
    args.way_to_training_files = 'stdin_input.txt'
make_model(args.way_to_training_files, args.way_to_model_file, args.lowercase)