something_about_tokenizing

from xml.dom import minidom
from nltk.tokenize import WordPunctTokenizer

raw = '''<?xml version="1.0" ?>
<zAppointments reminder="15">
    <appointment>
        <description>1181251680</description>
        <description>Если я видел дальше других, то потому что стоял на плечах гигантов</description>
        <uid>040000008200E000</uid>
    </appointment>
</zAppointments>'''


def get_text(xml):
    xml_text = minidom.parseString(xml)
    text = xml_text.getElementsByTagName('description')[1].firstChild.data
    return text


def clear_text(text):
    symbols = [',', '.', ':', ';', '!']
    for s in symbols:
        text = text.replace(s, ' ')
    return text


def get_tokens(text):
    tok = WordPunctTokenizer()
    tokens = tok.tokenize(text)
    return tokens


def get_len_list(my_list):
    return len(set([w.lower() for w in my_list]))


text = get_text(raw)
# print(text)

tokens = get_tokens(text)
# print(tokens)

text2 = clear_text(text)
# print(text2)

tokens2 = get_tokens(text2)
# print(tokens2)

n = get_len_list(tokens2)
print(n)