Tatoeba Language Study

#! /usr/bin/python3

from subprocess import Popen
import os
import sys
import time
import random
import requests
import urllib.parse
import re
import alsaaudio
import audioop
import speech_recognition as sr
import _thread
import wave
import pickle

SAVE_FILE = "study.save"
READ_WPM = 300
STUDY_SESSION_SIZE = 5
tmpAudioFile = "/tmp/textToSpeech.mp3"
tmpMicrophoneRecording = "/tmp/microphoneRecording.wav"

VOCAB_STAT_NEW = 0
VOCAB_STAT_GOOD = 1
VOCAB_STAT_GREAT = 2

VOCAB_STAT_GOOD_LEVEL = 5
VOCAB_STAT_GREAT_LEVEL = 10

class color:
   PURPLE = '\033[95m'
   CYAN = '\033[96m'
   DARKCYAN = '\033[36m'
   BLUE = '\033[94m'
   GREEN = '\033[92m'
   YELLOW = '\033[93m'
   RED = '\033[91m'
   BOLD = '\033[1m'
   UNDERLINE = '\033[4m'
   END = '\033[0m'

if len(sys.argv) < 5:
    print("Usage: program [language code] [word list] [target language csv] [native language.csv] [links csv]")
    print("Example: program nl nl.txt nld.csv eng.csv links_nld_eng.csv")
    exit()

cards = []
words = []

languageCode = sys.argv[1]
wordFile = sys.argv[2]
targetFile = sys.argv[3]
nativeFile = sys.argv[4]
linksFile = sys.argv[5]

def clearScreen():
    os.system('clear')

class Card:
    def __init__(self, sentence):
        self.sentence = sentence
        self.translations = []

    def __str__(self):
        rep = color.BOLD + "[SENTENCE] " + color.END + self.sentence + "\n"
        for translation in self.translations:
            rep += color.BOLD + "[TRANSLATION] " + color.END + translation + "\n"
        return rep.strip()

    def say(self):
        textToSpeech(self.sentence, languageCode)

    def read(self):
        speedRead(self.sentence)

    def getRelated(self):
        for x in range(100): # just so it doesn't look forever
            words = superStrip(self.sentence).split(" ")
            random.shuffle(words)
            while True:
                deck = searchCardsList(words)
                if self in deck: deck.remove(self)
                if len(deck):
                    result = random.choice(deck)
                    return result
                else:
                    words.pop()
                if len(words) == 0: break
        print("It took too long to find a related sentence!")

class Word:
    def __init__(self, text, freq):
        self.text = text
        self.freq = freq
        if self.text in wordStudySave:
            self.studied = wordStudySave[self.text]
        else:
            self.studied = 0
        self.freqPercent = None

    def say(self):
        textToSpeech(self.text, languageCode)

    def study(self):
        self.studied += 1
        wordStudySave[self.text] = self.studied
        saveData()

    def getVocabLevel(self):
        if self.studied >= VOCAB_STAT_GREAT_LEVEL:
            return VOCAB_STAT_GREAT
        elif self.studied >= VOCAB_STAT_GOOD_LEVEL:
            return VOCAB_STAT_GOOD
        else:
            return VOCAB_STAT_NEW

    def getVocabLevelColor(self):
        level = self.getVocabLevel()
        if level == VOCAB_STAT_GREAT:
            return color.GREEN
        elif level == VOCAB_STAT_GOOD:
            return color.YELLOW
        else:
            return color.RED

    def __str__(self):
        rep = color.BOLD + "[WORD] " + color.END + self.text + "\n"
        rep += color.BOLD + "[FREQUENCY] " + color.END + self.freq + "/" + wordCountString + " (" + self.freqPercent + "%)\n"
        rep += color.BOLD + "[STUDIED] " + color.END + str(self.studied) + "\n"
        return rep.strip()

    def printVocab(self):
        rep = color.BOLD + "[WORD] " + self.getVocabLevelColor() + self.text + color.END + "\n"
        rep += color.BOLD + "[STUDIED] " + color.END + str(self.studied) + "\n"
        print(rep)

def load():
    print("Loading sentences...")
    sentences = {}
    with open(targetFile) as f:
        for line in f:
            parts = line.strip().split("\t")
            id = parts[0]
            text = parts[2]
            card = Card(text)
            cards.append(card)
            sentences[id] = card
    print("Loaded " + str(len(sentences)) + " sentences.")
    print("Loading links...")
    reverseLinks = {}
    with open(linksFile) as f:
        for line in f:
            parts = line.strip().split("\t")
            id = parts[0]
            linkId = parts[1]
            reverseLinks[linkId] = id
    print("Loaded " + str(len(reverseLinks)) + " links.")
    print("Loading translations...")
    total = 0
    with open(nativeFile) as f:
        for line in f:
            parts = line.strip().split("\t")
            id = parts[0]
            text = parts[2]
            try:
                sentenceId = reverseLinks[id]
                sentences[sentenceId].translations.append(text)
                total += 1
            except KeyError:
                pass
    print("Loaded " + str(total) + " translations.")
    print("Loading words...")
    with open(wordFile) as f:
        for line in f:
            parts = line.strip().split(" ")
            word = parts[0]
            freq = parts[1]
            words.append(Word(word, freq))
    print("Loaded " + str(len(words)) + " words.")
    print("Making precalculations...")
    global wordCountString
    wordCountString = str(len(words))
    for word in words:
        word.freqPercent = str(int(1000000.0 * int(word.freq) / len(words)) / 10000)
    input("Press enter when ready.")

def inputNumber(prompt=""):
    inp = input(prompt)
    try:
        return int(inp)
    except:
        return None

def inputBoolean(prompt, default=None):
    if default == True:
        c = "(Y/n)"
    elif default == False:
        c = "(y/N)"
    else:
        c = "(y/n)"
    inp = None
    while inp != "y" and inp != "n" and (inp != "" or default == None):
        inp = input(prompt + " " + c + " ").strip().lower()
    if inp == "":
        inp = default
    else:
        inp = inp == "y"
    return inp

def inputChoice(prompt, choices):
    keys = list(choices.keys())
    sorted(keys)
    inp = None
    while inp == None:
        print(color.BOLD + prompt + color.END)
        i = 0
        for choice in keys:
            i += 1
            print(str(i) + ") " + choice)
        inp = inputNumber("> ")
        if inp == None:
            print("That is not a number!")
        elif inp < 1 or inp > len(keys):
            print("That was not one of the choices!")
            inp = None
    return choices[keys[inp - 1]]

MENU_OPTION_NEW = 0
MENU_OPTION_REVIEW = 1
MENU_OPTION_WORD = 2
MENU_OPTION_MORPH = 3
MENU_OPTION_RANDOM = 4
MENU_OPTION_VOCAB = 5
MENU_OPTION_WORDS = 6
MENU_OPTION_SENTENCES = 7
MENU_OPTION_SEARCH = 8
MENU_OPTION_QUIT = 9
MENU_OPTION_TRANSCRIBE = 10
MENU_OPTION_WORD_TRANSCRIBE = 11

def inputMenuOption():
    return inputChoice("Select a mode:", {  "New Words":                        MENU_OPTION_NEW,
                                            "Review Words":                     MENU_OPTION_REVIEW,
                                            "Word Practice":                    MENU_OPTION_WORD,
                                            "Sentence Morph":                   MENU_OPTION_MORPH,
                                            "Random Sentences":                 MENU_OPTION_RANDOM,
                                            "Transcribe Sentences":             MENU_OPTION_TRANSCRIBE,
                                            "Transcribe Sentences With Word":   MENU_OPTION_WORD_TRANSCRIBE,
                                            "Vocabulary Stats":                 MENU_OPTION_VOCAB,
                                            "List Words":                       MENU_OPTION_WORDS,
                                            "List Sentences":                   MENU_OPTION_SENTENCES,
                                            "Search Sentences":                 MENU_OPTION_SEARCH,
                                            "Quit":                             MENU_OPTION_QUIT
                                            })

def welcome():
    print("Welcome to Chunk Study!")

def bye():
    print("Goodbye!")

def superStrip(s):
    s = s.lower()
    s = re.sub(r'([^\s\w]|_)+', '', s)
    s = re.sub(' +',' ', s)
    return s

def compare(a, b):
    return superStrip(a) == superStrip(b)

def compareList(a, items):
    for b in items:
        if compare(a, b): return True
    return False

def getRandomCard():
    card = None
    while card == None or len(card.translations) == 0:
        card = random.choice(cards)
    return card

def transcribe(card):
    while True:
        card.say()
        inp = input(color.BOLD + "Transcribe:" + color.END + " ")
        if inp: return compare(inp, card.sentence)

def read(card):
    while True:
        input("Press enter when ready to read.")
        card.read()
        inp = input(color.BOLD + "Copy:" + color.END + " ")
        if inp: return compare(inp, card.sentence)

def speak(card):
    print(color.BOLD + "Say:" + color.END + " " + card.sentence)
    input("Press enter when ready.")
    transcriptions = speechRecognition()
    if transcriptions == None or len(transcriptions) == 0: return False
    for t in transcriptions:
        if compare(t["text"], card.sentence):
            return True
    return transcriptions[0]["text"]

def translate(card):
    print(color.BOLD + "Translate:" + color.END + " " + card.sentence)
    if compareList(input("> "), card.translations):
        return True
    else:
        return False

def reverseTranslate(card):
    print(color.BOLD + "Translate:" + color.END + " " + card.translations[0])
    if compare(input("> "), card.sentence):
        return True
    else:
        return False

def studyCard(card, word=None):
    clearScreen()
    if word: word.say()
    if reverseTranslate(card):
        print(color.GREEN + color.BOLD + "Awesome!" + color.END)
    else:
        print(color.RED + color.BOLD + "Incorrect: " + color.END + card.sentence)

    input("Press enter to continue.")
    clearScreen()
    if word: word.say()
    if read(card):
        print(color.GREEN + color.BOLD + "Exactly!" + color.END)
    else:
        print(color.RED + color.BOLD + "Sorry: " + color.END + card.sentence)

    input("Press enter to continue.")
    clearScreen()
    if transcribe(card):
        print(color.GREEN + color.BOLD + "Correct!" + color.END)
    else:
        print(color.RED + color.BOLD + "Wrong: " + color.END + card.sentence)

    input("Press enter to continue.")
    clearScreen()
    if word: word.say()
    result = speak(card)
    if result == False:
        print(color.RED + color.BOLD + "Couldn't recognize you!" + color.END)
    elif result == True:
        print(color.GREEN + color.BOLD + "Good!" + color.END)
    else:
        print(color.RED + color.BOLD + "Not quite!" + color.END + "  It sounded like you said \"" + result + "\"")

    input("Press enter to continue.")
    clearScreen()
    if word: word.say()
    if translate(card):
        print(color.GREEN + color.BOLD + "Nice!" + color.END)
    else:
        print(color.RED + color.BOLD + "Nope: " + color.END + card.translations[0])

    print()
    print(card)
    print()
    input("Press enter to continue.")

def listSentences():
    for card in cards:
        print(card)
        print()
    print("Total: " + str(len(cards)))
    input("Press enter to return.")

def listWords():
    for word in words:
        print(word)
        print()
    print("Total: " + str(len(words)))
    input("Press enter to return.")

def searchSentences():
    s = input("Enter a search string: ").strip().lower()
    for card in cards:
        if s in card.sentence.lower():
            print(card)
            print()
    input("Press enter to return.")

def searchCards(word):
    deck = []
    for card in cards:
        if len(card.translations) and word.lower() in card.sentence.lower():
            deck.append(card)
    return deck

def searchCardsList(words):
    deck = []
    for card in cards:
        if len(card.translations):
            contains = True
            for word in words:
                if not word.lower() in card.sentence.lower():
                    contains = False
                    break
            if contains: deck.append(card)
    return deck

def getWord(s):
    for word in words:
        if s.lower() == word.text.lower():
            return word

def vocabStats():
    unknown = 0
    new = 0
    good = 0
    great = 0
    for word in words:
        if word.studied:
            word.printVocab()
        if word.studied == 0:                       unknown += 1
        elif word.studied < VOCAB_STAT_GOOD_LEVEL:  new += 1
        elif word.studied < VOCAB_STAT_GREAT_LEVEL: good += 1
        else:                                       great += 1
    print(color.BOLD + "[UNKNOWN] " + color.END + str(unknown) + color.END)
    print(color.BOLD + color.RED + "[BEGINNER] " + color.END + color.RED + str(new) + color.END)
    print(color.BOLD + color.YELLOW + "[INTERMEDIATE] " + color.END + color.YELLOW + str(good) + color.END)
    print(color.BOLD + color.GREEN + "[ADVANCED] " + color.END + color.GREEN + str(great) + color.END)
    input("Press enter to return.")

def transcription():
    while True:
        card = getRandomCard()
        while True:
            clearScreen()
            if transcribe(card):
                print(color.GREEN + color.BOLD + "Correct!" + color.END)
            else:
                print(color.RED + color.BOLD + "Wrong: " + color.END + card.sentence)
            print()
            print(card)
            print()
            choice = inputChoice("What would you like to do next?", {
                                    "Repeat Sentence":  0,
                                    "Next Sentence":    1,
                                    "Quit":             2
                                    })
            if choice != 0: break
        if choice != 1: break

def transcribeWordPractice():
    while True:
        clearScreen()
        while True:
            while True:
                w = input("Enter a word to study: ")
                word = getWord(w)
                if word == None:
                    print("That is an unknown word!")
                else:
                    print()
                    word.printVocab()
                    break
            deck = searchCards(word.text)
            random.shuffle(deck)
            if len(deck):
                print(str(len(deck)) + " sentences found containing that word.")
                if inputBoolean("Is this okay?", True): break
            else:
                print("No sentences found containing that word!")
        if len(deck):
            while True:
                for card in deck:
                    while True:
                        clearScreen()
                        if transcribe(card):
                            print(color.GREEN + color.BOLD + "Correct!" + color.END)
                        else:
                            print(color.RED + color.BOLD + "Wrong: " + color.END + card.sentence)
                        print()
                        print(card)
                        word.study()
                        print()
                        word.printVocab()
                        word.say()
                        choice = inputChoice("What would you like to do next?", {
                                                "Repeat Sentence":  0,
                                                "Next Sentence":    1,
                                                "Change Word":      2,
                                                "Quit":             3
                                                })
                        if choice != 0: break
                    if choice != 1: break
                if choice != 1: break
                if not inputBoolean("That is all of the sentences for this word. Would you like to repeat them?", False): break
        else:
            print("There are no sentences containing that word.")
        if choice == 3:
            break
        elif choice != 2:
            if not inputBoolean("Would you like to study a different word?", True): break

def studySession(words):
    print("Words for this study session:")
    print()
    for word in words:
        word.printVocab()
    input("Press enter to continue.")
    for word in words:
        clearScreen()
        print("Now studying:")
        print()
        word.printVocab()
        word.say()
        input("Press enter to continue.")
        deck = searchCards(word.text)
        if word.studied < VOCAB_STAT_GOOD_LEVEL:
            goal = VOCAB_STAT_GOOD_LEVEL
        elif word.studied < VOCAB_STAT_GREAT_LEVEL:
            goal = VOCAB_STAT_GREAT_LEVEL
        else:
            goal = word.studied + 1
        while word.studied < goal:
            card = random.choice(deck)
            while True:
                studyCard(card, word)
                word.study()
                print()
                word.printVocab()
                choice = inputChoice("What would you like to do next?", {
                                        "Repeat Sentence":  0,
                                        "Next Sentence":    1,
                                        "Quit":             2
                                        })
                if choice != 0: break
            if choice != 1: break
        if choice != 1: break

def newWords():
    clearScreen()
    studyWords = []
    for word in words:
        if word.studied == 0:
            studyWords.append(word)
        if len(studyWords) >= STUDY_SESSION_SIZE: break
    if len(studyWords) == 0:
        print("There are no more new words to study!")
        print("Review them instead.")
        input("Press enter to return.")
    else:
        studySession(studyWords)

def reviewWords():
    introducedWords = []
    for word in words:
        if word.studied > 0:
            introducedWords.append(word)
    worstWords = sorted(introducedWords, key=lambda word: word.studied, reverse=False)[0:STUDY_SESSION_SIZE]
    if len(worstWords) == 0:
        print("There are no words to study!")
        print("Learn new words instead.")
        input("Press enter to return.")
    else:
        studySession(worstWords)

def wordPractice():
    while True:
        clearScreen()
        while True:
            while True:
                w = input("Enter a word to study: ")
                word = getWord(w)
                if word == None:
                    print("That is an unknown word!")
                else:
                    print()
                    word.printVocab()
                    break
            deck = searchCards(word.text)
            random.shuffle(deck)
            if len(deck):
                print(str(len(deck)) + " sentences found containing that word.")
                if inputBoolean("Is this okay?", True): break
            else:
                print("No sentences found containing that word!")
        if len(deck):
            while True:
                for card in deck:
                    while True:
                        studyCard(card, word)
                        word.study()
                        print()
                        word.printVocab()
                        choice = inputChoice("What would you like to do next?", {
                                                "Repeat Sentence":  0,
                                                "Next Sentence":    1,
                                                "Change Word":      2,
                                                "Quit":             3
                                                })
                        if choice != 0: break
                    if choice != 1: break
                if choice != 1: break
                if not inputBoolean("That is all of the sentences for this word. Would you like to repeat them?", False): break
        else:
            print("There are no sentences containing that word.")
        if choice == 3:
            break
        elif choice != 2:
            if not inputBoolean("Would you like to study a different word?", True): break

def sentenceMorph():
    card = getRandomCard()
    while True:
        while True:
            studyCard(card)
            choice = inputChoice("What would you like to do next?", {
                                    "Repeat Sentence":  0,
                                    "Next Sentence":    1,
                                    "Quit":             2
                                    })
            if choice != 0: break
        if choice != 1: break
        card = card.getRelated()

def randomSentence():
    while True:
        card = getRandomCard()
        while True:
            studyCard(card)
            choice = inputChoice("What would you like to do next?", {
                                    "Repeat Sentence":  0,
                                    "Next Sentence":    1,
                                    "Quit":             2
                                    })
            if choice != 0: break
        if choice != 1: break

def main():
    clearScreen()
    welcome()
    while True:
        choice = inputMenuOption()
        if choice == MENU_OPTION_NEW:
            newWords()
        elif choice == MENU_OPTION_REVIEW:
            reviewWords()
        elif choice == MENU_OPTION_WORD:
            wordPractice()
        elif choice == MENU_OPTION_WORD_TRANSCRIBE:
            transcribeWordPractice()
        elif choice == MENU_OPTION_MORPH:
            sentenceMorph()
        elif choice == MENU_OPTION_RANDOM:
            randomSentence()
        elif choice == MENU_OPTION_TRANSCRIBE:
            transcription()
        elif choice == MENU_OPTION_VOCAB:
            vocabStats()
        elif choice == MENU_OPTION_WORDS:
            listWords()
        elif choice == MENU_OPTION_SENTENCES:
            listSentences()
        elif choice == MENU_OPTION_SEARCH:
            searchSentences()
        elif choice == MENU_OPTION_QUIT:
            break
        else:
            break
        clearScreen()
    bye()

def setup():
    global microphone
    global audioInput

    clearScreen()

    cards = alsaaudio.cards()
    choices = {}
    for card in cards:
        choices[card] = card
    microphone = inputChoice("Select a microphone source:", choices)

    audioInput = alsaaudio.PCM(alsaaudio.PCM_CAPTURE, alsaaudio.PCM_NONBLOCK, microphone)
    audioInput.setchannels(1)
    audioInput.setrate(44100)
    audioInput.setformat(alsaaudio.PCM_FORMAT_S16_LE)
    audioInput.setperiodsize(160)

def getFile(url):
    return requests.Session().get(url).content

audioProcess = None

def playAudioFile(path):
    global audioProcess
    fnull = open(os.devnull, "w+")
    if audioProcess != None: audioProcess.terminate()
    audioProcess = Popen(["cvlc", path, "vlc://quit"], stdin=fnull, stdout=fnull, stderr=fnull)

def exportFile(path, data):
    f = open(path, "wb")
    f.write(data)
    f.close()

def exportWav(path, data):
    f = wave.open(path, "w")
    f.setparams((1, 2, 44100, 0, 'NONE', 'not compressed'))
    f.writeframes(data)
    f.close()

def textToSpeech(text, language="en"):
    data = getFile("http://translate.google.com/translate_tts?tl=" + language + "&q=" + text.replace(" ", "%20"))
    exportFile(tmpAudioFile, data)
    playAudioFile(tmpAudioFile)

def getMicrophoneData():
    global micData
    micData = b""
    audioInput.read()
    while recording:
        length, data = audioInput.read()
        micData += data
        time.sleep(0.001)

def recordAudio():
    global recording
    recording = True
    _thread.start_new_thread(getMicrophoneData,())
    input(color.BOLD + color.RED + "[RECORDING] " + color.END + "Press enter to stop...")
    print(color.BOLD + "Finished recording." + color.END)
    recording = False
    exportWav(tmpMicrophoneRecording, micData)
    return tmpMicrophoneRecording

def speechRecognition():
    rec = sr.Recognizer(language=languageCode)
    audio = recordAudio()
    playAudioFile(audio)
    with sr.WavFile(audio) as source:
        print("Analyzing audio...")
        audio = rec.listen(source)
    try:
        return rec.recognize(audio, True)
    except LookupError:
        return None

def speedRead(string, wpm=READ_WPM):
    string = re.sub(' +',' ', string)
    words = string.split(" ")
    clearScreen()
    for word in words:
        print(word)
        time.sleep(1 / (wpm / 60.0))
        clearScreen()

def saveData():
    f = open(SAVE_FILE, "wb")
    pickle.dump(wordStudySave, f)
    f.close()

def loadData():
    global wordStudySave
    try:
        f = open(SAVE_FILE, "rb")
        wordStudySave = pickle.load(f)
        f.close()
    except FileNotFoundError:
        wordStudySave = {}
        print("Save file not found.")

loadData()
setup()
load()
main()