import sqlite3
import os.path
import os

os.chdir('C:/Users/Trevor/Desktop')
conn = sqlite3.connect("a.db")
cursor = conn.cursor()
all_words = []

for file in os.listdir('gutenberg'):
    print('processing {}'.format(file))
    with open(os.path.join('gutenberg', file), 'r', encoding='utf-8') as file_in:
        all_words.extend(file_in.read().split())

print('{} words'.format(len(all_words)))
freq_dict = {}
for word in all_words:
    if word in freq_dict:
        freq_dict[word] += 1
    else:
        freq_dict[word] = 1

for word, freq in freq_dict.items():
    cursor.execute('INSERT INTO X VALUES(?, ?)', (word, freq))
conn.commit()
conn.close()