def get_non_empty_lines(filename):
    with open(filename) as fp:
        for line in fp:
            stripped_line = line.strip()
            if stripped_line:
                yield stripped_line

for line in get_non_empty_lines('tweets.json'):
    tweet = json.loads(line)
    tokens = preprocess(tweet['text'])