regex = re.compile('[%s]' % re.escape(string.punctuation))

token_without_punctuation = []

for x in tokenized_doc1:
    y = []
    for token in x:
        tokens = regex.sub(ur"p{P}(?<![-.])", "", token)
          y.append(tokens)

    token_without_punctuation.append(y)

print(token_without_punctuation)

[['The', 'intelligent', 'directory', 'enquiry', 'assistant', '(', 'YPA', ')', 'project', 'is', 'an', 'example', '(', 'going', 'back', 'quite', 'a', 'few', 'years', 'now', '...', ')', 'where', 'the', 'extraction', 'of', 'information', 'from', 'partially', 'structured', 'data', 'together', 'with', 'engineering', 'issues', 'played', 'major', 'roles', 'in', 'making', 'the', 'YPA', 'a', 'usable', 'online', 'system', '.'], ['I', 'am', 'developing', 'techniques', 'that', 'allow', 'the', 'extraction', 'of', 'conceptual', 'information', 'from', 'document', 'collections', 'and', 'the', 'utilization', 'of', 'such', 'knowledge', 'in', 'retrieval', 'tasks', '.'], ['The', 'type', 'of', 'documents', 'can', 'range', 'from', 'Web', 'pages', 'to', 'newspaper', 'articles', 'or', 'other', 'forms', 'of', 'vaguely/partially', 'structured', 'data', '.']]