Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # Define pre-processing function
- def preprocess_with_stopwords_with_lemmatization(text):
- # Lowercase
- text = text.lower()
- # Replace "5G" or "5 G" with "5 G"
- text = re.sub(r'\b(5\s?)g\b', '5g', text)
- # Strip white space
- text = text.strip()
- # Remove spaces between specific punctuation marks
- text = re.sub(r'\s*([.,/!])\s*', r'\1', text)
- # Remove URLs
- text = re.sub(r'https?\s*:\s*\/\s*\/\s*\S+', '', text)
- # Initialize inflect engine
- p = inflect.engine()
- # Replace numbers with words
- text = re.sub(r'\d+', lambda x: p.number_to_words(x.group()), text)
- # Remove punctuation except exclamation mark
- text = re.sub(r'[^\w\s!]', '', text)
- # Tokenization
- tokens = word_tokenize(text)
- # Lemmatization
- lemmatizer = WordNetLemmatizer()
- tokens = [lemmatizer.lemmatize(word) for word in tokens]
- # Join tokens back to string
- text = ' '.join(tokens)
- return text
- # Define pre-processing function
- def preprocess_without_stopwords_with_lemmatization(text):
- # Lowercase
- text = text.lower()
- # Replace "5G" or "5 G" with "5 G"
- text = re.sub(r'\b(5\s?)g\b', '5g', text)
- # Strip white space
- text = text.strip()
- # Remove spaces between specific punctuation marks
- text = re.sub(r'\s*([.,/!])\s*', r'\1', text)
- # Remove URLs
- text = re.sub(r'https?\s*:\s*\/\s*\/\s*\S+', '', text)
- # Initialize inflect engine
- p = inflect.engine()
- # Replace numbers with words
- text = re.sub(r'\d+', lambda x: p.number_to_words(x.group()), text)
- # Remove punctuation except exclamation mark
- text = re.sub(r'[^\w\s!]', '', text)
- # Tokenization
- tokens = word_tokenize(text)
- # Remove stop words
- tokens = [word for word in tokens if word not in stopwords.words('english')]
- # Lemmatization
- lemmatizer = WordNetLemmatizer()
- tokens = [lemmatizer.lemmatize(word) for word in tokens]
- # Join tokens back to string
- text = ' '.join(tokens)
- return text
- # Define pre-processing function
- def preprocess_with_stopwords_without_lemmatization(text):
- # Lowercase
- text = text.lower()
- # Replace "5G" or "5 G" with "5 G"
- text = re.sub(r'\b(5\s?)g\b', '5g', text)
- # Strip white space
- text = text.strip()
- # Remove spaces between specific punctuation marks
- text = re.sub(r'\s*([.,/!])\s*', r'\1', text)
- # Remove URLs
- text = re.sub(r'https?\s*:\s*\/\s*\/\s*\S+', '', text)
- # Initialize inflect engine
- p = inflect.engine()
- # Replace numbers with words
- text = re.sub(r'\d+', lambda x: p.number_to_words(x.group()), text)
- # Remove punctuation except exclamation mark
- text = re.sub(r'[^\w\s!]', '', text)
- # Tokenization
- tokens = word_tokenize(text)
- # Join tokens back to string
- text = ' '.join(tokens)
- return text
- # Define pre-processing function
- def preprocess_without_stopwords_without_lemmatization(text):
- # Lowercase
- text = text.lower()
- # Replace "5G" or "5 G" with "5 G"
- text = re.sub(r'\b(5\s?)g\b', '5g', text)
- # Strip white space
- text = text.strip()
- # Remove spaces between specific punctuation marks
- text = re.sub(r'\s*([.,/!])\s*', r'\1', text)
- # Remove URLs
- text = re.sub(r'https?\s*:\s*\/\s*\/\s*\S+', '', text)
- # Initialize inflect engine
- p = inflect.engine()
- # Replace numbers with words
- text = re.sub(r'\d+', lambda x: p.number_to_words(x.group()), text)
- # Remove punctuation except exclamation mark
- text = re.sub(r'[^\w\s!]', '', text)
- # Tokenization
- tokens = word_tokenize(text)
- # Remove stop words
- tokens = [word for word in tokens if word not in stopwords.words('english')]
- # Join tokens back to string
- text = ' '.join(tokens)
- return text
Advertisement
Add Comment
Please, Sign In to add comment