Vas0acc

Untitled

Nov 10th, 2025
263
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 3.79 KB | None | 0 0
  1. # Define pre-processing function
  2. def preprocess_with_stopwords_with_lemmatization(text):
  3.     # Lowercase
  4.     text = text.lower()
  5.     # Replace "5G" or "5 G" with "5 G"
  6.     text = re.sub(r'\b(5\s?)g\b', '5g', text)
  7.     # Strip white space
  8.     text = text.strip()
  9.     # Remove spaces between specific punctuation marks
  10.     text = re.sub(r'\s*([.,/!])\s*', r'\1', text)
  11.     # Remove URLs
  12.     text = re.sub(r'https?\s*:\s*\/\s*\/\s*\S+', '', text)
  13.     # Initialize inflect engine
  14.     p = inflect.engine()
  15.     # Replace numbers with words
  16.     text = re.sub(r'\d+', lambda x: p.number_to_words(x.group()), text)
  17.     # Remove punctuation except exclamation mark
  18.     text = re.sub(r'[^\w\s!]', '', text)
  19.     # Tokenization
  20.     tokens = word_tokenize(text)
  21.     # Lemmatization
  22.     lemmatizer = WordNetLemmatizer()
  23.     tokens = [lemmatizer.lemmatize(word) for word in tokens]
  24.     # Join tokens back to string
  25.     text = ' '.join(tokens)
  26.     return text
  27.  
  28.  
  29. # Define pre-processing function
  30. def preprocess_without_stopwords_with_lemmatization(text):
  31.     # Lowercase
  32.     text = text.lower()    
  33.     # Replace "5G" or "5 G" with "5 G"
  34.     text = re.sub(r'\b(5\s?)g\b', '5g', text)
  35.     # Strip white space
  36.     text = text.strip()
  37.     # Remove spaces between specific punctuation marks
  38.     text = re.sub(r'\s*([.,/!])\s*', r'\1', text)
  39.     # Remove URLs
  40.     text = re.sub(r'https?\s*:\s*\/\s*\/\s*\S+', '', text)
  41.     # Initialize inflect engine
  42.     p = inflect.engine()
  43.     # Replace numbers with words
  44.     text = re.sub(r'\d+', lambda x: p.number_to_words(x.group()), text)
  45.     # Remove punctuation except exclamation mark
  46.     text = re.sub(r'[^\w\s!]', '', text)
  47.     # Tokenization
  48.     tokens = word_tokenize(text)
  49.     # Remove stop words
  50.     tokens = [word for word in tokens if word not in stopwords.words('english')]
  51.     # Lemmatization
  52.     lemmatizer = WordNetLemmatizer()
  53.     tokens = [lemmatizer.lemmatize(word) for word in tokens]
  54.     # Join tokens back to string
  55.     text = ' '.join(tokens)
  56.  
  57.     return text
  58.  
  59. # Define pre-processing function
  60. def preprocess_with_stopwords_without_lemmatization(text):
  61.     # Lowercase
  62.     text = text.lower()    
  63.     # Replace "5G" or "5 G" with "5 G"
  64.     text = re.sub(r'\b(5\s?)g\b', '5g', text)
  65.     # Strip white space
  66.     text = text.strip()
  67.     # Remove spaces between specific punctuation marks
  68.     text = re.sub(r'\s*([.,/!])\s*', r'\1', text)
  69.     # Remove URLs
  70.     text = re.sub(r'https?\s*:\s*\/\s*\/\s*\S+', '', text)
  71.     # Initialize inflect engine
  72.     p = inflect.engine()
  73.     # Replace numbers with words
  74.     text = re.sub(r'\d+', lambda x: p.number_to_words(x.group()), text)
  75.     # Remove punctuation except exclamation mark
  76.     text = re.sub(r'[^\w\s!]', '', text)
  77.     # Tokenization
  78.     tokens = word_tokenize(text)
  79.     # Join tokens back to string
  80.     text = ' '.join(tokens)
  81.     return text
  82.  
  83. # Define pre-processing function
  84. def preprocess_without_stopwords_without_lemmatization(text):
  85.     # Lowercase
  86.     text = text.lower()    
  87.     # Replace "5G" or "5 G" with "5 G"
  88.     text = re.sub(r'\b(5\s?)g\b', '5g', text)
  89.     # Strip white space
  90.     text = text.strip()
  91.     # Remove spaces between specific punctuation marks
  92.     text = re.sub(r'\s*([.,/!])\s*', r'\1', text)
  93.     # Remove URLs
  94.     text = re.sub(r'https?\s*:\s*\/\s*\/\s*\S+', '', text)
  95.     # Initialize inflect engine
  96.     p = inflect.engine()
  97.     # Replace numbers with words
  98.     text = re.sub(r'\d+', lambda x: p.number_to_words(x.group()), text)
  99.     # Remove punctuation except exclamation mark
  100.     text = re.sub(r'[^\w\s!]', '', text)
  101.     # Tokenization
  102.     tokens = word_tokenize(text)
  103.     # Remove stop words
  104.     tokens = [word for word in tokens if word not in stopwords.words('english')]
  105.     # Join tokens back to string
  106.     text = ' '.join(tokens)
  107.     return text
Advertisement
Add Comment
Please, Sign In to add comment