filter_user_db.py

import json
import re
import logging
import os

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(levelname)s] %(message)s',
    handlers=[
        logging.StreamHandler()
    ]
)

# Path to your user database JSON file
USER_DATABASE_FILE = 'user_database.json'

# Path to the output file for filtered users
FILTERED_USERS_FILE = 'filtered_users.json'

# First category keywords (Roles)
FIRST_CATEGORY_KEYWORDS = [

]
#  TO DO ,       'professor', 'lecturer', 'researcher', 'scientist', 'phd', 'dr\.', 'doctor',
#   'postdoc', 'post-doctoral', 'associate professor', 'assistant professor', 'science communicator', 'reporter', 'journalist'


# Second category keywords (Fields and Related Terms)
SECOND_CATEGORY_KEYWORDS= [
    # 'climatologist', 'meteorologist', 'ecologist', 'biologist', 'physicist',
    # 'chemist', 'geologist', 'oceanographer', 'environmentalist', 'sociologist',
    # 'anthropologist', 'economist', 'policy analyst',
    # 'statistician', 'conservationist', 'environmental scientist',
    # 'climate scientist', 'sustainability expert',
    # 'climate', 'global warming', 'sustainability', 'renewable energy',
    # 'biodiversity', 'ecosystem', 'environment', 'conservation',
    # 'sustainable development', 'carbon footprint', 'greenhouse gases',
    # 'climate change', 'environmental policy', 'ecology', 'environmental science',
    # 'environmental studies'
]

# Excluded phrases (Profiles containing these phrases will be excluded)
EXCLUDED_PHRASES  = [
    'researcher', 'scientist', 'professor', 'phd', 'dr.', 'postdoc', 'ph.d.' 'dr',
   'teaching associate', 'fellow', 'lecturer', 'prof.', 'science communicator'
]

#  TO DO ,       , 'researcher',  'reporter', 'journalist'

# Function to compile regex patterns for single-word keywords
def compile_single_word_pattern(keywords):
    # Escape keywords to handle any special regex characters
    escaped_keywords = [re.escape(keyword) for keyword in keywords]
    # Add optional 's' or 'es' for plural forms
    pattern = r'\b(' + '|'.join(escaped_keywords) + r')(?:s|es)?\b'
    return re.compile(pattern, re.IGNORECASE)

# Function to compile regex patterns for multi-word phrases
def compile_multi_word_pattern(phrases):
    # Escape phrases to handle any special regex characters
    escaped_phrases = [re.escape(phrase) for phrase in phrases]
    # Join phrases without word boundaries
    pattern = r'(' + '|'.join(escaped_phrases) + r')'
    return re.compile(pattern, re.IGNORECASE)

# Separate single-word keywords and multi-word phrases
def separate_keywords(keywords):
    single_word_keywords = []
    multi_word_phrases = []
    for keyword in keywords:
        if ' ' in keyword or '-' in keyword or '.' in keyword or '\\' in keyword:
            multi_word_phrases.append(keyword)
        else:
            single_word_keywords.append(keyword)
    return single_word_keywords, multi_word_phrases

# Separate first category keywords
first_single_words, first_multi_words = separate_keywords(FIRST_CATEGORY_KEYWORDS)
# Separate second category keywords
second_single_words, second_multi_words = separate_keywords(SECOND_CATEGORY_KEYWORDS)

# Compile patterns
first_single_word_pattern = compile_single_word_pattern(first_single_words)
first_multi_word_pattern = compile_multi_word_pattern(first_multi_words)
second_single_word_pattern = compile_single_word_pattern(second_single_words)
second_multi_word_pattern = compile_multi_word_pattern(second_multi_words)
excluded_phrase_pattern = compile_multi_word_pattern(EXCLUDED_PHRASES)

def filter_profiles():
    try:
        # Check if the user database file exists
        if not os.path.exists(USER_DATABASE_FILE):
            logging.error(f"The file {USER_DATABASE_FILE} does not exist.")
            return

        # Load the user database
        with open(USER_DATABASE_FILE, 'r', encoding='utf-8') as f:
            user_database = json.load(f)

        # Ensure that the user_database is a list
        if not isinstance(user_database, list):
            logging.error(f"The file {USER_DATABASE_FILE} should contain a JSON array of user profiles.")
            return

        logging.info(f'Loaded {len(user_database)} user profiles from {USER_DATABASE_FILE}')

        filtered_users = []

        # Iterate over each user profile
        for idx, user in enumerate(user_database, start=1):
            subject = user.get('subject', user)  # Adjusted to handle different data structures
            display_name = subject.get('displayName', '')
            description = subject.get('description', '')
            combined_text = f'{display_name} {description}'

            # Check for excluded phrases first
            if excluded_phrase_pattern.search(combined_text):
                continue  # Skip this profile

            # First category matching
            first_match = (
                first_single_word_pattern.search(combined_text) or
                first_multi_word_pattern.search(combined_text)
            )

            # Second category matching
            second_match = (
                second_single_word_pattern.search(combined_text) or
                second_multi_word_pattern.search(combined_text)
            )

            if first_match and second_match:
                # Extract only desired fields
                filtered_user = {
                    'uri': user.get('uri', ''),
                    'subject': {
                        'did': subject.get('did', ''),
                        'handle': subject.get('handle', ''),
                        'displayName': display_name,
                        'avatar': subject.get('avatar', ''),
                        'description': description,
                        'createdAt': subject.get('createdAt', ''),
                        'indexedAt': subject.get('indexedAt', '')
                    }
                }
                filtered_users.append(filtered_user)

            # Optional: Log progress every 100 profiles
            if idx % 100 == 0:
                logging.info(f'Processed {idx} profiles...')

        logging.info(f'Found {len(filtered_users)} matching profiles.')

        # Save the filtered users to a new JSON file
        with open(FILTERED_USERS_FILE, 'w', encoding='utf-8') as f:
            json.dump(filtered_users, f, ensure_ascii=False, indent=4)
        logging.info(f'Saved filtered profiles to {FILTERED_USERS_FILE}')

    except json.JSONDecodeError as jde:
        logging.error(f'JSON decoding error: {jde}')
    except Exception as e:
        logging.error(f'An unexpected error occurred: {e}')

if __name__ == '__main__':
    filter_profiles()