make_network_db.py

import json
import requests
import os
import traceback  # Added for detailed exception logging
from urllib.parse import quote

# Replace these with your Bluesky username and app password
USERNAME = os.getenv('BLUESKY_USERNAME')
APP_PASSWORD = os.getenv('BLUESKY_PASSWORD')

#  Paths to the JSON files
INITIAL_USERS_FILE = 'members_output.json'  # The file you provided
DATABASE_FILE = 'user_database.json'
PROCESSED_INITIAL_USERS_FILE = 'processed_initial_users.json'
SKIPPED_USERS_FILE = 'skipped_users.json'

# Base URL for the Bluesky API
API_BASE_URL = 'https://bsky.social'
def login(username, password):
    url = f'{API_BASE_URL}/xrpc/com.atproto.server.createSession'
    payload = {
        'identifier': username,
        'password': password
    }
    response = requests.post(url, json=payload)
    response.raise_for_status()
    data = response.json()
    return data['accessJwt']

def get_profile(access_token, actor):
    actor_encoded = quote(actor, safe=':')
    url = f'{API_BASE_URL}/xrpc/app.bsky.actor.getProfile?actor={actor_encoded}'
    headers = {
        'Authorization': f'Bearer {access_token}'
    }
    response = requests.get(url, headers=headers)
    if response.status_code == 400:
        print(f'Could not get profile for actor {actor}: {response.text}')
        return None
    response.raise_for_status()
    return response.json()

def get_following(access_token, actor_did):
    url = f'{API_BASE_URL}/xrpc/app.bsky.graph.getFollows'  # Gets the list of users the actor is following
    headers = {
        'Authorization': f'Bearer {access_token}'
    }
    params = {
        'actor': actor_did,
        'limit': 100  # Max limit per API
    }
    following = []
    cursor = None
    while True:
        if cursor:
            params['cursor'] = cursor
        response = requests.get(url, headers=headers, params=params)
        if response.status_code == 400:
            print(f'Could not get following for actor {actor_did}: {response.text}')
            break
        response.raise_for_status()
        data = response.json()
        following.extend(data.get('follows', []))  # The key is 'follows' in the API response
        cursor = data.get('cursor')
        if not cursor:
            break
    return following

def save_json_file(filename, data):
    with open(filename, 'w') as f:
        json.dump(data, f, indent=4)
    print(f'Saved data to {filename}')

def main():
    try:
        # Authenticate and get access token
        print('Logging in...')
        access_token = login(USERNAME, APP_PASSWORD)
        print('Logged in successfully.')

        # Load or initialize the database
        if os.path.exists(DATABASE_FILE):
            with open(DATABASE_FILE, 'r') as f:
                user_database = json.load(f)
        else:
            user_database = []
            save_json_file(DATABASE_FILE, user_database)

        # Load or initialize the list of processed initial users
        if os.path.exists(PROCESSED_INITIAL_USERS_FILE):
            with open(PROCESSED_INITIAL_USERS_FILE, 'r') as f:
                processed_initial_users = json.load(f)
        else:
            processed_initial_users = []
            save_json_file(PROCESSED_INITIAL_USERS_FILE, processed_initial_users)

        # Load or initialize the list of skipped users
        if os.path.exists(SKIPPED_USERS_FILE):
            with open(SKIPPED_USERS_FILE, 'r') as f:
                skipped_users = json.load(f)
        else:
            skipped_users = []
            save_json_file(SKIPPED_USERS_FILE, skipped_users)

        # Create sets for quick lookup
        existing_users = set(user['handle'] for user in user_database)
        processed_initial_users_set = set(processed_initial_users)
        skipped_users_set = set(skipped_users)

        # Load initial users from members_output.json
        with open(INITIAL_USERS_FILE, 'r') as f:
            initial_users_data = json.load(f)

        initial_users = []
        initial_users_set = set()

        # Extract handles from the initial users data
        for item in initial_users_data:
            subject = item.get('subject', {})
            handle = subject.get('handle')
            if handle:
                initial_users.append(handle)
                initial_users_set.add(handle)
            else:
                print('No handle found for one of the initial users.')

        print(f'Found {len(initial_users)} initial users from {INITIAL_USERS_FILE}.')

        # Process each user's follows using limit and cursor
        for username in initial_users:
            if username in processed_initial_users_set:
                print(f'Skipping initial user {username}: Already processed.')
                continue

            print(f'Processing user: {username}')
            try:
                # Get the user's profile to obtain their DID
                profile = get_profile(access_token, username)
                if profile is None:
                    print(f'Skipping user {username}: Unable to get profile.')
                    if username not in skipped_users_set:
                        skipped_users.append(username)
                        skipped_users_set.add(username)
                        save_json_file(SKIPPED_USERS_FILE, skipped_users)
                    continue
                user_did = profile['did']

                # Fetch the list of users this user is following using limit and cursor
                following = get_following(access_token, user_did)

                # Use the data from 'following' directly without additional API calls
                for follow in following:
                    followed_username = follow.get('handle')
                    if (followed_username not in existing_users and
                            followed_username not in initial_users_set):
                        # Add the 'follow' data directly to user_database
                        user_database.append(follow)
                        existing_users.add(followed_username)

                # Mark the initial user as processed
                processed_initial_users.append(username)
                processed_initial_users_set.add(username)

                # Save data after each user is processed
                save_json_file(DATABASE_FILE, user_database)
                save_json_file(PROCESSED_INITIAL_USERS_FILE, processed_initial_users)
                save_json_file(SKIPPED_USERS_FILE, skipped_users)

                print(f'Finished processing user: {username}')

            except requests.HTTPError as e:
                print(f'HTTP error processing {username}: {e}')
                traceback.print_exc()
                if username not in skipped_users_set:
                    skipped_users.append(username)
                    skipped_users_set.add(username)
                    save_json_file(SKIPPED_USERS_FILE, skipped_users)
            except Exception as e:
                print(f'Error processing {username}: {e}')
                traceback.print_exc()
                if username not in skipped_users_set:
                    skipped_users.append(username)
                    skipped_users_set.add(username)
                    save_json_file(SKIPPED_USERS_FILE, skipped_users)

        print('Processing completed.')

    except Exception as e:
        print(f'An unexpected error occurred: {e}')
        traceback.print_exc()

if __name__ == '__main__':
    main()