WSP2

import requests

def fetch_robots_txt(domain):
    url = f"https://{domain}/robots.txt"
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            print(f"✅ robots.txt content from {domain}:\n")
            print(response.text)
        else:
            print(f"⚠️ Failed to retrieve robots.txt. Status code: {response.status_code}")
    except requests.exceptions.RequestException as e:
        print(f"❌ Error fetching robots.txt: {e}")

# Example usage
fetch_robots_txt("en.wikipedia.org")