WSP3

import requests
from bs4 import BeautifulSoup

def scrape_dataset_count():
    url = "https://catalog.data.gov/dataset"

    # Header needed to mimic a standard browser request
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    }

    try:
        response = requests.get(url, headers=headers, timeout=10)

        if response.status_code == 200:
            soup = BeautifulSoup(response.text, "html.parser")

            # Target the specific span class highlighted in your browser inspection
            count_span = soup.find("span", class_="text-heavy text-secondary font-sans-lg")

            if count_span:
                # Extracts the raw text (e.g., "557,486")
                count = count_span.text.strip()
                print(f"Number of datasets available on Data.gov: {count}")
            else:
                print("Could not locate the dataset count element on the page.")
        else:
            print(f"failed to retrieve webpage. Status code: {response.status_code}")

    except requests.exceptions.RequestException as e:
        print(f"Error fetching Data.gov webpage: {e}")

 scrape_dataset_count()