solkey_scraper.py

import time
from bs4 import BeautifulSoup
from openpyxl import Workbook
from selenium import webdriver

url = 'https://www.duurzaamloket.nl/SolKey_X014/index.php?SchemeNo=0&Offset=1&SearchText=&PageCnt=500'

# Set up the Selenium WebDriver (replace the path with the path to your WebDriver)
driver = webdriver.Chrome(executable_path='/path/to/chromedriver')
driver.get(url)

# Wait for the table to load
time.sleep(5)

# Parse the page content with BeautifulSoup
soup = BeautifulSoup(driver.page_source, 'html.parser')
driver.quit()

# Find the table and extract the data
table = soup.find('table', class_='LicDetails')
headers = [th.get_text(strip=True) for th in table.find_all('th')]
rows = table.find_all('tr')[1:]

# Create a new Excel workbook and write the data
wb = Workbook()
ws = wb.active

# Write headers to the first row
for col_num, header in enumerate(headers, 1):
    ws.cell(row=1, column=col_num, value=header)

# Write table data to the worksheet
for row_num, row in enumerate(rows, 2):
    cols = row.find_all('td')
    for col_num, col in enumerate(cols, 1):
        ws.cell(row=row_num, column=col_num, value=col.get_text(strip=True))

# Save the workbook to an Excel file
wb.save('solkey_data.xlsx')