selo_scraper2.py

#Import all modules/libraries

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
from tqdm import tqdm
import os



print("Starting scraping for second site..")

company_links = []
results2 = []
final_region_links = []

r = requests.get("https://www.bobex.be/control/directory")
soup = BeautifulSoup(r.content, "html.parser")

region_links = [link.get("href") for link in soup.findAll(class_="linkblock")]

#all_links = ["https://"+"".join(link.split("/")[1:3])+"/control/directory?language=nl&ti="+link.split("/")[3]+"&region.id="+link[-1] for link in all_links]


print("Getting links for all regions...")
for region_link in tqdm(region_links):
    r = requests.get(region_link)
    soup = BeautifulSoup(r.content, "html.parser")

    services_list = [item.get("data-ti") for item in soup.find("select", {"id":"type.id"}).findAll("option")]
    values = [item.get("value") for item in soup.find("select", {"id":"type.id"}).findAll("option")]

    for index, item in enumerate(services_list):
        sublink = "https://www.bobex.be/control/search_local2?language=nl&ti="+services_list[index]+"&type.id="+values[index]+"&region.id="+region_link[-1]
        final_region_links.append(sublink.strip())


#Take the set of final_region_links keeping its order/arrangement
final_region_links = sorted(set(final_region_links), key=final_region_links.index)




#Get all company links from final_region_links
print("Getting all company links...")
for link_item in tqdm(final_region_links):
    r = requests.get(link_item)
    soup = BeautifulSoup(r.content, "html.parser")

    cards = soup.findAll("div", class_="col-12")

    for card in cards:
        try:
            company_link = card.find("a").get("href")
            company_links.append(company_link)

        except Exception:
            pass



company_links = [link for link in company_links if link[-1] in ("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") and "http" in link]

company_links = sorted(set(company_links), key=company_links.index)

print("There are " + str(len(set(company_links))) + " company links")


#Getting the required data from all company links
for link in tqdm(company_links[0:]):
    obj = {}
    obj["Company Link"] = link

    raw_services = ""
    services = ""
    company_name = ""

    try:
        r = requests.get(link)
        soup = BeautifulSoup(r.content, "html.parser")

    except Exception as e:
        soup = ""
        print(e)
        pass


    try:
        company_name = soup.find("h1", class_="font1em7").text.strip()

    except:
        pass


    try:
        raw_services = soup.find("div", {"id":"activities"}).findAll("a")
        services = ", ".join([item.text.strip() for item in raw_services])

    except Exception as e:
        pass


    obj["Services"] = services
    obj["Company Name"] = company_name

    results2.append(obj)



#results2 = [item for item in results2 if item["Services"] != ""]


df2 = pd.DataFrame(results2)
df2 = df2[["Company Link", "Company Name", "Services"]]
df2 = pd.concat([df2, df2["Services"].str.split(', ', expand=True)], axis=1)
df2 = df2.drop_duplicates(["Company Link"])
df2.to_excel("results2.xlsx", index=False)


print("Done!")
print("Check " + str(os.getcwd()) + " for the result")