import requests
import random
import csv
import time
from bs4 import BeautifulSoup

headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'}


f = csv.writer(open('zomato_data.csv', 'w'))
f.writerow(['title', 'address', 'district', 'cost_for_two', 'cuisines', 'rating', 'numVotes'])


response = requests.get("https://www.zomato.com/san-francisco/restaurants?page=807", headers=headers)
content = response.content
bs = BeautifulSoup(content, "html.parser")

zomato_containers = bs.find_all("div", {"class": "search-snippet-card"})

for zomato_container in zomato_containers:

    title = zomato_container.find("a", {"class": "result-title"}).get_text()

    try:
        address = zomato_container.find("div", {"class": "search-result-address"}).get_text()
        if address is None:
            address = 'N/A'
        district = zomato_container.find("a", {"class": "search_result_subzone"}).get_text()
        if district is None:
            district = 'N/A'
        cost_for_two = zomato_container.select_one('[class*="col-s-11 col-m-12 pl0"]').get_text(separator='|', strip=True).split('|')
        cost_for_two = cost_for_two[1] if len(cost_for_two) > 1 else cost_for_two[0]
        if cost_for_two is None:
            cost_for_two = 'N/A'
        cuisines = zomato_container.find("div", {"class": "res-snippet-small-establishment mt5"}).get_text()
        if cuisines is None:
            cuisines = 'N/A'
        rating = zomato_container.select_one('.rating-popup').text.strip()
        if rating is None:
            rating = 'N/A'
        numVotes = zomato_container.select_one('[class^=rating-votes-div]').text  # match on elements with class attribute whose values starts with rating-votes-div
        if numVotes  is None:
            numVotes = 'N/A'

    except AttributeError:
        continue

    print("restaurant_title: ", title)
    print("restaurant_address: ", address)
    print("restaurant_district: ", district)
    print("cost_for_two: ", cost_for_two)
    print("restaurant_cuisines: ", cuisines)
    print("rating: ", rating)
    print("numVotes: ", numVotes)

    f.writerow([title, address, district, cost_for_two, cuisines, rating, numVotes])