WebCrawler

import requests
from bs4 import BeautifulSoup

title = []
price = []
def spider(max_pages):
    page = 1
    fw = open('price.txt','w')
    while page <= max_pages:
        url = 'https://www.minimalistwatches.com/collections/all?page=' + str(page)
        source_code = requests.get(url)
        plain_text = source_code.text
        soup = BeautifulSoup(plain_text, "html.parser")
        for elem in soup.select('h3.product-name a'):
            #href = "https://sfbay.craigslist.org" + link.get('href')
            title.append(elem['title'])

        for elem in soup.select('span.price span.money'):
            price.append(elem.text)

        page += 1
    for i in range(len(title)):
        print(title[i] + " : " + price[i])
        fw.write(title[i] + " : " + price[i] + "\n")
    fw.close



spider(3)



#Second file

import requests
from bs4 import BeautifulSoup


def spider():
    page = 1
    fw = open('professors2.txt', 'w')
    email = []
    name = []

    url = "https://www.cs.columbia.edu/people/faculty/"
    source_code = requests.get(url)
    plain_text = source_code.text
    soup = BeautifulSoup(plain_text, "html.parser")
    for elem in soup.select('div.faculty-contact-info ul li a'):
        email.append(elem.text)
    for elem in soup.select('div.faculty-name-and-title span a'):
        name.append(elem.text)

    page += 1
    for i in range(len(email)):
        fw.write(name[i] + " : " + email[i] + "\n")
        fw.close

spider()


Consumer Key (API Key)  E9Mmd8pQkqKupDWEBcpMS5sRj
Consumer Secret (API Secret)    xKfvXA9qGLzQmQ9EXhkMY3KBQ2LCciXEqDNva52P5fdAr2jhsZ
Access Token    952555751578505218-IrKOHHgX7xqrKdapTlYoWbQmIldV5bl
Access Token Secret Ie7U1hG89cHLSPBrvqYVoKULMAagJX7uKN0sYF1qZ2vnk




import tweepy
from time import sleep
from credentials import *

# Access and authorize our Twitter credentials from credentials.py
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)

# For loop to iterate over tweets with #ocean, limit to 10

# Print out usernames of the last 10 people to use #ocean
    #print('Tweet by: @' + tweet.user.screen_name)

for tweet in tweepy.Cursor(api.search,
                           q='#RaspberryPi',
                           since='2018-1-1',
                           until='2018-2-18',
                           #geocode='1.3552217,103.8231561,100km',
                           lang='en').items(10):

    print( tweet.text + 'Tweet by: @' + tweet.user.screen_name)

    #favorites the tweet
    tweet.favorite()


    #retweets the tweet
    tweet.retweet()
    #follows user
    if not tweet.user.following:
        tweet.user.follow()

    sleep(5)







#Credentials.ypy

consumer_key = "E9Mmd8pQkqKupDWEBcpMS5sRj"
consumer_secret = "xKfvXA9qGLzQmQ9EXhkMY3KBQ2LCciXEqDNva52P5fdAr2jhsZ"
access_token = "952555751578505218-IrKOHHgX7xqrKdapTlYoWbQmIldV5bl"
access_token_secret = "Ie7U1hG89cHLSPBrvqYVoKULMAagJX7uKN0sYF1qZ2vnk"