Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- from selenium import webdriver
- from selenium.webdriver.support.ui import WebDriverWait
- from selenium.webdriver.support import expected_conditions as EC
- from selenium.webdriver.common.by import By
- from selenium.common.exceptions import TimeoutException
- from selenium.common.exceptions import StaleElementReferenceException
- links.append('https://www.phantasytour.com/bands/phish/threads/4398112/trump-treson-thread-lxxxii#page/')
- list_of_pages = list(range(16))
- list_of_posters = list(range(30))
- last_page = list(range(19))
- for k in list(range(len(links))):
- print('Processing Topic ' + links[k].strip('https://www.phantasytour.com/bands/phish/threads/').replace('/','-') + '...')
- write_log.write("Processing Topic " + links[k].strip('https://www.phantasytour.com/bands/phish/threads/').replace('/','-') + '\n')
- poster_file = 'C:/users/matthew/documents/posters' + links[k].strip('https://www.phantasytour.com/bands/phish/threads/').replace('/','-') + '.txt'
- write_poster = open(poster_file,'w')
- time_file = 'C:/users/matthew/documents/time' + links[k].strip('https://www.phantasytour.com/bands/phish/threads/').replace('/','-') + '.txt'
- write_time = open(time_file,'w')
- driver = webdriver.Firefox()
- for i in list_of_pages:
- url = links[k] + str(i+1)
- driver.get(url)
- try:
- myElem = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CLASS_NAME, "poster_name")))
- myElem1 = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CLASS_NAME, "posted_at")))
- print ('Page ' + str(i+1) + ' loaded successfully.')
- write_log.write('Page ' + str(i+1) + ' loaded successfully.' + '\n')
- except TimeoutException:
- print ('Loading took too much time!')
- for j in list_of_posters:
- try:
- write_poster.write(driver.find_elements_by_class_name('poster_name')[j].text.strip(' •').strip(' (OP)').strip('» ') + '\n')
- write_time.write(driver.find_elements_by_class_name('posted_at')[j].text + '\n')
- except StaleElementReferenceException:
- pass
- url = links[k] + str(17)
- driver.get(url)
- try:
- myElem = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'poster_name')))
- myElem1 = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'posted_at')))
- print ('Page 17 loaded successfully.')
- write_log.write('Page 17 loaded successfully.' + '\n')
- except TimeoutException:
- print ('Loading took too much time!')
- for l in last_page:
- try:
- write_poster.write(driver.find_elements_by_class_name('poster_name')[l].text.strip(' •').strip(' (OP)').strip('» ') + '\n')
- write_time.write(driver.find_elements_by_class_name('posted_at')[l].text + '\n')
- except StaleElementReferenceException:
- pass
- write_poster.close()
- write_time.close()
- driver.quit()
- write_log.close()
- ##499 post topics:
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4433455/trump-treason-thread-cliii#page/') ##153
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4432579/trump-treason-thread-clii#page/')#152
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4431842/trump-treason-thread-cli#page/') ##151
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4430848/trump-treason-thread-cl#page/') ##150
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4429834/trump-treason-thread-cxlix-judgement-day#page/') ##149
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4429070/trump-treason-thread-cxlviii#page/') ##148
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4428558/trump-treason-thread-cxlvii#page/') ##147
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4428101/trump-treason-cxlvi-no-witty-subject-name#page/') ##146
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4427890/trump-treason-cxlvi-manaforts-in-jail#page/') ##146 again
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4427328/trump-treason-thread-cxlv#page/') ##145
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4427081/trump-treason-thread-cxliv#page/') ##144
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4426790/trump-treason-thread-cxliii#page/') ##143
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4425968/trump-treason-thread-cxlii#page/') ##142
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4425081/trump-treason-thread-cxl#page/') ##140
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4424337/trump-treason-thread-139#page/') ##139
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4423893/trump-treason-thread-cxxxviii#page/') ##138
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4423498/trump-treason-thread-cxxxvii#page/') ##137
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4423228/trump-treason-thread-cxxxvi#page/') ##136
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4422468/trump-treason-thread-cxxxv#page/') ##135
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4422262/trump-treason-thread-cxxxiv#page/') ##134
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4422025/trump-treason-thread-cxxxiii#page/') ##133
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4421749/trump-treason-thread-cxxxii#page/') ##132
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4421546/trump-treason-thread-cxxxi#page/') ##131
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4421331/trump-treason-thread-cxxx#page/') ##130
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4421192/trump-treason-thread-cxxix#page/') ##129
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4420637/trump-treason-thread-cxxviii#page/') ##128
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4420253/trump-treason-thread-cxxvii#page/') ##127
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4420042/trump-treason-thread-cxxvi#page/') ##126
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4419518/trump-treason-thread-cxxv#page/') ##125
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4418811/trump-treason-thread-cxxiv#page/') ##124
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4418566/trump-treason-thread-cxxiii#page/') ##123
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4418294/trump-treason-thread-cxxii#page/') ##122
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4418093/trump-treason-thread-cxxi#page/') ##121
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4417870/trump-treason-thread-cxx#page/') ##120
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4417194/trump-treason-thread-cxix#page/') ##119
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4416925/trump-treason-cxviii#page/') ##118
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4416798/trump-treason-thread-cxvii#page/') ##117
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4416628/trump-treason-thread-cxvi#page/') ##116
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4416365/trump-treason-thread-cxv#page/') ##115
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4415418/trump-treason-thread-cxiv#page/') ##114
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4415402/trump-treason-cxiii#page/') ##113
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4414919/trump-treason-thread-cxiii#page/') ##113 again
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4414556/trump-treason-thread-cxii#page/') ##112
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4414123/trump-treason-thread-cxi#page/') ##111
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4413673/trump-treason-thread-cx#page/') ##110
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4413499/trump-treason-thread-cix#page/') ##109
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4413045/trump-treason-thread-cviii#page/') ##108
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4412675/trump-treason-thread-cvii#page/') ##107
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4412004/trump-treason-thread-cv#page/') ##105
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4411606/trump-treason-thread-civ#page/') ##104
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4411467/trump-treason-thread-ciii#page/') ##103
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4410831/trump-treason-thread-c#page/') ##100
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4410237/trump-treason-thread-xcviiii#page/') ##99
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4409594/trump-treason-thread-xcviii#page/') ##98
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4408929/trump-treason-thread-xcvii#page/') ##97
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4407675/trump-treason-thread-xcvi#page/') ##96
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4407204/trump-treason-thread-xcv#page/') ##95
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4406100/trump-treason-thread-xciv#page/') ##94
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4405496/trump-treason-thread-xciii#page/') ##93
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4405033/trump-treason-thread-xcii#page/') ##92
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4404225/trump-treason-thread-xci#page/') ##91
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4403273/trump-treason-thread-xc#page/') ##90
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4402719/trump-treason-thread-lxxxviv#page/') ##89
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4401746/trump-treason-thread-lxxxviii#page/') ##88
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4401160/trump-treason-thread-lxxxvii#page/') ##87
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4400749/trump-treason-thread-lxxxvi#page/') ###86
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4399847/trump-treason-thread-lxxxv#page/') ##85
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4399511/trump-treason-thread-lxxxiv#page/') ##84
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4398757/trump-treason-thread-lxxxiii#page/') ##83
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4397688/trump-treason-thread-lxxxi#page/') ##81
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4397056/trump-treason-thread-lxxx#page/') ##80
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4396521/trump-treason-thread-lxxix#page/') #79
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4395655/trump-treason-lxxviii#page/') ##78
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4394913/trump-treason-lxxvii#page/') ##77
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4394156/trump-treason-thread-lxxvi#page/') ##76
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4393120/trump-treason-thread-lxxv#page/') ##75
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4392139/trump-treason-tread-lxxiv#page/') ##74
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4391386/trump-treason-thread-lxxiii#page/') ##73
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4390835/trump-treason-thread-lxxii#page/') ##72
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4390104/trump-treason-thread-lxx#page/') ##70
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4389327/trump-treason-thread-lxix#page/') ##69
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4389074/trump-treason-thread-lxix#page/') ##69 again
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4388638/trump-treason-thread-lxviii#page/') ##68
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4387925/trump-treason-thread-lxvii#page/') ##67
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4386910/trump-treason-thread-lxvi#page/') ##66
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4386166/trump-treason-thread-lxv#page/') ##65
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4384766/trump-treason-thread-lxiv#page/') ##64
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4384098/trump-treason-thread-lxiii#page/') ##63
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4383696/trump-treason-thread-lxii#page/') ##62
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4382925/trump-treason-thread-lxi#page/') ##61
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4382347/trump-treason-thread-lx#page/') ##60
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4382020/trump-treason-lix#page/') ##59
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4380805/trump-treason-thread-lviii#page/') ##58
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4379560/trump-treason-thread-lvii#page/') ##57
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4377502/trump-treason-thread-lv#page/') ##55
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4376853/trump-treason-thread-liv#page/') ##54
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4376542/trump-treason-thread-liii#page/') ##53
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4376128/trump-treason-thread-lii#page/') ##52
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4373824/trump-treason-thread-l#page/') ##50
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4372711/trump-treason-thread-xlix#page/') ##49
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4371007/trump-treason-thread-xlviii#page/') ##48
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4367502/trump-treason-thread-xlvii#page/') ##47
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4365968/trump-treason-thread-xlvi#page/') ##46
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4364834/trump-treason-thread-45#page/') ##45
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4363899/trump-treason-thread-xliv#page/') ##44
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4363436/trump-treason-thread-xliii#page/') ##43
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4362579/trump-treason-thread-xli#page/') ##41
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4362059/trump-treason-thread-xl#page/') ##40
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4361309/trump-treason-thread-xxxix#page/') ##39
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4359565/trump-treason-thread-xxxviii#page/') ##38
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4355967/trump-treason-thread-xxxvii#page/') ##37
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4352739/trump-treason-thread-xxxvi#page/') ##36
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4350803/trump-treason-thread-xxxv#page/') ##35
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4349851/trump-treason-thread-xxxiv#page/') ##34
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4348044/trump-treason-thread-xxxiii#page/') ##33
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4343097/trump-treason-thread-xxxii#page/') ##32
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4332422/trump-treason-thread-xxxi#page/') ##31
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4326050/trump-treason-thread-xxx#page/') ##30
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4324605/trump-treason-thread-xxix#page/') ##29
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4320657/trump-treason-thread-xxviii#page/') ##28
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4317650/trump-treason-thread-xxvii#page/') ##27
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4316890/trump-treason-thread-xxvi#page/') ##26
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4315904/trump-treason-thread-xxv#page/') ##25
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4315614/trump-treason-thread-xxiv#page/') ##24
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4315253/trump-treason-thread-xxiii#page/') ##23
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4313610/trump-treason-thread-xxii#page/') ##22
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4313013/trump-treason-thread-xxi#page/') ##21
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4310998/trump-treason-thread-xx#page/') ##20
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4309852/trump-treason-thread-xix#page/') ##19
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4308584/trump-treason-thread-xviii#page/') ##18
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4307919/trump-treason-thread-xvii#page/') ##17
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4307609/trump-treason-thread-xvi#page/') ##16
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4306321/trump-treason-thread-xv#page/') ##15
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4304536/trump-treason-thread-xiv#page/') ##14
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4304062/trump-treason-thread-xiii#page/') ##13
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4303639/part-xii-flynn-trump-russia-tick-tock#page/') ##12
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4300294/part-xi-flynn-trump-russia-tick-tock#page/') ##11
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4298538/part-x-fbi-flynn-trump-russia#page/') ##10
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4297718/part-ix-fbi-flynn-trump-russia#page/') ##9
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4295914/part-vi-fbi-investigating-trump-russia#page/') ##6
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4295480/part-v-fbi-investigating-trump-russia#page/') ##5
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4290623/part-2-trump-and-his-russian-collusion#page/') ##2
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4288789/trump-will-go-down-for-collusion-with-russia#page/') ##1
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4433823/trump-treason-thread-cliv#page/') ##154
- ##498 post topics:
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4425572/trump-treason-thread-cxli#page/') ##141 498
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4412184/trump-treason-thread-cvi#page/') ##106 498
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4377922/trump-treason-thread-lvi#page/') ##56 498
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4374856/trump-treason-thread-li#page/') ##51 498
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4363122/trump-treason-thread-xlii#page/') ##42 498
- ##494 post:
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4390424/trump-treason-thread-lxxi#page/') ##71 494
- ##481 post:
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4411211/trump-treason-thread-cii-losing-his-sht#page/') ##102 481
- ##500 post:
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4410995/trump-treason-thread-ci#page/') ##101 500
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4296419/part-viii-fbi-investigating-trump-russia#page/') ##8 500
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4294376/part-iv-fbi-investigating-trump-russia#page/') ##4 500
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4293739/part-iii-fbi-investigating-trump-russia#page/') ##3 500
- ##502 post:
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4296190/part-vii-fbi-investigating-trump-russia#page/') ##7 502
- ##Last topic:
- ##links.append('https://www.phantasytour.com/bands/phish/threads/4434284/trump-treason-thread-clv-trump-is-russian#/page/')
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement