Guest User

Scapre Daily Sangram

a guest
Apr 30th, 2012
41
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.09 KB | None | 0 0
  1. import codecs
  2. from urllib import urlopen
  3. from BeautifulSoup import BeautifulSoup
  4.  
  5.  
  6. def scrape(folder, start, end):
  7.     print '-*-*-*-*-*-*-*-*- Scarping Starts -*-*-*-*-*-*-*-*-*-'
  8.     for I in range(start, end+1):
  9.         url = 'http://www.dailysangram.com/news_details.php?news_id=' + str(I)
  10.         print url , ' : '
  11.         wp = urlopen(url)
  12.         soup =BeautifulSoup(wp)
  13.         if soup:
  14.             doc = ''
  15.             texts = soup.findAll(True, {'class':'news-details'})
  16.             if texts:
  17.                 for text in texts:
  18.                     doc += text.getText() + '\n'
  19.         if doc.strip()=='':
  20.             continue
  21.         print doc
  22. #        confirm = raw_input('Write in document (y/n)? ')
  23. #        if confirm != 'n':
  24.         fp = codecs.open(folder+'/'+str(I)+'.dat', 'w', encoding='utf-8')
  25.         fp.writelines(doc)
  26.         fp.close()
  27.         print '-------------- end -----------------\n\n'
  28.     print '-*-*-*-*-*-*-*-*-*- Done All Scraping -*-*-*-*-*-*-*-*-*-*-*-'
  29.  
  30.  
  31. "------------------------------------------------------"
  32.  
  33. scrape('test', 24700, 24800)
Advertisement
Add Comment
Please, Sign In to add comment