Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def parse_blog(self, soup):
- for block in soup.find_all('li', class_='g'):
- if block.get('id') == 'newsbox':
- continue
- item = self.init_item()
- try:
- item['title'] = block.find('h3', class_="r").text.strip()
- url = block.find('h3', class_="r").a.get('href')
- link = urlparse.parse_qs(url).get('/url?q')
- if link:
- item['link'] = link[0]
- else:
- item['link'] = url
- except:
- print traceback.format_exc()
- try:
- tmp = block.find('div', class_="f slp")
- if tmp:
- date_author = tmp.text
- da = date_author.split(' by ')
- if len(da) == 1:
- item['date'] = da[0]
- if len(da) == 2:
- item['date'], item['author'] = da[0], da[1]
- except:
- print traceback.format_exc()
- try:
- item['excerpt'] = block.find('div',
- class_='st').text.strip()
- except:
- try:
- item['excerpt'] = block.find('span',
- class_='st').text.strip()
- except:
- print traceback.format_exc()
- try:
- tmp = item['excerpt'].split('-', 1)
- if len(tmp) == 2:
- ds = tmp[0]
- if 'days ago' in ds:
- days = int(ds.split()[0])
- date = datetime.date.today() - datetime.timedelta(days=days)
- date = date.strftime('%b %d, %Y')
- item['date'] = date
- else:
- item['date'] = ds
- except:
- print traceback.format_exc()
- if item:
- item['search_url'] = self.search_url
- item['type'] = 'blog'
- self.count += 1
- self.work_q.put(item)
Advertisement
Add Comment
Please, Sign In to add comment