FooDownunder

gevents vs urllib3

Jun 10th, 2015
339
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.96 KB | None | 0 0
  1. import sys
  2.  
  3. import gevent
  4. from gevent import monkey
  5.  
  6. gevent.monkey.patch_all(thread=False)
  7.  
  8. import grequests
  9. import urllib3
  10. http = urllib3.PoolManager()
  11.  
  12. def call_back(resp):
  13. content = resp.content
  14.  
  15. def worker(url, use_urllib2=False):
  16. if use_urllib2:
  17. content = http.request('GET', url)
  18.  
  19. else:
  20. rs = [grequests.get(u) for u in url]
  21. resps = grequests.map(rs)
  22. for resp in resps:
  23. call_back(resp)
  24.  
  25. urls = ['http://www.google.com/']*50
  26.  
  27. def by_requests():
  28. worker(urls)
  29. def by_urllib2():
  30. jobs = [gevent.spawn(worker, url, True) for url in urls]
  31. gevent.joinall(jobs)
  32.  
  33. if __name__=='__main__':
  34. from timeit import Timer
  35. t = Timer(stmt="by_requests()", setup="from __main__ import by_requests")
  36. print 'by requests: %s seconds'%t.timeit(number=3)
  37. t = Timer(stmt="by_urllib2()", setup="from __main__ import by_urllib2")
  38. print 'by urllib3: %s seconds'%t.timeit(number=3)
Advertisement
Add Comment
Please, Sign In to add comment