Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- from urllib import request
- urls = (
- 'http://math.stackexchange.com/election/5?tab=primary',
- 'http://serverfault.com/election/5?tab=primary',
- 'http://stackoverflow.com/election/7?tab=primary',
- )
- for url in urls:
- print('fetching {} ...'.format(url))
- request.urlopen(url).read()
- fetching http://math.stackexchange.com/election/5?tab=primary ...
- fetching http://serverfault.com/election/5?tab=primary ...
- fetching http://stackoverflow.com/election/7?tab=primary ...
- Traceback (most recent call last):
- File "examples/t.py", line 11, in <module>
- request.urlopen(url).read()
- File "/opt/local/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 161, in urlopen
- return opener.open(url, data, timeout)
- File "/opt/local/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 469, in open
- response = meth(req, response)
- File "/opt/local/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 579, in http_response
- 'http', request, response, code, msg, hdrs)
- File "/opt/local/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 507, in error
- return self._call_chain(*args)
- File "/opt/local/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 441, in _call_chain
- result = func(*args)
- File "/opt/local/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/urllib/request.py", line 587, in http_error_default
- raise HTTPError(req.full_url, code, msg, hdrs, fp)
- urllib.error.HTTPError: HTTP Error 403: Forbidden
- from urllib import request
- urls = (
- 'http://math.stackexchange.com/election/5?tab=primary',
- 'http://serverfault.com/election/5?tab=primary',
- 'http://stackoverflow.com/election/7?tab=primary',
- )
- for url in urls:
- print('fetching {} ...'.format(url))
- try:
- request.urlopen(url).read()
- except:
- print('got an exception, changing user-agent to urllib3 default')
- req = request.Request(url)
- req.add_header('User-Agent', 'Python-urllib/3.4')
- try:
- request.urlopen(req)
- except:
- print('got another exception, changing user-agent to something else')
- req.add_header('User-Agent', 'not-Python-urllib/3.4')
- request.urlopen(req)
- fetching http://math.stackexchange.com/election/5?tab=primary ...
- success with url: http://math.stackexchange.com/election/5?tab=primary
- fetching http://serverfault.com/election/5?tab=primary ...
- success with url: http://serverfault.com/election/5?tab=primary
- fetching http://stackoverflow.com/election/7?tab=primary ...
- got an exception, changing user-agent to urllib default
- got another exception, changing user-agent to something else
- success with url: http://stackoverflow.com/election/7?tab=primary
- import requests
- urls = (
- 'http://math.stackexchange.com/election/5?tab=primary',
- 'http://serverfault.com/election/5?tab=primary',
- 'http://stackoverflow.com/election/7?tab=primary',
- )
- for url in urls:
- print('fetching {} ...'.format(url))
- data = requests.get(url)
- import requests
- urls = (
- 'http://math.stackexchange.com/election/5?tab=primary',
- 'http://serverfault.com/election/5?tab=primary',
- 'http://stackoverflow.com/election/7?tab=primary',
- )
- with requests.Session() as session:
- for url in urls:
- print('fetching {} ...'.format(url))
- data = session.get(url)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement