Advertisement
Guest User

Untitled

a guest
Mar 18th, 2017
243
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.17 KB | None | 0 0
  1. def get_data(url):
  2.     urlprime = url
  3.     if '?format=json' in urlprime:
  4.         url = urlprime
  5.     else:
  6.         url = urlprime + '?format=json'
  7.     headers = {
  8.         'User-Agent': ua
  9.     }
  10.  
  11.     cj = cookielib.LWPCookieJar(COOKIE_FILE)
  12.     try:
  13.         cj.load(COOKIE_FILE, ignore_discard=True)
  14.     except:
  15.         pass
  16.  
  17.     s = requests.Session()
  18.     s.cookies = cj
  19.     resp = s.get(url, headers=headers, verify=False)
  20.     data1 = resp.json()
  21.    
  22.     try:
  23.         pages = data1['paging']
  24.         if int(pages['pageNumber']) == 1 :
  25.             global datadump
  26.             datadump = resp.json()
  27.         else:
  28.             pass
  29.    
  30.         if int(pages['pageNumber']) < int(pages['totalPages']):        
  31.             nextpage = int(pages['pageNumber']) + 1
  32.             urlnew = url[:-5]
  33.             if '&pn=' in url:
  34.                 url = urlnew + '&pn=' + str(nextpage)
  35.             else:
  36.                 url = url + '&pn=' + str(nextpage)
  37.             get_data(url)
  38.             datadump['programs'].append(data1['programs'])
  39.         else:
  40.             pass
  41.         pass
  42.     except KeyError:
  43.         datadump = resp.json()
  44.         pass
  45.  
  46.     return datadump
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement