Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def get_data(url):
- urlprime = url
- if '?format=json' in urlprime:
- url = urlprime
- else:
- url = urlprime + '?format=json'
- headers = {
- 'User-Agent': ua
- }
- cj = cookielib.LWPCookieJar(COOKIE_FILE)
- try:
- cj.load(COOKIE_FILE, ignore_discard=True)
- except:
- pass
- s = requests.Session()
- s.cookies = cj
- resp = s.get(url, headers=headers, verify=False)
- data1 = resp.json()
- try:
- pages = data1['paging']
- if int(pages['pageNumber']) == 1 :
- global datadump
- datadump = resp.json()
- else:
- pass
- if int(pages['pageNumber']) < int(pages['totalPages']):
- nextpage = int(pages['pageNumber']) + 1
- urlnew = url[:-5]
- if '&pn=' in url:
- url = urlnew + '&pn=' + str(nextpage)
- else:
- url = url + '&pn=' + str(nextpage)
- get_data(url)
- datadump['programs'].append(data1['programs'])
- else:
- pass
- pass
- except KeyError:
- datadump = resp.json()
- pass
- return datadump
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement