Advertisement
Guest User

Untitled

a guest
Jul 16th, 2019
75
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.92 KB | None | 0 0
  1. 46 while remaining > 0:$
  2. 47 LOG.info(f"Position: {from_query}")$
  3. 48 size = remaining if remaining < batch_size else batch_size$
  4. 49 request_url = f"{BASE_URL}/associations?size={size}&from={from_query}"$
  5. 50 req = session.request(url=request_url, method='GET')$
  6. 51 if req.status_code == 404:$
  7. 52 raise Exception (f"404, {request_url} not found")$
  8. 53 hits = req.json()["hits"]["hits"]$
  9. 54 #print(f"hits={len(hits)}")$
  10. 55 #with open('hits.json', 'w') as f:$
  11. 56 # json.dump(hits, f, indent=4)$
  12. 57 document.append(hits)$
  13. 58 #break$
  14. 59 #print(len(hits)[0])$
  15. 60 retrieved = len(hits)$
  16. 61 remaining = remaining - retrieved$
  17. 62 from_query = from_query + retrieved$
  18. 63 LOG.info(f"Remaining: {remaining}")$
  19. 64 LOG.info("Finished downloading")$
  20. 65 return document$
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement