Advertisement
anacrolix

github language popularity using concurrent futures

Dec 18th, 2011
140
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.55 KB | None | 0 0
  1. #!/usr/bin/env python3.2
  2.  
  3. import logging, urllib.request, urllib.parse, re, concurrent.futures, itertools
  4.  
  5. def language_popularity(lang):
  6.     qlang = urllib.parse.quote(lang)
  7.     try:
  8.         sub = urllib.request.urlopen("http://github.com/languages/" + qlang).read()
  9.     except:
  10.         logging.exception('Error fetching %r', lang)
  11.         return
  12.     match = re.search(b'is the #(\\d+)', sub)
  13.     if match:
  14.         rank = int(match.group(1))
  15.     else:
  16.         rank = 1
  17.     return rank, lang
  18.  
  19. def languages():
  20.     for match in re.finditer(b'/languages/([^/]+?)"', urllib.request.urlopen('http://github.com/languages').read()):
  21.         yield urllib.parse.unquote(match.group(1).decode())
  22.  
  23. def main():
  24.     futures = []
  25.     with concurrent.futures.ThreadPoolExecutor(45) as pool:
  26.         for lang in languages():
  27.             futures.append(pool.submit(language_popularity, lang))
  28.         ranks = {}
  29.         completed = concurrent.futures.as_completed(futures)
  30.         for rank in itertools.count(1):
  31.             while rank not in ranks:
  32.                 try:
  33.                     result = next(completed).result()
  34.                 except StopIteration:
  35.                     break
  36.                 if result is None:
  37.                     continue
  38.                 new_rank, lang = result
  39.                 ranks[new_rank] = lang
  40.             if rank not in ranks:
  41.                 break
  42.             print(rank, ranks[rank])
  43.         for rank1 in sorted(r for r in ranks if r >= rank):
  44.             print(rank1, ranks[rank1])
  45.  
  46. if __name__ == '__main__':
  47.     main()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement