Advertisement
Guest User

iterate by chunks timings

a guest
Jan 24th, 2012
3,194
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 2.50 KB | None | 0 0
  1. import itertools
  2. import timeit
  3.  
  4.        
  5. def grouper_it(n, iterable):
  6.     it = iter(iterable)
  7.     while True:
  8.         chunk_it = itertools.islice(it, n)
  9.         try:
  10.             first_el = next(chunk_it)
  11.         except StopIteration:
  12.             return
  13.         yield itertools.chain((first_el,), chunk_it)
  14.  
  15. def grouper(n, iterable):
  16.     it = iter(iterable)
  17.     while True:
  18.        chunk = tuple(itertools.islice(it, n))
  19.        if not chunk:
  20.            return
  21.        yield chunk
  22.        
  23. def dummy_element_consumer(func, n=3, data_size=10000):
  24.     for chunk in func(n, xrange(data_size)):
  25.         for el in chunk:
  26.             dummy = el
  27.            
  28. def dummy_chunk_consumer(func, n=3, data_size=10000):
  29.     for chunk in func(n, xrange(data_size)):
  30.         dummy = tuple(chunk)
  31.  
  32.  
  33. for consumer in (dummy_element_consumer, dummy_chunk_consumer):
  34.     print consumer.__name__
  35.     for data_size, npasses in ((100, 10000), (10000, 1000)):
  36.         print ' data_size: %s, passes: %s' % (data_size, npasses)
  37.         for n in (3, 10, 100, 1000):
  38.             print '  grouper   ', n, timeit.timeit(lambda : consumer(grouper, n, data_size), number=npasses)
  39.             print '  grouper_it', n, timeit.timeit(lambda : consumer(grouper_it, n, data_size), number=npasses)
  40.            
  41.         print
  42. ------------------
  43. Results:
  44. dummy_element_consumer
  45.  data_size: 100, passes: 10000
  46.   grouper    3 0.75289614549
  47.   grouper_it 3 0.682097099117
  48.   grouper    10 0.283187205571
  49.   grouper_it 10 0.287213798965
  50.   grouper    100 0.137917801202
  51.   grouper_it 100 0.132218313015
  52.   grouper    1000 0.138246513214
  53.   grouper_it 1000 0.133138039654
  54.  
  55.  data_size: 10000, passes: 1000
  56.   grouper    3 7.03408728375
  57.   grouper_it 3 6.03008289606
  58.   grouper    10 2.48628471741
  59.   grouper_it 10 2.28650824356
  60.   grouper    100 1.05532127117
  61.   grouper_it 100 0.764705173135
  62.   grouper    1000 0.752920695266
  63.   grouper_it 1000 0.611105924492
  64.  
  65. dummy_chunk_consumer
  66.  data_size: 100, passes: 10000
  67.   grouper    3 0.761046326397
  68.   grouper_it 3 1.00031595122
  69.   grouper    10 0.258946700939
  70.   grouper_it 10 0.361995614131
  71.   grouper    100 0.105095489586
  72.   grouper_it 100 0.14534625367
  73.   grouper    1000 0.105786698303
  74.   grouper_it 1000 0.145586671485
  75.  
  76.  data_size: 10000, passes: 1000
  77.   grouper    3 7.10003329424
  78.   grouper_it 3 9.44492849201
  79.   grouper    10 2.20704193098
  80.   grouper_it 10 3.02586649712
  81.   grouper    100 0.738520806174
  82.   grouper_it 100 0.903916032582
  83.   grouper    1000 0.385726813393
  84.   grouper_it 1000 0.479506690848
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement