SHARE
TWEET

iterate by chunks timings

a guest Jan 24th, 2012 1,117 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import itertools
  2. import timeit
  3.  
  4.        
  5. def grouper_it(n, iterable):
  6.     it = iter(iterable)
  7.     while True:
  8.         chunk_it = itertools.islice(it, n)
  9.         try:
  10.             first_el = next(chunk_it)
  11.         except StopIteration:
  12.             return
  13.         yield itertools.chain((first_el,), chunk_it)
  14.  
  15. def grouper(n, iterable):
  16.     it = iter(iterable)
  17.     while True:
  18.        chunk = tuple(itertools.islice(it, n))
  19.        if not chunk:
  20.            return
  21.        yield chunk
  22.        
  23. def dummy_element_consumer(func, n=3, data_size=10000):
  24.     for chunk in func(n, xrange(data_size)):
  25.         for el in chunk:
  26.             dummy = el
  27.            
  28. def dummy_chunk_consumer(func, n=3, data_size=10000):
  29.     for chunk in func(n, xrange(data_size)):
  30.         dummy = tuple(chunk)
  31.  
  32.  
  33. for consumer in (dummy_element_consumer, dummy_chunk_consumer):
  34.     print consumer.__name__
  35.     for data_size, npasses in ((100, 10000), (10000, 1000)):
  36.         print ' data_size: %s, passes: %s' % (data_size, npasses)
  37.         for n in (3, 10, 100, 1000):
  38.             print '  grouper   ', n, timeit.timeit(lambda : consumer(grouper, n, data_size), number=npasses)
  39.             print '  grouper_it', n, timeit.timeit(lambda : consumer(grouper_it, n, data_size), number=npasses)
  40.            
  41.         print
  42. ------------------
  43. Results:
  44. dummy_element_consumer
  45.  data_size: 100, passes: 10000
  46.   grouper    3 0.75289614549
  47.   grouper_it 3 0.682097099117
  48.   grouper    10 0.283187205571
  49.   grouper_it 10 0.287213798965
  50.   grouper    100 0.137917801202
  51.   grouper_it 100 0.132218313015
  52.   grouper    1000 0.138246513214
  53.   grouper_it 1000 0.133138039654
  54.  
  55.  data_size: 10000, passes: 1000
  56.   grouper    3 7.03408728375
  57.   grouper_it 3 6.03008289606
  58.   grouper    10 2.48628471741
  59.   grouper_it 10 2.28650824356
  60.   grouper    100 1.05532127117
  61.   grouper_it 100 0.764705173135
  62.   grouper    1000 0.752920695266
  63.   grouper_it 1000 0.611105924492
  64.  
  65. dummy_chunk_consumer
  66.  data_size: 100, passes: 10000
  67.   grouper    3 0.761046326397
  68.   grouper_it 3 1.00031595122
  69.   grouper    10 0.258946700939
  70.   grouper_it 10 0.361995614131
  71.   grouper    100 0.105095489586
  72.   grouper_it 100 0.14534625367
  73.   grouper    1000 0.105786698303
  74.   grouper_it 1000 0.145586671485
  75.  
  76.  data_size: 10000, passes: 1000
  77.   grouper    3 7.10003329424
  78.   grouper_it 3 9.44492849201
  79.   grouper    10 2.20704193098
  80.   grouper_it 10 3.02586649712
  81.   grouper    100 0.738520806174
  82.   grouper_it 100 0.903916032582
  83.   grouper    1000 0.385726813393
  84.   grouper_it 1000 0.479506690848
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
Top