Guest User

Untitled

a guest
Nov 21st, 2017
77
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.45 KB | None | 0 0
  1. df_users.groupby(['userID', 'requestDate']).apply(feature_rollup)
  2.  
  3. def logged_apply(g, func, *args, **kwargs):
  4. step_percentage = 100. / len(g)
  5. import sys
  6. sys.stdout.write('apply progress: 0%')
  7. sys.stdout.flush()
  8.  
  9. def logging_decorator(func):
  10. def wrapper(*args, **kwargs):
  11. progress = wrapper.count * step_percentage
  12. sys.stdout.write('33[D 33[D' * 4 + format(progress, '3.0f') + '%')
  13. sys.stdout.flush()
  14. wrapper.count += 1
  15. return func(*args, **kwargs)
  16. wrapper.count = 0
  17. return wrapper
  18.  
  19. logged_func = logging_decorator(func)
  20. res = g.apply(logged_func, *args, **kwargs)
  21. sys.stdout.write('33[D 33[D' * 4 + format(100., '3.0f') + '%' + 'n')
  22. sys.stdout.flush()
  23. return res
  24.  
  25. In [11]: g = df_users.groupby(['userID', 'requestDate'])
  26.  
  27. In [12]: f = feature_rollup
  28.  
  29. In [13]: logged_apply(g, f)
  30. apply progress: 100%
  31. Out[13]:
  32. ...
  33.  
  34. from pandas.core.groupby import DataFrameGroupBy
  35. DataFrameGroupBy.logged_apply = logged_apply
  36.  
  37. In [21]: g.logged_apply(f)
  38. apply progress: 100%
  39. Out[21]:
  40. ...
  41.  
  42. import pandas as pd
  43. import numpy as np
  44. from tqdm import tqdm, tqdm_pandas
  45.  
  46. df = pd.DataFrame(np.random.randint(0, int(1e8), (10000, 1000)))
  47.  
  48. # Create and register a new `tqdm` instance with `pandas`
  49. # (can use tqdm_gui, optional kwargs, etc.)
  50. tqdm_pandas(tqdm())
  51.  
  52. # Now you can use `progress_apply` instead of `apply`
  53. df.groupby(0).progress_apply(lambda x: x**2)
  54.  
  55. df_users.groupby(['userID', 'requestDate']).apply(feature_rollup)
  56.  
  57. from tqdm import tqdm, tqdm_pandas
  58. tqdm_pandas(tqdm())
  59. df_users.groupby(['userID', 'requestDate']).progress_apply(feature_rollup)
  60.  
  61. from functools import wraps
  62.  
  63. def logging_decorator(func):
  64.  
  65. @wraps
  66. def wrapper(*args, **kwargs):
  67. wrapper.count += 1
  68. print "The function I modify has been called {0} times(s).".format(
  69. wrapper.count)
  70. func(*args, **kwargs)
  71. wrapper.count = 0
  72. return wrapper
  73.  
  74. modified_function = logging_decorator(feature_rollup)
  75.  
  76. def count_wrapper(func,total, print_at):
  77.  
  78. def wrapper(*args):
  79. wrapper.count += 1
  80. if wrapper.count % wrapper.print_at == 0:
  81. clear_output()
  82. sys.stdout.write( "%d / %d"%(calc_time.count,calc_time.total) )
  83. sys.stdout.flush()
  84. return func(*args)
  85. wrapper.count = 0
  86. wrapper.total = total
  87. wrapper.print_at = print_at
  88.  
  89. return wrapper
  90.  
  91. from IPython.core.display import clear_output
Add Comment
Please, Sign In to add comment