API tools faq
paste
Advertisement
nikson1

vw trace

nikson1
Jun 22nd, 2018
82
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 14.88 KB | None | 0 0
raw download clone embed print report
  1. [angara] 2018-06-22 20:25:38,034 - ERROR - failed interaction lr weights
  2. Traceback (most recent call last):
  3. File "hhkardinal/train/__init__.py", line 26, in wrapper
  4. return function(*args, **kwargs)
  5. File "/home/nikson/PycharmProjects/hh.kardinal/hhkardinal/angara/train/interaction_lr/weights_computer.py", line 141, in main
  6. fit_weights_all(args.ncores)
  7. File "/home/nikson/PycharmProjects/hh.kardinal/hhkardinal/angara/train/interaction_lr/weights_computer.py", line 136, in fit_weights_all
  8. Parallel(n_jobs=ncores)(delayed(fit_weights)(cfg) for cfg in INTERACTION_LR_CFGS)
  9. File "/usr/local/lib/python2.7/dist-packages/joblib/parallel.py", line 789, in __call__
  10. self.retrieve()
  11. File "/usr/local/lib/python2.7/dist-packages/joblib/parallel.py", line 740, in retrieve
  12. raise exception
  13. JoblibCalledProcessError: JoblibCalledProcessError
  14. ___________________________________________________________________________
  15. Multiprocessing exception:
  16. ...........................................................................
  17. /usr/lib/python2.7/runpy.py in _run_module_as_main(mod_name='hhkardinal.angara.train.interaction_lr.weights_computer', alter_argv=1)
  18. 157 pkg_name = mod_name.rpartition('.')[0]
  19. 158 main_globals = sys.modules["__main__"].__dict__
  20. 159 if alter_argv:
  21. 160 sys.argv[0] = fname
  22. 161 return _run_code(code, main_globals, None,
  23. --> 162 "__main__", fname, loader, pkg_name)
  24. fname = '/home/nikson/PycharmProjects/hh.kardinal/hhkardinal/angara/train/interaction_lr/weights_computer.py'
  25. loader = <pkgutil.ImpLoader instance>
  26. pkg_name = 'hhkardinal.angara.train.interaction_lr'
  27. 163
  28. 164 def run_module(mod_name, init_globals=None,
  29. 165 run_name=None, alter_sys=False):
  30. 166 """Execute a module's code without importing it
  31.  
  32. ...........................................................................
  33. /usr/lib/python2.7/runpy.py in _run_code(code=<code object <module> at 0x7f38776b6230, file "/...rain/interaction_lr/weights_computer.py", line 3>, run_globals={'CONFIG_TEMPLATE': {'cache_name': 'train.cache', 'invert_name': 'model.invert', 'model_name': 'train.model', 'predict_name': 'predict.txt', 'readable_model_name': 'readable.model', 'train_name': 'train.vw'}, 'INTERACTION_LR_CFGS': [<hhkardinal.angara.features.InteractionLRConfig object>], 'INTERACTION_LR_TRUNCATED': [<hhkardinal.angara.features.InteractionLRConfig object>, <hhkardinal.angara.features.InteractionLRConfig object>, <hhkardinal.angara.features.InteractionLRConfig object>], 'INTERCEPT': 'intercept', 'Level': <enum 'Level'>, 'MetaNode': <class 'hhkardinal.feature_group.core.MetaNode'>, 'OrderedDict': <class 'collections.OrderedDict'>, 'PARAMS_CONFIG_TEMPLATE': {'predict_params': '--loss_function logistic --link=logistic', 'train_params': '--loss_function logistic'}, 'Parallel': <class 'joblib.parallel.Parallel'>, 'TOP_COUNT': 60, ...}, init_globals=None, mod_name='__main__', mod_fname='/home/nikson/PycharmProjects/hh.kardinal/hhkardinal/angara/train/interaction_lr/weights_computer.py', mod_loader=<pkgutil.ImpLoader instance>, pkg_name='hhkardinal.angara.train.interaction_lr')
  34. 67 run_globals.update(init_globals)
  35. 68 run_globals.update(__name__ = mod_name,
  36. 69 __file__ = mod_fname,
  37. 70 __loader__ = mod_loader,
  38. 71 __package__ = pkg_name)
  39. ---> 72 exec code in run_globals
  40. code = <code object <module> at 0x7f38776b6230, file "/...rain/interaction_lr/weights_computer.py", line 3>
  41. run_globals = {'CONFIG_TEMPLATE': {'cache_name': 'train.cache', 'invert_name': 'model.invert', 'model_name': 'train.model', 'predict_name': 'predict.txt', 'readable_model_name': 'readable.model', 'train_name': 'train.vw'}, 'INTERACTION_LR_CFGS': [<hhkardinal.angara.features.InteractionLRConfig object>], 'INTERACTION_LR_TRUNCATED': [<hhkardinal.angara.features.InteractionLRConfig object>, <hhkardinal.angara.features.InteractionLRConfig object>, <hhkardinal.angara.features.InteractionLRConfig object>], 'INTERCEPT': 'intercept', 'Level': <enum 'Level'>, 'MetaNode': <class 'hhkardinal.feature_group.core.MetaNode'>, 'OrderedDict': <class 'collections.OrderedDict'>, 'PARAMS_CONFIG_TEMPLATE': {'predict_params': '--loss_function logistic --link=logistic', 'train_params': '--loss_function logistic'}, 'Parallel': <class 'joblib.parallel.Parallel'>, 'TOP_COUNT': 60, ...}
  42. 73 return run_globals
  43. 74
  44. 75 def _run_module_code(code, init_globals=None,
  45. 76 mod_name=None, mod_fname=None,
  46.  
  47. ...........................................................................
  48. /home/nikson/PycharmProjects/hh.kardinal/hhkardinal/angara/train/interaction_lr/weights_computer.py in <module>()
  49. 140 def main(args):
  50. 141 fit_weights_all(args.ncores)
  51. 142
  52. 143
  53. 144 if __name__ == '__main__':
  54. --> 145 main(train_argument_parser('generate interaction lr weights').parse_args())
  55.  
  56. ...........................................................................
  57. /home/nikson/PycharmProjects/hh.kardinal/hhkardinal/train/__init__.py in wrapper(*args=(Namespace(ncores=14, ntasks=14),), **kwargs={})
  58. 21 def decorator(function):
  59. 22 @wraps(function)
  60. 23 def wrapper(*args, **kwargs):
  61. 24 init_train_env()
  62. 25 try:
  63. ---> 26 return function(*args, **kwargs)
  64. args = (Namespace(ncores=14, ntasks=14),)
  65. kwargs = {}
  66. 27 except Exception:
  67. 28 logger.exception(fail_message)
  68. 29 sys.exit(2)
  69. 30
  70.  
  71. ...........................................................................
  72. /home/nikson/PycharmProjects/hh.kardinal/hhkardinal/angara/train/interaction_lr/weights_computer.py in main(args=Namespace(ncores=14, ntasks=14))
  73. 136 Parallel(n_jobs=ncores)(delayed(fit_weights)(cfg) for cfg in INTERACTION_LR_CFGS)
  74. 137
  75. 138
  76. 139 @with_train_env(fail_message='failed interaction lr weights')
  77. 140 def main(args):
  78. --> 141 fit_weights_all(args.ncores)
  79. args.ncores = 14
  80. 142
  81. 143
  82. 144 if __name__ == '__main__':
  83. 145 main(train_argument_parser('generate interaction lr weights').parse_args())
  84.  
  85. ...........................................................................
  86. /home/nikson/PycharmProjects/hh.kardinal/hhkardinal/angara/train/interaction_lr/weights_computer.py in fit_weights_all(ncores=14)
  87. 131 node.accessor(angara_sample).save_meta((weights_lrqa.astype(np.float32), weights_lrqb.astype(np.float32)))
  88. 132 # fit_interaction_meta(node, interaction_sample))
  89. 133
  90. 134
  91. 135 def fit_weights_all(ncores):
  92. --> 136 Parallel(n_jobs=ncores)(delayed(fit_weights)(cfg) for cfg in INTERACTION_LR_CFGS)
  93. ncores = 14
  94. 137
  95. 138
  96. 139 @with_train_env(fail_message='failed interaction lr weights')
  97. 140 def main(args):
  98.  
  99. ...........................................................................
  100. /usr/local/lib/python2.7/dist-packages/joblib/parallel.py in __call__(self=Parallel(n_jobs=14), iterable=<generator object <genexpr>>)
  101. 784 if pre_dispatch == "all" or n_jobs == 1:
  102. 785 # The iterable was consumed all at once by the above for loop.
  103. 786 # No need to wait for async callbacks to trigger to
  104. 787 # consumption.
  105. 788 self._iterating = False
  106. --> 789 self.retrieve()
  107. self.retrieve = <bound method Parallel.retrieve of Parallel(n_jobs=14)>
  108. 790 # Make sure that we get a last message telling us we are done
  109. 791 elapsed_time = time.time() - self._start_time
  110. 792 self._print('Done %3i out of %3i | elapsed: %s finished',
  111. 793 (len(self._output), len(self._output),
  112.  
  113. ---------------------------------------------------------------------------
  114. Sub-process traceback:
  115. ---------------------------------------------------------------------------
  116. CalledProcessError Fri Jun 22 20:25:36 2018
  117. PID: 4137 Python 2.7.6: /usr/bin/python
  118. ...........................................................................
  119. /usr/local/lib/python2.7/dist-packages/joblib/parallel.py in __call__(self=<joblib.parallel.BatchedCalls object>)
  120. 126 def __init__(self, iterator_slice):
  121. 127 self.items = list(iterator_slice)
  122. 128 self._size = len(self.items)
  123. 129
  124. 130 def __call__(self):
  125. --> 131 return [func(*args, **kwargs) for func, args, kwargs in self.items]
  126. func = <function fit_weights>
  127. args = (<hhkardinal.angara.features.InteractionLRConfig object>,)
  128. kwargs = {}
  129. self.items = [(<function fit_weights>, (<hhkardinal.angara.features.InteractionLRConfig object>,), {})]
  130. 132
  131. 133 def __len__(self):
  132. 134 return self._size
  133. 135
  134.  
  135. ...........................................................................
  136. /home/nikson/PycharmProjects/hh.kardinal/hhkardinal/angara/train/interaction_lr/weights_computer.py in fit_weights(cfg=<hhkardinal.angara.features.InteractionLRConfig object>)
  137. 111
  138. 112 cmd_config.update(PARAMS_CONFIG_TEMPLATE)
  139. 113 cmd_config['train_params'] += ' {}'.format(build_parameters_line(config.interaction_lr['parameters']))
  140. 114
  141. 115 cfgs_transformed = get_cfgs_transformed(cfg)
  142. --> 116 weights = compute_weights(cmd_config, target, get_features_sizes(cfg), [get_cfgs_transformed(cfg)])
  143. weights = undefined
  144. cmd_config = {'cache_name': 'angara/main_sample/interaction_lr/ed3eef0632c4ebe114c22aad4e4cad741eabb2dc_train.cache', 'invert_name': 'angara/main_sample/interaction_lr/ed3eef0632c4ebe114c22aad4e4cad741eabb2dc_model.invert', 'model_name': 'angara/main_sample/interaction_lr/ed3eef0632c4ebe114c22aad4e4cad741eabb2dc_train.model', 'predict_name': 'angara/main_sample/interaction_lr/ed3eef0632c4ebe114c22aad4e4cad741eabb2dc_predict.txt', 'predict_params': '--loss_function logistic --link=logistic', 'readable_model_name': 'angara/main_sample/interaction_lr/ed3eef0632c4ebe114c22aad4e4cad741eabb2dc_readable.model', 'train_name': 'angara/main_sample/interaction_lr/ed3eef0632c4ebe114c22aad4e4cad741eabb2dc_train.vw', 'train_params': '--loss_function logistic --passes 1 --ftrl_alpha...wer_t 0.82074853371 --ftrl --l2 7.33506369226e-05'}
  145. target = 317 0
  146. 318 0
  147. 319 0
  148. 320...me: graded_target, Length: 17600791, dtype: int64
  149. cfg = <hhkardinal.angara.features.InteractionLRConfig object>
  150. 117
  151. 118 weights_left = csr_matrix(weights[cfgs_transformed[0]])
  152. 119 weights_right = csr_matrix(weights[cfgs_transformed[1]])
  153. 120 weights_lrqa = csr_matrix(weights['lrq', cfgs_transformed[0]])
  154.  
  155. ...........................................................................
  156. /home/nikson/PycharmProjects/hh.kardinal/hhkardinal/angara/train/interaction_lr/vw.py in compute_weights(config={'cache_name': '/home/nikson/PycharmProjects/hh.kardinal/cache/a...eef0632c4ebe114c22aad4e4cad741eabb2dc_train.cache', 'invert_name': '/home/nikson/PycharmProjects/hh.kardinal/cache/a...ef0632c4ebe114c22aad4e4cad741eabb2dc_model.invert', 'model_name': '/home/nikson/PycharmProjects/hh.kardinal/cache/a...eef0632c4ebe114c22aad4e4cad741eabb2dc_train.model', 'predict_name': '/home/nikson/PycharmProjects/hh.kardinal/cache/a...eef0632c4ebe114c22aad4e4cad741eabb2dc_predict.txt', 'predict_params': '--loss_function logistic --link=logistic', 'readable_model_name': '/home/nikson/PycharmProjects/hh.kardinal/cache/a...0632c4ebe114c22aad4e4cad741eabb2dc_readable.model', 'train_name': '/home/nikson/PycharmProjects/hh.kardinal/cache/a...ed3eef0632c4ebe114c22aad4e4cad741eabb2dc_train.vw', 'train_params': '--loss_function logistic --passes 1 --ftrl_alpha...74853371 --ftrl --l2 7.33506369226e-05 --lrq ab20'}, target=317 0
  157. 318 0
  158. 319 0
  159. 320...me: graded_target, Length: 17600791, dtype: int64, features_sizes=OrderedDict([('a', 10000), ('b', 10000)]), pairs_cfgs=[('a', 'b')])
  160. 194 interactions = ' {}'.format(' '.join('--lrq {}{}{}'.format(cfg[0], cfg[1], RANK) for cfg in pairs_cfgs))
  161. 195 logger.error(interactions)
  162. 196 config['train_params'] += interactions
  163. 197
  164. 198 call(TRAIN_CMD_TEMPLATE.format(**config), shell=True)
  165. --> 199 call(WEIGHTS_CMD_TEMPLATE.format(**config), shell=True)
  166. config = {'cache_name': '/home/nikson/PycharmProjects/hh.kardinal/cache/a...eef0632c4ebe114c22aad4e4cad741eabb2dc_train.cache', 'invert_name': '/home/nikson/PycharmProjects/hh.kardinal/cache/a...ef0632c4ebe114c22aad4e4cad741eabb2dc_model.invert', 'model_name': '/home/nikson/PycharmProjects/hh.kardinal/cache/a...eef0632c4ebe114c22aad4e4cad741eabb2dc_train.model', 'predict_name': '/home/nikson/PycharmProjects/hh.kardinal/cache/a...eef0632c4ebe114c22aad4e4cad741eabb2dc_predict.txt', 'predict_params': '--loss_function logistic --link=logistic', 'readable_model_name': '/home/nikson/PycharmProjects/hh.kardinal/cache/a...0632c4ebe114c22aad4e4cad741eabb2dc_readable.model', 'train_name': '/home/nikson/PycharmProjects/hh.kardinal/cache/a...ed3eef0632c4ebe114c22aad4e4cad741eabb2dc_train.vw', 'train_params': '--loss_function logistic --passes 1 --ftrl_alpha...74853371 --ftrl --l2 7.33506369226e-05 --lrq ab20'}
  167. 200
  168. 201 weights_dict = extract_weights(config['invert_name'])
  169. 202 logger.error(weights_dict[LRQ, 'a'])
  170. 203 logger.error(weights_dict[LRQ, 'b'])
  171.  
  172. ...........................................................................
  173. /usr/lib/python2.7/subprocess.py in check_call(*popenargs=('vw -i /home/nikson/PycharmProjects/hh.kardinal/c...ebe114c22aad4e4cad741eabb2dc_model.invert --quiet',), **kwargs={'shell': True})
  174. 535 retcode = call(*popenargs, **kwargs)
  175. 536 if retcode:
  176. 537 cmd = kwargs.get("args")
  177. 538 if cmd is None:
  178. 539 cmd = popenargs[0]
  179. --> 540 raise CalledProcessError(retcode, cmd)
  180. retcode = 137
  181. cmd = 'vw -i /home/nikson/PycharmProjects/hh.kardinal/c...ebe114c22aad4e4cad741eabb2dc_model.invert --quiet'
  182. 541 return 0
  183. 542
  184. 543
  185. 544 def check_output(*popenargs, **kwargs):
  186.  
  187. CalledProcessError: Command 'vw -i /home/nikson/PycharmProjects/hh.kardinal/cache/angara/main_sample/interaction_lr/ed3eef0632c4ebe114c22aad4e4cad741eabb2dc_train.model -t /home/nikson/PycharmProjects/hh.kardinal/cache/angara/main_sample/interaction_lr/ed3eef0632c4ebe114c22aad4e4cad741eabb2dc_train.vw --invert_hash=/home/nikson/PycharmProjects/hh.kardinal/cache/angara/main_sample/interaction_lr/ed3eef0632c4ebe114c22aad4e4cad741eabb2dc_model.invert --quiet' returned non-zero exit status 137
  188. ___________________________________________________________________________
  189. nikson@beholder:~/PycharmProjects/hh.kardinal$
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!