API tools faq
paste
Advertisement
prefrontalvortex

theano_lazylinker_fail

prefrontalvortex
Feb 23rd, 2017
378
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
C 47.48 KB | None | 0 0
raw download clone embed print report
  1. >>> import theano
  2. ===============================
  3. 00001   #include <Python.h>
  4. 00002   #include "theano_mod_helper.h"
  5. 00003   #include "structmember.h"
  6. 00004   #include <sys/time.h>
  7. 00005  
  8. 00006   #if PY_VERSION_HEX >= 0x03000000
  9. 00007   #include "numpy/npy_3kcompat.h"
  10. 00008   #define PyCObject_AsVoidPtr  NpyCapsule_AsVoidPtr
  11. 00009   #define PyCObject_GetDesc  NpyCapsule_GetDesc
  12. 00010   #define PyCObject_Check NpyCapsule_Check
  13. 00011   #endif
  14. 00012  
  15. 00013   #ifndef Py_TYPE
  16. 00014   #define Py_TYPE(obj) obj->ob_type
  17. 00015   #endif
  18. 00016  
  19. 00017   /**
  20. 00018  
  21. 00019   TODO:
  22. 00020   - Check max supported depth of recursion
  23. 00021   - CLazyLinker should add context information to errors caught during evaluation. Say what node we were on, add the traceback attached to the node.
  24. 00022   - Clear containers of fully-useed intermediate results if allow_gc is 1
  25. 00023   - Add timers for profiling
  26. 00024   - Add support for profiling space used.
  27. 00025  
  28. 00026  
  29. 00027     */
  30. 00028   static double pytime(const struct timeval * tv)
  31. 00029   {
  32. 00030     struct timeval t;
  33. 00031     if (!tv)
  34. 00032       {
  35. 00033         tv = &t;
  36. 00034         gettimeofday(&t, NULL);
  37. 00035       }
  38. 00036     return (double) tv->tv_sec + (double) tv->tv_usec / 1000000.0;
  39. 00037   }
  40. 00038  
  41. 00039   /**
  42. 00040     Helper routine to convert a PyList of integers to a c array of integers.
  43. 00041     */
  44. 00042   static int unpack_list_of_ssize_t(PyObject * pylist, Py_ssize_t **dst, Py_ssize_t *len,
  45. 00043                                     const char* kwname)
  46. 00044   {
  47. 00045     Py_ssize_t buflen, *buf;
  48. 00046     if (!PyList_Check(pylist))
  49. 00047       {
  50. 00048         PyErr_Format(PyExc_TypeError, "%s must be list", kwname);
  51. 00049         return -1;
  52. 00050       }
  53. 00051     assert (NULL == *dst);
  54. 00052     *len = buflen = PyList_Size(pylist);
  55. 00053     *dst = buf = (Py_ssize_t*)calloc(buflen, sizeof(Py_ssize_t));
  56. 00054     assert(buf);
  57. 00055     for (int ii = 0; ii < buflen; ++ii)
  58. 00056       {
  59. 00057         PyObject * el_i = PyList_GetItem(pylist, ii);
  60. 00058         Py_ssize_t n_i = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
  61. 00059         if (PyErr_Occurred())
  62. 00060           {
  63. 00061             free(buf);
  64. 00062             *dst = NULL;
  65. 00063             return -1;
  66. 00064           }
  67. 00065         buf[ii] = n_i;
  68. 00066       }
  69. 00067     return 0;
  70. 00068   }
  71. 00069  
  72. 00070   /**
  73. 00071  
  74. 00072     CLazyLinker
  75. 00073  
  76. 00074  
  77. 00075     */
  78. 00076   typedef struct {
  79. 00077       PyObject_HEAD
  80. 00078       /* Type-specific fields go here. */
  81. 00079       PyObject * nodes; // the python list of nodes
  82. 00080       PyObject * thunks; // python list of thunks
  83. 00081       PyObject * pre_call_clear; //list of cells to clear on call.
  84. 00082       int allow_gc;
  85. 00083       Py_ssize_t n_applies;
  86. 00084       int n_vars;    // number of variables in the graph
  87. 00085       int * var_computed; // 1 or 0 for every variable
  88. 00086       PyObject ** var_computed_cells;
  89. 00087       PyObject ** var_value_cells;
  90. 00088       Py_ssize_t **dependencies; // list of vars dependencies for GC
  91. 00089       Py_ssize_t *n_dependencies;
  92. 00090  
  93. 00091       Py_ssize_t n_output_vars;
  94. 00092       Py_ssize_t * output_vars; // variables that *must* be evaluated by call
  95. 00093  
  96. 00094       int * is_lazy; // 1 or 0 for every thunk
  97. 00095  
  98. 00096       Py_ssize_t * var_owner; // nodes[[var_owner[var_idx]]] is var[var_idx]->owner
  99. 00097       int * var_has_owner; //  1 or 0
  100. 00098  
  101. 00099       Py_ssize_t * node_n_inputs;
  102. 00100       Py_ssize_t * node_n_outputs;
  103. 00101       Py_ssize_t ** node_inputs;
  104. 00102       Py_ssize_t ** node_outputs;
  105. 00103       Py_ssize_t * node_inputs_outputs_base; // node_inputs and node_outputs point into this
  106. 00104       Py_ssize_t * node_n_prereqs;
  107. 00105       Py_ssize_t ** node_prereqs;
  108. 00106  
  109. 00107       Py_ssize_t * update_storage; // input cells to update with the last outputs in output_vars
  110. 00108       Py_ssize_t n_updates;
  111. 00109  
  112. 00110       void ** thunk_cptr_fn;
  113. 00111       void ** thunk_cptr_data;
  114. 00112       PyObject * call_times;
  115. 00113       PyObject * call_counts;
  116. 00114       int do_timing;
  117. 00115       int need_update_inputs;
  118. 00116       int position_of_error; // -1 for no error, otw the index into `thunks` that failed.
  119. 00117   } CLazyLinker;
  120. 00118  
  121. 00119  
  122. 00120   static void
  123. 00121   CLazyLinker_dealloc(PyObject* _self)
  124. 00122   {
  125. 00123     CLazyLinker* self = (CLazyLinker *) _self;
  126. 00124     free(self->thunk_cptr_fn);
  127. 00125     free(self->thunk_cptr_data);
  128. 00126  
  129. 00127     free(self->is_lazy);
  130. 00128  
  131. 00129     free(self->update_storage);
  132. 00130  
  133. 00131     if (self->node_n_prereqs)
  134. 00132       {
  135. 00133         for (int i = 0; i < self->n_applies; ++i)
  136. 00134           {
  137. 00135             free(self->node_prereqs[i]);
  138. 00136           }
  139. 00137       }
  140. 00138     free(self->node_n_prereqs);
  141. 00139     free(self->node_prereqs);
  142. 00140     free(self->node_inputs_outputs_base);
  143. 00141     free(self->node_n_inputs);
  144. 00142     free(self->node_n_outputs);
  145. 00143     free(self->node_inputs);
  146. 00144     free(self->node_outputs);
  147. 00145  
  148. 00146     if (self->dependencies)
  149. 00147       {
  150. 00148         for (int i = 0; i < self->n_vars; ++i)
  151. 00149           {
  152. 00150             free(self->dependencies[i]);
  153. 00151           }
  154. 00152         free(self->dependencies);
  155. 00153         free(self->n_dependencies);
  156. 00154       }
  157. 00155  
  158. 00156     free(self->var_owner);
  159. 00157     free(self->var_has_owner);
  160. 00158     free(self->var_computed);
  161. 00159     if (self->var_computed_cells)
  162. 00160       {
  163. 00161         for (int i = 0; i < self->n_vars; ++i)
  164. 00162           {
  165. 00163             Py_DECREF(self->var_computed_cells[i]);
  166. 00164             Py_DECREF(self->var_value_cells[i]);
  167. 00165           }
  168. 00166       }
  169. 00167     free(self->var_computed_cells);
  170. 00168     free(self->var_value_cells);
  171. 00169     free(self->output_vars);
  172. 00170  
  173. 00171     Py_XDECREF(self->nodes);
  174. 00172     Py_XDECREF(self->thunks);
  175. 00173     Py_XDECREF(self->call_times);
  176. 00174     Py_XDECREF(self->call_counts);
  177. 00175     Py_XDECREF(self->pre_call_clear);
  178. 00176     Py_TYPE(self)->tp_free((PyObject*)self);
  179. 00177   }
  180. 00178   static PyObject *
  181. 00179   CLazyLinker_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
  182. 00180   {
  183. 00181       CLazyLinker *self;
  184. 00182  
  185. 00183       self = (CLazyLinker *)type->tp_alloc(type, 0);
  186. 00184       if (self != NULL) {
  187. 00185         self->nodes = NULL;
  188. 00186         self->thunks = NULL;
  189. 00187         self->pre_call_clear = NULL;
  190. 00188  
  191. 00189         self->allow_gc = 1;
  192. 00190         self->n_applies = 0;
  193. 00191         self->n_vars = 0;
  194. 00192         self->var_computed = NULL;
  195. 00193         self->var_computed_cells = NULL;
  196. 00194         self->var_value_cells = NULL;
  197. 00195         self->dependencies = NULL;
  198. 00196         self->n_dependencies = NULL;
  199. 00197  
  200. 00198         self->n_output_vars = 0;
  201. 00199         self->output_vars = NULL;
  202. 00200  
  203. 00201         self->is_lazy = NULL;
  204. 00202  
  205. 00203         self->var_owner = NULL;
  206. 00204         self->var_has_owner = NULL;
  207. 00205  
  208. 00206         self->node_n_inputs = NULL;
  209. 00207         self->node_n_outputs = NULL;
  210. 00208         self->node_inputs = NULL;
  211. 00209         self->node_outputs = NULL;
  212. 00210         self->node_inputs_outputs_base = NULL;
  213. 00211         self->node_prereqs = NULL;
  214. 00212         self->node_n_prereqs = NULL;
  215. 00213  
  216. 00214         self->update_storage = NULL;
  217. 00215         self->n_updates = 0;
  218. 00216  
  219. 00217         self->thunk_cptr_data = NULL;
  220. 00218         self->thunk_cptr_fn = NULL;
  221. 00219         self->call_times = NULL;
  222. 00220         self->call_counts = NULL;
  223. 00221         self->do_timing = 0;
  224. 00222  
  225. 00223         self->need_update_inputs = 0;
  226. 00224         self->position_of_error = -1;
  227. 00225       }
  228. 00226       return (PyObject *)self;
  229. 00227   }
  230. 00228  
  231. 00229   static int
  232. 00230   CLazyLinker_init(CLazyLinker *self, PyObject *args, PyObject *kwds)
  233. 00231   {
  234. 00232       static char *kwlist[] = {
  235. 00233         (char*)"nodes",
  236. 00234         (char*)"thunks",
  237. 00235         (char*)"pre_call_clear",
  238. 00236         (char*)"allow_gc",
  239. 00237         (char*)"call_counts",
  240. 00238         (char*)"call_times",
  241. 00239         (char*)"compute_map_list",
  242. 00240         (char*)"storage_map_list",
  243. 00241         (char*)"base_input_output_list",
  244. 00242         (char*)"node_n_inputs",
  245. 00243         (char*)"node_n_outputs",
  246. 00244         (char*)"node_input_offset",
  247. 00245         (char*)"node_output_offset",
  248. 00246         (char*)"var_owner",
  249. 00247         (char*)"is_lazy_list",
  250. 00248         (char*)"output_vars",
  251. 00249         (char*)"node_prereqs",
  252. 00250         (char*)"node_output_size",
  253. 00251         (char*)"update_storage",
  254. 00252         (char*)"dependencies",
  255. 00253         NULL};
  256. 00254  
  257. 00255       PyObject *compute_map_list=NULL,
  258. 00256                *storage_map_list=NULL,
  259. 00257                *base_input_output_list=NULL,
  260. 00258                *node_n_inputs=NULL,
  261. 00259                *node_n_outputs=NULL,
  262. 00260                *node_input_offset=NULL,
  263. 00261                *node_output_offset=NULL,
  264. 00262                *var_owner=NULL,
  265. 00263                *is_lazy=NULL,
  266. 00264                *output_vars=NULL,
  267. 00265                *node_prereqs=NULL,
  268. 00266                *node_output_size=NULL,
  269. 00267                *update_storage=NULL,
  270. 00268                *dependencies=NULL;
  271. 00269  
  272. 00270       assert(!self->nodes);
  273. 00271       if (! PyArg_ParseTupleAndKeywords(args, kwds, "OOOiOOOOOOOOOOOOOOOO", kwlist,
  274. 00272                                         &self->nodes,
  275. 00273                                         &self->thunks,
  276. 00274                                         &self->pre_call_clear,
  277. 00275                                         &self->allow_gc,
  278. 00276                                         &self->call_counts,
  279. 00277                                         &self->call_times,
  280. 00278                                         &compute_map_list,
  281. 00279                                         &storage_map_list,
  282. 00280                                         &base_input_output_list,
  283. 00281                                         &node_n_inputs,
  284. 00282                                         &node_n_outputs,
  285. 00283                                         &node_input_offset,
  286. 00284                                         &node_output_offset,
  287. 00285                                         &var_owner,
  288. 00286                                         &is_lazy,
  289. 00287                                         &output_vars,
  290. 00288                                         &node_prereqs,
  291. 00289                                         &node_output_size,
  292. 00290                                         &update_storage,
  293. 00291                                         &dependencies
  294. 00292                                         ))
  295. 00293           return -1;
  296. 00294       Py_INCREF(self->nodes);
  297. 00295       Py_INCREF(self->thunks);
  298. 00296       Py_INCREF(self->pre_call_clear);
  299. 00297       Py_INCREF(self->call_counts);
  300. 00298       Py_INCREF(self->call_times);
  301. 00299  
  302. 00300       Py_ssize_t n_applies = PyList_Size(self->nodes);
  303. 00301  
  304. 00302       self->n_applies = n_applies;
  305. 00303       self->n_vars = PyList_Size(var_owner);
  306. 00304  
  307. 00305       if (PyList_Size(self->thunks) != n_applies) return -1;
  308. 00306       if (PyList_Size(self->call_counts) != n_applies) return -1;
  309. 00307       if (PyList_Size(self->call_times) != n_applies) return -1;
  310. 00308  
  311. 00309       // allocated and initialize thunk_cptr_data and thunk_cptr_fn
  312. 00310       if (n_applies)
  313. 00311         {
  314. 00312           self->thunk_cptr_data = (void**)calloc(n_applies, sizeof(void*));
  315. 00313           self->thunk_cptr_fn = (void**)calloc(n_applies, sizeof(void*));
  316. 00314           self->is_lazy = (int*)calloc(n_applies, sizeof(int));
  317. 00315           self->node_prereqs = (Py_ssize_t**)calloc(n_applies, sizeof(Py_ssize_t*));
  318. 00316           self->node_n_prereqs = (Py_ssize_t*)calloc(n_applies, sizeof(Py_ssize_t));
  319. 00317           assert(self->node_prereqs);
  320. 00318           assert(self->node_n_prereqs);
  321. 00319           assert(self->is_lazy);
  322. 00320           assert(self->thunk_cptr_fn);
  323. 00321           assert(self->thunk_cptr_data);
  324. 00322  
  325. 00323           for (int i = 0; i < n_applies; ++i)
  326. 00324             {
  327. 00325               PyObject * thunk = PyList_GetItem(self->thunks, i);
  328. 00326               //thunk is borrowed
  329. 00327               if (PyObject_HasAttrString(thunk, "cthunk"))
  330. 00328                 {
  331. 00329                   PyObject * cthunk = PyObject_GetAttrString(thunk, "cthunk");
  332. 00330                   //new reference
  333. 00331                   assert (cthunk && PyCObject_Check(cthunk));
  334. 00332                   self->thunk_cptr_fn[i] = PyCObject_AsVoidPtr(cthunk);
  335. 00333                   self->thunk_cptr_data[i] = PyCObject_GetDesc(cthunk);
  336. 00334                   Py_DECREF(cthunk);
  337. 00335                   // cthunk is kept alive by membership in self->thunks
  338. 00336                 }
  339. 00337  
  340. 00338               PyObject * el_i = PyList_GetItem(is_lazy, i);
  341. 00339               self->is_lazy[i] = PyNumber_AsSsize_t(el_i, NULL);
  342. 00340  
  343. 00341               /* now get the prereqs */
  344. 00342               el_i = PyList_GetItem(node_prereqs, i);
  345. 00343               assert (PyList_Check(el_i));
  346. 00344               self->node_n_prereqs[i] = PyList_Size(el_i);
  347. 00345               if (self->node_n_prereqs[i])
  348. 00346                 {
  349. 00347                   self->node_prereqs[i] = (Py_ssize_t*)malloc(
  350. 00348                                 PyList_Size(el_i)*sizeof(Py_ssize_t));
  351. 00349                   for (int j = 0; j < PyList_Size(el_i); ++j)
  352. 00350                     {
  353. 00351                       PyObject * el_ij = PyList_GetItem(el_i, j);
  354. 00352                       Py_ssize_t N = PyNumber_AsSsize_t(el_ij, PyExc_IndexError);
  355. 00353                       if (PyErr_Occurred())
  356. 00354                         return -1;
  357. 00355                       // N < n. variables
  358. 00356                       assert(N < PyList_Size(var_owner));
  359. 00357                       self->node_prereqs[i][j] = N;
  360. 00358                     }
  361. 00359                 }
  362. 00360             }
  363. 00361         }
  364. 00362       if (PyList_Check(base_input_output_list))
  365. 00363         {
  366. 00364           Py_ssize_t n_inputs_outputs_base = PyList_Size(base_input_output_list);
  367. 00365           self->node_inputs_outputs_base = (Py_ssize_t*)calloc(n_inputs_outputs_base,sizeof(Py_ssize_t));
  368. 00366           assert(self->node_inputs_outputs_base);
  369. 00367           for (int i = 0; i < n_inputs_outputs_base; ++i)
  370. 00368             {
  371. 00369               PyObject *el_i = PyList_GetItem(base_input_output_list, i);
  372. 00370               Py_ssize_t idx = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
  373. 00371               if (PyErr_Occurred()) return -1;
  374. 00372               self->node_inputs_outputs_base[i] = idx;
  375. 00373             }
  376. 00374           self->node_n_inputs = (Py_ssize_t*)calloc(n_applies,sizeof(Py_ssize_t));
  377. 00375           assert(self->node_n_inputs);
  378. 00376           self->node_n_outputs = (Py_ssize_t*)calloc(n_applies,sizeof(Py_ssize_t));
  379. 00377           assert(self->node_n_outputs);
  380. 00378           self->node_inputs = (Py_ssize_t**)calloc(n_applies,sizeof(Py_ssize_t*));
  381. 00379           assert(self->node_inputs);
  382. 00380           self->node_outputs = (Py_ssize_t**)calloc(n_applies,sizeof(Py_ssize_t*));
  383. 00381           assert(self->node_outputs);
  384. 00382           for (int i = 0; i < n_applies; ++i)
  385. 00383             {
  386. 00384               Py_ssize_t N;
  387. 00385               N = PyNumber_AsSsize_t(PyList_GetItem(node_n_inputs, i),PyExc_IndexError);
  388. 00386               if (PyErr_Occurred()) return -1;
  389. 00387               assert (N <= n_inputs_outputs_base);
  390. 00388               self->node_n_inputs[i] = N;
  391. 00389               N = PyNumber_AsSsize_t(PyList_GetItem(node_n_outputs, i),PyExc_IndexError);
  392. 00390               if (PyErr_Occurred()) return -1;
  393. 00391               assert (N <= n_inputs_outputs_base);
  394. 00392               self->node_n_outputs[i] = N;
  395. 00393               N = PyNumber_AsSsize_t(PyList_GetItem(node_input_offset, i),PyExc_IndexError);
  396. 00394               if (PyErr_Occurred()) return -1;
  397. 00395               assert (N <= n_inputs_outputs_base);
  398. 00396               self->node_inputs[i] = &self->node_inputs_outputs_base[N];
  399. 00397               N = PyNumber_AsSsize_t(PyList_GetItem(node_output_offset, i),PyExc_IndexError);
  400. 00398               if (PyErr_Occurred()) return -1;
  401. 00399               assert (N <= n_inputs_outputs_base);
  402. 00400               self->node_outputs[i] = &self->node_inputs_outputs_base[N];
  403. 00401             }
  404. 00402         }
  405. 00403       else
  406. 00404         {
  407. 00405           PyErr_SetString(PyExc_TypeError, "base_input_output_list must be list");
  408. 00406           return -1;
  409. 00407         }
  410. 00408  
  411. 00409       // allocation for var_owner
  412. 00410       if (PyList_Check(var_owner))
  413. 00411         {
  414. 00412           self->var_owner = (Py_ssize_t*)calloc(self->n_vars,sizeof(Py_ssize_t));
  415. 00413           self->var_has_owner = (int*)calloc(self->n_vars,sizeof(int));
  416. 00414           self->var_computed = (int*)calloc(self->n_vars,sizeof(int));
  417. 00415           self->var_computed_cells = (PyObject**)calloc(self->n_vars,sizeof(PyObject*));
  418. 00416           self->var_value_cells = (PyObject**)calloc(self->n_vars,sizeof(PyObject*));
  419. 00417           for (int i = 0; i < self->n_vars; ++i)
  420. 00418             {
  421. 00419               PyObject * el_i = PyList_GetItem(var_owner, i);
  422. 00420               if (el_i == Py_None)
  423. 00421                 {
  424. 00422                   self->var_has_owner[i] = 0;
  425. 00423                 }
  426. 00424               else
  427. 00425                 {
  428. 00426                   Py_ssize_t N = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
  429. 00427                   if (PyErr_Occurred()) return -1;
  430. 00428                   assert (N <= n_applies);
  431. 00429                   self->var_owner[i] = N;
  432. 00430                   self->var_has_owner[i] = 1;
  433. 00431                 }
  434. 00432               self->var_computed_cells[i] = PyList_GetItem(compute_map_list, i);
  435. 00433               Py_INCREF(self->var_computed_cells[i]);
  436. 00434               self->var_value_cells[i] = PyList_GetItem(storage_map_list, i);
  437. 00435               Py_INCREF(self->var_value_cells[i]);
  438. 00436             }
  439. 00437         }
  440. 00438       else
  441. 00439         {
  442. 00440           PyErr_SetString(PyExc_TypeError, "var_owner must be list");
  443. 00441           return -1;
  444. 00442         }
  445. 00443  
  446. 00444       if (dependencies != Py_None)
  447. 00445         {
  448. 00446           self->dependencies = (Py_ssize_t**)calloc(self->n_vars, sizeof(Py_ssize_t *));
  449. 00447           self->n_dependencies = (Py_ssize_t*)calloc(self->n_vars, sizeof(Py_ssize_t));
  450. 00448           assert(self->dependencies);
  451. 00449           assert(self->n_dependencies);
  452. 00450  
  453. 00451           for (int i = 0; i < self->n_vars; ++i)
  454. 00452             {
  455. 00453               PyObject *tmp = PyList_GetItem(dependencies, i);
  456. 00454               // refcounting - tmp is borrowed
  457. 00455               if (unpack_list_of_ssize_t(tmp, &self->dependencies[i], &self->n_dependencies[i],
  458. 00456                                          "dependencies"))
  459. 00457                 return -1;
  460. 00458             }
  461. 00459         }
  462. 00460  
  463. 00461       if (unpack_list_of_ssize_t(output_vars, &self->output_vars, &self->n_output_vars,
  464. 00462                                  "output_vars"))
  465. 00463         return -1;
  466. 00464       for (int i = 0; i < self->n_output_vars; ++i)
  467. 00465         {
  468. 00466           assert(self->output_vars[i] < self->n_vars);
  469. 00467         }
  470. 00468       if (unpack_list_of_ssize_t(update_storage, &self->update_storage, &self->n_updates,
  471. 00469                                  "updates_storage"))
  472. 00470         return -1;
  473. 00471       return 0;
  474. 00472   }
  475. 00473   static void set_position_of_error(CLazyLinker * self, int owner_idx)
  476. 00474   {
  477. 00475     if (self->position_of_error == -1)
  478. 00476       {
  479. 00477         self->position_of_error = owner_idx;
  480. 00478       }
  481. 00479   }
  482. 00480   static PyObject * pycall(CLazyLinker * self, Py_ssize_t node_idx, int verbose)
  483. 00481   {
  484. 00482     // call thunk to see which inputs it wants
  485. 00483     PyObject * thunk = PyList_GetItem(self->thunks, node_idx);
  486. 00484     // refcounting - thunk is borrowed
  487. 00485     PyObject * rval = NULL;
  488. 00486     if (self->do_timing)
  489. 00487       {
  490. 00488         double t0 = pytime(NULL);
  491. 00489         if (verbose) fprintf(stderr, "calling via Python (node %i)\n", (int)node_idx);
  492. 00490         rval = PyObject_CallObject(thunk, NULL);
  493. 00491         if (rval)
  494. 00492           {
  495. 00493             double t1 = pytime(NULL);
  496. 00494             double ti = PyFloat_AsDouble(
  497. 00495                            PyList_GetItem(self->call_times, node_idx));
  498. 00496             PyList_SetItem(self->call_times, node_idx,
  499. 00497                            PyFloat_FromDouble(t1 - t0 + ti));
  500. 00498             PyObject * count = PyList_GetItem(self->call_counts, node_idx);
  501. 00499             long icount = PyInt_AsLong(count);
  502. 00500             PyList_SetItem(self->call_counts, node_idx,
  503. 00501                            PyInt_FromLong(icount + 1));
  504. 00502         }
  505. 00503       }
  506. 00504     else
  507. 00505       {
  508. 00506         if (verbose)
  509. 00507           {
  510. 00508             fprintf(stderr, "calling via Python (node %i)\n", (int)node_idx);
  511. 00509           }
  512. 00510         rval = PyObject_CallObject(thunk, NULL);
  513. 00511       }
  514. 00512     return rval;
  515. 00513   }
  516. 00514   static int c_call(CLazyLinker * self, Py_ssize_t node_idx, int verbose)
  517. 00515   {
  518. 00516     void * ptr_addr = self->thunk_cptr_fn[node_idx];
  519. 00517     int (*fn)(void*) = (int (*)(void*))(ptr_addr);
  520. 00518     if (verbose) fprintf(stderr, "calling non-lazy shortcut (node %i)\n", (int)node_idx);
  521. 00519     int err = 0;
  522. 00520     if (self->do_timing)
  523. 00521       {
  524. 00522         double t0 = pytime(NULL);
  525. 00523         err = fn(self->thunk_cptr_data[node_idx]);
  526. 00524         double t1 = pytime(NULL);
  527. 00525         double ti = PyFloat_AsDouble(PyList_GetItem(self->call_times, node_idx));
  528. 00526         PyList_SetItem(self->call_times, node_idx, PyFloat_FromDouble(t1 - t0 + ti));
  529. 00527         PyObject * count = PyList_GetItem(self->call_counts, node_idx);
  530. 00528         long icount = PyInt_AsLong(count);
  531. 00529         PyList_SetItem(self->call_counts, node_idx, PyInt_FromLong(icount+1));
  532. 00530       }
  533. 00531     else
  534. 00532       {
  535. 00533         err = fn(self->thunk_cptr_data[node_idx]);
  536. 00534       }
  537. 00535  
  538. 00536     if (err)
  539. 00537       {
  540. 00538         // cast the argument to a PyList (as described near line 226 of cc.py)
  541. 00539         PyObject * __ERROR = ((PyObject**)self->thunk_cptr_data[node_idx])[0];
  542. 00540         assert (PyList_Check(__ERROR));
  543. 00541         assert (PyList_Size(__ERROR) == 3);
  544. 00542         PyObject * err_type = PyList_GetItem(__ERROR, 0); //stolen ref
  545. 00543         PyObject * err_msg = PyList_GetItem(__ERROR, 1); //stolen ref
  546. 00544         PyObject * err_trace = PyList_GetItem(__ERROR, 2); //stolen ref
  547. 00545         PyList_SET_ITEM(__ERROR, 0, Py_None); Py_INCREF(Py_None); //clobbers old ref
  548. 00546         PyList_SET_ITEM(__ERROR, 1, Py_None); Py_INCREF(Py_None); //clobbers old ref
  549. 00547         PyList_SET_ITEM(__ERROR, 2, Py_None); Py_INCREF(Py_None); //clobbers old ref
  550. 00548  
  551. 00549         assert(!PyErr_Occurred()); // because CLinker hid the exception in __ERROR aka data
  552. 00550         PyErr_Restore(err_type, err_msg, err_trace); //steals refs to args
  553. 00551       }
  554. 00552     if (err) set_position_of_error(self, node_idx);
  555. 00553     return err;
  556. 00554   }
  557. 00555   static
  558. 00556   int lazy_rec_eval(CLazyLinker * self, Py_ssize_t var_idx, PyObject*one, PyObject*zero)
  559. 00557   {
  560. 00558     PyObject *rval = NULL;
  561. 00559     int verbose = 0;
  562. 00560     int err = 0;
  563. 00561  
  564. 00562     if (verbose) fprintf(stderr, "lazy_rec computing %i\n", (int)var_idx);
  565. 00563  
  566. 00564     if (self->var_computed[var_idx] || !self->var_has_owner[var_idx])
  567. 00565       return 0;
  568. 00566  
  569. 00567     Py_ssize_t owner_idx = self->var_owner[var_idx];
  570. 00568  
  571. 00569     // STEP 1: compute the pre-requirements of the node
  572. 00570     // Includes input nodes for non-lazy ops.
  573. 00571     for (int i = 0; i < self->node_n_prereqs[owner_idx]; ++i)
  574. 00572       {
  575. 00573         Py_ssize_t prereq_idx = self->node_prereqs[owner_idx][i];
  576. 00574         if (!self->var_computed[prereq_idx])
  577. 00575           {
  578. 00576             err = lazy_rec_eval(self, prereq_idx, one, zero);
  579. 00577             if (err) return err;
  580. 00578           }
  581. 00579         assert (self->var_computed[prereq_idx]);
  582. 00580       }
  583. 00581  
  584. 00582     // STEP 2: compute the node itself
  585. 00583     if (self->is_lazy[owner_idx])
  586. 00584       {
  587. 00585         // update the compute_map cells corresponding to the inputs of this thunk
  588. 00586         for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
  589. 00587           {
  590. 00588             int in_idx = self->node_inputs[owner_idx][i];
  591. 00589             if (self->var_computed[in_idx])
  592. 00590               {
  593. 00591                 Py_INCREF(one);
  594. 00592                 err = PyList_SetItem(self->var_computed_cells[in_idx], 0, one);
  595. 00593               }
  596. 00594             else
  597. 00595               {
  598. 00596                 Py_INCREF(zero);
  599. 00597                 err = PyList_SetItem(self->var_computed_cells[in_idx], 0, zero);
  600. 00598               }
  601. 00599             if (err) goto fail;
  602. 00600           }
  603. 00601  
  604. 00602         rval = pycall(self, owner_idx, verbose);
  605. 00603         // refcounting - rval is new ref
  606. 00604         //TODO: to prevent infinite loops
  607. 00605         // - consider check that a thunk does not ask for an input that is already computed
  608. 00606         if (rval == NULL)
  609. 00607           {
  610. 00608             assert (PyErr_Occurred());
  611. 00609             err = 1;
  612. 00610             goto fail;
  613. 00611           }
  614. 00612  
  615. 00613         //update the computed-ness of any output cells
  616. 00614         for (int i = 0; i < self->node_n_outputs[owner_idx]; ++i)
  617. 00615           {
  618. 00616             int out_idx = self->node_outputs[owner_idx][i];
  619. 00617             PyObject * el_i = PyList_GetItem(self->var_computed_cells[out_idx], 0);
  620. 00618             Py_ssize_t N = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
  621. 00619             if (PyErr_Occurred())
  622. 00620               {
  623. 00621                 err = -1;
  624. 00622                 goto pyfail;
  625. 00623               }
  626. 00624             assert (N==0 || N==1);
  627. 00625             self->var_computed[out_idx] = N;
  628. 00626           }
  629. 00627         if (!self->var_computed[var_idx])
  630. 00628           {
  631. 00629             /*
  632. 00630              * If self is not computed after the call, this means that some
  633. 00631              * inputs are needed.  Compute the ones on the returned list
  634. 00632              * and try to compute the current node again (with recursive call).
  635. 00633              * This allows a node to request more nodes more than once before
  636. 00634              * finally yielding a result.
  637. 00635              */
  638. 00636             if (!PyList_Check(rval))
  639. 00637               {
  640. 00638                 //TODO: More helpful error to help find *which node* made this
  641. 00639                 // bad thunk
  642. 00640                 PyErr_SetString(PyExc_TypeError,
  643. 00641                                 "lazy thunk should return a list");
  644. 00642                 err = 1;
  645. 00643                 goto pyfail;
  646. 00644               }
  647. 00645  
  648. 00646             if (!PyList_Size(rval))
  649. 00647               {
  650. 00648                 PyErr_SetString(PyExc_ValueError,
  651. 00649                                 "lazy thunk returned empty list without computing output");
  652. 00650                 err = 1;
  653. 00651                 goto pyfail;
  654. 00652               }
  655. 00653  
  656. 00654             for (int i = 0; i < PyList_Size(rval); ++i)
  657. 00655               {
  658. 00656                 PyObject * el_i = PyList_GetItem(rval, i);
  659. 00657                 Py_ssize_t N = PyNumber_AsSsize_t(el_i, PyExc_IndexError);
  660. 00658                 if (PyErr_Occurred())
  661. 00659                   {
  662. 00660                     err = 1;
  663. 00661                     goto pyfail;
  664. 00662                   }
  665. 00663                 assert (N <= self->node_n_inputs[owner_idx]);
  666. 00664                 Py_ssize_t input_idx = self->node_inputs[owner_idx][N];
  667. 00665                 err = lazy_rec_eval(self, input_idx, one, zero);
  668. 00666                 if (err) goto pyfail;
  669. 00667               }
  670. 00668  
  671. 00669             Py_DECREF(rval);
  672. 00670             /*
  673. 00671              * We intentionally skip all the end-of-function processing
  674. 00672              * (mark outputs, GC) as it will be performed by the call
  675. 00673              * that actually manages to compute the result.
  676. 00674              */
  677. 00675             return lazy_rec_eval(self, var_idx, one, zero);
  678. 00676           }
  679. 00677  
  680. 00678         Py_DECREF(rval);
  681. 00679       }
  682. 00680     else //owner is not a lazy op. Ensure all intputs are evaluated.
  683. 00681       {
  684. 00682         // loop over inputs to owner
  685. 00683         // call lazy_rec_eval on each one that is not computed.
  686. 00684         // if there's an error, pass it up the stack
  687. 00685         for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
  688. 00686           {
  689. 00687             Py_ssize_t input_idx = self->node_inputs[owner_idx][i];
  690. 00688             if (!self->var_computed[input_idx])
  691. 00689               {
  692. 00690                 err = lazy_rec_eval(self, input_idx, one, zero);
  693. 00691                 if (err) return err;
  694. 00692               }
  695. 00693             assert (self->var_computed[input_idx]);
  696. 00694           }
  697. 00695  
  698. 00696         // call the thunk for this owner.
  699. 00697         if (self->thunk_cptr_fn[owner_idx])
  700. 00698           {
  701. 00699             err = c_call(self, owner_idx, verbose);
  702. 00700             if (err) goto fail;
  703. 00701           }
  704. 00702         else
  705. 00703           {
  706. 00704             rval = pycall(self, owner_idx, verbose);
  707. 00705             //rval is new ref
  708. 00706             if (rval) //pycall returned normally (no exception)
  709. 00707               {
  710. 00708                 if (rval == Py_None)
  711. 00709                   {
  712. 00710                     Py_DECREF(rval); //ignore a return of None
  713. 00711                   }
  714. 00712                 else if (PyList_Check(rval))
  715. 00713                   {
  716. 00714                     PyErr_SetString(PyExc_TypeError,
  717. 00715                                     "non-lazy thunk should return None, not list");
  718. 00716                     err = 1;
  719. 00717                     goto pyfail;
  720. 00718                   }
  721. 00719                 else // don't know what it returned, but it wasn't right.
  722. 00720                   {
  723. 00721                     PyErr_SetObject(PyExc_TypeError, rval);
  724. 00722                     err = 1;
  725. 00723                     // We don't release rval since we put it in the error above
  726. 00724                     goto fail;
  727. 00725                   }
  728. 00726               }
  729. 00727             else // pycall returned NULL (internal error)
  730. 00728               {
  731. 00729                 err = 1;
  732. 00730                 goto fail;
  733. 00731               }
  734. 00732           }
  735. 00733       }
  736. 00734  
  737. 00735     // loop over all outputs and mark them as computed
  738. 00736     for (int i = 0; i < self->node_n_outputs[owner_idx]; ++i)
  739. 00737       {
  740. 00738         self->var_computed[self->node_outputs[owner_idx][i]] = 1;
  741. 00739       }
  742. 00740  
  743. 00741     // Free vars that are not needed anymore
  744. 00742     if (self->allow_gc)
  745. 00743       {
  746. 00744         for (int i = 0; i < self->node_n_inputs[owner_idx]; ++i)
  747. 00745           {
  748. 00746             int cleanup = 1;
  749. 00747             Py_ssize_t i_idx = self->node_inputs[owner_idx][i];
  750. 00748             if (!self->var_has_owner[i_idx])
  751. 00749               continue;
  752. 00750  
  753. 00751             for (int j = 0; j < self->n_output_vars; ++j)
  754. 00752               {
  755. 00753                 if (i_idx == self->output_vars[j])
  756. 00754                   {
  757. 00755                     cleanup = 0;
  758. 00756                     break;
  759. 00757                   }
  760. 00758               }
  761. 00759             if (!cleanup) continue;
  762. 00760  
  763. 00761             for (int j = 0; j < self->n_dependencies[i_idx]; ++j)
  764. 00762               {
  765. 00763                 if (!self->var_computed[self->dependencies[i_idx][j]])
  766. 00764                   {
  767. 00765                     cleanup = 0;
  768. 00766                     break;
  769. 00767                   }
  770. 00768               }
  771. 00769             if (!cleanup) continue;
  772. 00770  
  773. 00771             Py_INCREF(Py_None);
  774. 00772             err = PyList_SetItem(self->var_value_cells[i_idx], 0, Py_None);
  775. 00773   //See the Stack gc implementation for why we change it to 2 and not 0.
  776. 00774             self->var_computed[i_idx] = 2;
  777. 00775             if (err) goto fail;
  778. 00776           }
  779. 00777       }
  780. 00778  
  781. 00779     return 0;
  782. 00780    pyfail:
  783. 00781     Py_DECREF(rval);
  784. 00782    fail:
  785. 00783     set_position_of_error(self, owner_idx);
  786. 00784     return err;
  787. 00785   }
  788. 00786  
  789. 00787   static PyObject *
  790. 00788   CLazyLinker_call(PyObject *_self, PyObject *args, PyObject *kwds)
  791. 00789   {
  792. 00790     CLazyLinker * self = (CLazyLinker*)_self;
  793. 00791     static char *kwlist[] = {
  794. 00792       (char*)"time_thunks",
  795. 00793       (char *)"n_calls",
  796. 00794       NULL};
  797. 00795     int n_calls=1;
  798. 00796     if (! PyArg_ParseTupleAndKeywords(args, kwds, "|ii", kwlist,
  799. 00797                                       &self->do_timing,
  800. 00798                                       &n_calls))
  801. 00799       return NULL;
  802. 00800     int err = 0;
  803. 00801     self->position_of_error = -1;
  804. 00802     // create constants used to fill the var_compute_cells
  805. 00803     PyObject * one = PyInt_FromLong(1);
  806. 00804     PyObject * zero = PyInt_FromLong(0);
  807. 00805  
  808. 00806     // pre-allocate our return value
  809. 00807     Py_INCREF(Py_None);
  810. 00808     PyObject * rval = Py_None;
  811. 00809     //clear storage of pre_call_clear elements
  812. 00810     for (int call_i = 0; call_i < n_calls && (!err); ++call_i)
  813. 00811       {
  814. 00812         Py_ssize_t n_pre_call_clear = PyList_Size(self->pre_call_clear);
  815. 00813         assert(PyList_Check(self->pre_call_clear));
  816. 00814         for (int i = 0; i < n_pre_call_clear; ++i)
  817. 00815           {
  818. 00816             PyObject * el_i = PyList_GetItem(self->pre_call_clear, i);
  819. 00817             Py_INCREF(Py_None);
  820. 00818             PyList_SetItem(el_i, 0, Py_None);
  821. 00819           }
  822. 00820         //clear the computed flag out of all non-input vars
  823. 00821         for (int i = 0; i < self->n_vars; ++i)
  824. 00822           {
  825. 00823             self->var_computed[i] = !self->var_has_owner[i];
  826. 00824             if (self->var_computed[i])
  827. 00825               {
  828. 00826                 Py_INCREF(one);
  829. 00827                 PyList_SetItem(self->var_computed_cells[i], 0, one);
  830. 00828               }
  831. 00829             else
  832. 00830               {
  833. 00831                 Py_INCREF(zero);
  834. 00832                 PyList_SetItem(self->var_computed_cells[i], 0, zero);
  835. 00833               }
  836. 00834           }
  837. 00835  
  838. 00836         for (int i = 0; i < self->n_output_vars && (!err); ++i)
  839. 00837           {
  840. 00838             err = lazy_rec_eval(self, self->output_vars[i], one, zero);
  841. 00839           }
  842. 00840  
  843. 00841         if (!err)
  844. 00842           {
  845. 00843             // save references to outputs prior to updating storage containers
  846. 00844             assert (self->n_output_vars >= self->n_updates);
  847. 00845             Py_DECREF(rval);
  848. 00846             rval = PyList_New(self->n_output_vars);
  849. 00847             for (int i = 0; i < (self->n_output_vars); ++i)
  850. 00848               {
  851. 00849                 Py_ssize_t src = self->output_vars[i];
  852. 00850                 PyObject * item = PyList_GetItem(self->var_value_cells[src], 0);
  853. 00851                 if (self->var_computed[src] != 1)
  854. 00852                   {
  855. 00853                     err = 1;
  856. 00854                     PyErr_Format(PyExc_AssertionError,
  857. 00855                                  "The compute map of output %d should contain "
  858. 00856                                  "1 at the end of execution, not %d.",
  859. 00857                                  i, self->var_computed[src]);
  860. 00858                     break;
  861. 00859                   }
  862. 00860                 Py_INCREF(item);
  863. 00861                 PyList_SetItem(rval, i, item);
  864. 00862               }
  865. 00863           }
  866. 00864  
  867. 00865         if (!err)
  868. 00866           {
  869. 00867             // Update the inputs that have an update rule
  870. 00868             for (int i = 0; i < self->n_updates; ++i)
  871. 00869               {
  872. 00870                 PyObject* tmp = PyList_GetItem(rval, self->n_output_vars - self->n_updates + i);
  873. 00871                 Py_INCREF(tmp);
  874. 00872                 Py_ssize_t dst = self->update_storage[i];
  875. 00873                 PyList_SetItem(self->var_value_cells[dst], 0, tmp);
  876. 00874               }
  877. 00875           }
  878. 00876       }
  879. 00877  
  880. 00878     /*
  881. 00879       Clear everything that is left and not an output.  This is needed
  882. 00880       for lazy evaluation since the current GC algo is too conservative
  883. 00881       with lazy graphs.
  884. 00882     */
  885. 00883     if (self->allow_gc && !err)
  886. 00884       {
  887. 00885         for (Py_ssize_t i = 0; i < self->n_vars; ++i)
  888. 00886           {
  889. 00887             int do_cleanup = 1;
  890. 00888             if (!self->var_has_owner[i] || !self->var_computed[i])
  891. 00889               continue;
  892. 00890             for (int j = 0; j < self->n_output_vars; ++j)
  893. 00891               {
  894. 00892                 if (i == self->output_vars[j])
  895. 00893                   {
  896. 00894                     do_cleanup = 0;
  897. 00895                     break;
  898. 00896                   }
  899. 00897               }
  900. 00898             if (!do_cleanup)
  901. 00899               continue;
  902. 00900             Py_INCREF(Py_None);
  903. 00901             PyList_SetItem(self->var_value_cells[i], 0, Py_None);
  904. 00902           }
  905. 00903       }
  906. 00904     Py_DECREF(one);
  907. 00905     Py_DECREF(zero);
  908. 00906     if (err)
  909. 00907       {
  910. 00908         Py_DECREF(rval);
  911. 00909         return NULL;
  912. 00910       }
  913. 00911     return rval;
  914. 00912   }
  915. 00913  
  916. 00914   #if 0
  917. 00915   static PyMethodDef CLazyLinker_methods[] = {
  918. 00916       {
  919. 00917         //"name", (PyCFunction)CLazyLinker_accept, METH_VARARGS, "Return the name, combining the first and last name"
  920. 00918       },
  921. 00919       {NULL}  /* Sentinel */
  922. 00920   };
  923. 00921   #endif
  924. 00922  
  925. 00923  
  926. 00924   static PyObject *
  927. 00925   CLazyLinker_get_allow_gc(CLazyLinker *self, void *closure)
  928. 00926   {
  929. 00927       return PyBool_FromLong(self->allow_gc);
  930. 00928   }
  931. 00929  
  932. 00930   static int
  933. 00931   CLazyLinker_set_allow_gc(CLazyLinker *self, PyObject *value, void *closure)
  934. 00932   {
  935. 00933     if(!PyBool_Check(value))
  936. 00934       return -1;
  937. 00935  
  938. 00936     if (value == Py_True)
  939. 00937       self->allow_gc = true;
  940. 00938     else
  941. 00939       self->allow_gc = false;
  942. 00940     return 0;
  943. 00941   }
  944. 00942  
  945. 00943   static PyGetSetDef CLazyLinker_getset[] = {
  946. 00944     {(char*)"allow_gc",
  947. 00945      (getter)CLazyLinker_get_allow_gc,
  948. 00946      (setter)CLazyLinker_set_allow_gc,
  949. 00947      (char*)"do this function support allow_gc",
  950. 00948      NULL},
  951. 00949     {NULL, NULL, NULL, NULL}  /* Sentinel */
  952. 00950   };
  953. 00951   static PyMemberDef CLazyLinker_members[] = {
  954. 00952       {(char*)"nodes", T_OBJECT_EX, offsetof(CLazyLinker, nodes), 0,
  955. 00953        (char*)"list of nodes"},
  956. 00954       {(char*)"thunks", T_OBJECT_EX, offsetof(CLazyLinker, thunks), 0,
  957. 00955        (char*)"list of thunks in program"},
  958. 00956       {(char*)"call_counts", T_OBJECT_EX, offsetof(CLazyLinker, call_counts), 0,
  959. 00957        (char*)"number of calls of each thunk"},
  960. 00958       {(char*)"call_times", T_OBJECT_EX, offsetof(CLazyLinker, call_times), 0,
  961. 00959        (char*)"total runtime in each thunk"},
  962. 00960       {(char*)"position_of_error", T_INT, offsetof(CLazyLinker, position_of_error), 0,
  963. 00961        (char*)"position of failed thunk"},
  964. 00962       {(char*)"time_thunks", T_INT, offsetof(CLazyLinker, do_timing), 0,
  965. 00963        (char*)"bool: nonzero means call will time thunks"},
  966. 00964       {(char*)"need_update_inputs", T_INT, offsetof(CLazyLinker, need_update_inputs), 0,
  967. 00965        (char*)"bool: nonzero means Function.__call__ must implement update mechanism"},
  968. 00966       {NULL}  /* Sentinel */
  969. 00967   };
  970. 00968  
  971. 00969   static PyTypeObject lazylinker_ext_CLazyLinkerType = {
  972. 00970   #if defined(NPY_PY3K)
  973. 00971       PyVarObject_HEAD_INIT(NULL, 0)
  974. 00972   #else
  975. 00973       PyObject_HEAD_INIT(NULL)
  976. 00974       0,                         /*ob_size*/
  977. 00975   #endif
  978. 00976       "lazylinker_ext.CLazyLinker",             /*tp_name*/
  979. 00977       sizeof(CLazyLinker), /*tp_basicsize*/
  980. 00978       0,                         /*tp_itemsize*/
  981. 00979       CLazyLinker_dealloc,       /*tp_dealloc*/
  982. 00980       0,                         /*tp_print*/
  983. 00981       0,                         /*tp_getattr*/
  984. 00982       0,                         /*tp_setattr*/
  985. 00983       0,                         /*tp_compare*/
  986. 00984       0,                         /*tp_repr*/
  987. 00985       0,                         /*tp_as_number*/
  988. 00986       0,                         /*tp_as_sequence*/
  989. 00987       0,                         /*tp_as_mapping*/
  990. 00988       0,                         /*tp_hash */
  991. 00989       CLazyLinker_call,          /*tp_call*/
  992. 00990       0,                         /*tp_str*/
  993. 00991       0,                         /*tp_getattro*/
  994. 00992       0,                         /*tp_setattro*/
  995. 00993       0,                         /*tp_as_buffer*/
  996. 00994       Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,        /*tp_flags*/
  997. 00995       "CLazyLinker object",      /* tp_doc */
  998. 00996       0,                         /* tp_traverse */
  999. 00997       0,                         /* tp_clear */
  1000. 00998       0,                         /* tp_richcompare */
  1001. 00999       0,                         /* tp_weaklistoffset */
  1002. 01000       0,                         /* tp_iter */
  1003. 01001       0,                         /* tp_iternext */
  1004. 01002       0,//CLazyLinker_methods,       /* tp_methods */
  1005. 01003       CLazyLinker_members,       /* tp_members */
  1006. 01004       CLazyLinker_getset,        /* tp_getset */
  1007. 01005       0,                         /* tp_base */
  1008. 01006       0,                         /* tp_dict */
  1009. 01007       0,                         /* tp_descr_get */
  1010. 01008       0,                         /* tp_descr_set */
  1011. 01009       0,                         /* tp_dictoffset */
  1012. 01010       (initproc)CLazyLinker_init,/* tp_init */
  1013. 01011       0,                         /* tp_alloc */
  1014. 01012       CLazyLinker_new,           /* tp_new */
  1015. 01013   };
  1016. 01014  
  1017. 01015   static PyObject * get_version(PyObject *dummy, PyObject *args)
  1018. 01016   {
  1019. 01017     PyObject *result = PyFloat_FromDouble(0.21);
  1020. 01018     return result;
  1021. 01019   }
  1022. 01020  
  1023. 01021   static PyMethodDef lazylinker_ext_methods[] = {
  1024. 01022     {"get_version",  get_version, METH_VARARGS, "Get extension version."},
  1025. 01023     {NULL, NULL, 0, NULL}        /* Sentinel */
  1026. 01024   };
  1027. 01025  
  1028. 01026   #if defined(NPY_PY3K)
  1029. 01027   static struct PyModuleDef moduledef = {
  1030. 01028           PyModuleDef_HEAD_INIT,
  1031. 01029           "lazylinker_ext",
  1032. 01030           NULL,
  1033. 01031           -1,
  1034. 01032           lazylinker_ext_methods,
  1035. 01033           NULL,
  1036. 01034           NULL,
  1037. 01035           NULL,
  1038. 01036           NULL
  1039. 01037   };
  1040. 01038   #endif
  1041. 01039   #if defined(NPY_PY3K)
  1042. 01040   #define RETVAL m
  1043. 01041   PyMODINIT_FUNC
  1044. 01042   PyInit_lazylinker_ext(void) {
  1045. 01043   #else
  1046. 01044   #define RETVAL
  1047. 01045   PyMODINIT_FUNC
  1048. 01046   initlazylinker_ext(void)
  1049. 01047   {
  1050. 01048   #endif
  1051. 01049       PyObject* m;
  1052. 01050  
  1053. 01051       lazylinker_ext_CLazyLinkerType.tp_new = PyType_GenericNew;
  1054. 01052       if (PyType_Ready(&lazylinker_ext_CLazyLinkerType) < 0)
  1055. 01053           return RETVAL;
  1056. 01054   #if defined(NPY_PY3K)
  1057. 01055       m = PyModule_Create(&moduledef);
  1058. 01056   #else
  1059. 01057       m = Py_InitModule3("lazylinker_ext", lazylinker_ext_methods,
  1060. 01058                          "Example module that creates an extension type.");
  1061. 01059   #endif
  1062. 01060       Py_INCREF(&lazylinker_ext_CLazyLinkerType);
  1063. 01061       PyModule_AddObject(m, "CLazyLinker", (PyObject *)&lazylinker_ext_CLazyLinkerType);
  1064. 01062  
  1065. 01063       return RETVAL;
  1066. 01064   }
  1067. 01065  
  1068. ===============================
  1069. Problem occurred during compilation with the command line below:
  1070. /gpfs/software/gcc-4.9.2/bin/g++ -shared -g -march=haswell -mmmx -mno-3dnow -msse -msse2 -msse3 -mssse3 -mno-sse4a -mcx16 -msahf -mmovbe -maes -mno-sha -mpclmul -mpopcnt -mabm -mno-lwp -mfma -mno-fma4 -mno-xop -mbmi -mbmi2 -mno-tbm -mavx -mavx2 -msse4.2 -msse4.1 -mlzcnt -mno-rtm -mno-hle -mrdrnd -mf16c -mfsgsbase -mno-rdseed -mno-prfchw -mno-adx -mfxsr -mxsave -mxsaveopt -mno-avx512f -mno-avx512er -mno-avx512cd -mno-avx512pf -mno-prefetchwt1 --param l1-cache-size=32 --param l1-cache-line-size=64 --param l2-cache-size=15360 -mtune=generic -DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION -m64 -fPIC -I/gpfs/home/USERNAME/anaconda3/lib/python3.5/site-packages/numpy/core/include -I/gpfs/home/USERNAME/anaconda3/include/python3.5m -I/gpfs/home/USERNAME/anaconda3/lib/python3.5/site-packages/theano/gof -fvisibility=hidden -o /gpfs/home/USERNAME/.theano/compiledir_Linux-2.6-el6.x86_64-x86_64-with-centos-6.6-Final-x86_64-3.5.2-64/lazylinker_ext/lazylinker_ext.so /gpfs/home/USERNAME/.theano/compiledir_Linux-2.6-el6.x86_64-x86_64-with-centos-6.6-Final-x86_64-3.5.2-64/lazylinker_ext/mod.cpp -L/gpfs/home/USERNAME/anaconda3/lib -lpython3.5m
  1071. In file included from /gpfs/home/USERNAME/anaconda3/include/python3.5m/pyport.h:13:0,
  1072.                  from /gpfs/home/USERNAME/anaconda3/include/python3.5m/Python.h:50,
  1073.                  from /gpfs/home/USERNAME/.theano/compiledir_Linux-2.6-el6.x86_64-x86_64-with-centos-6.6-Final-x86_64-3.5.2-64/lazylinker_ext/mod.cpp:1:
  1074. /cm/shared/apps/intel/compilers_and_libraries/2016.2.181/compiler/include/stdint.h:79:10: warning: undefining "__STDC_LIMIT_MACROS_DEFINED_BY_CLANG"
  1075.  #  undef __STDC_LIMIT_MACROS_DEFINED_BY_CLANG
  1076.           ^
  1077. /cm/shared/apps/intel/compilers_and_libraries/2016.2.181/compiler/include/stdint.h:83:10: warning: undefining "__STDC_CONSTANT_MACROS_DEFINED_BY_CLANG"
  1078.  #  undef __STDC_CONSTANT_MACROS_DEFINED_BY_CLANG
  1079.           ^
  1080. In file included from /gpfs/home/USERNAME/anaconda3/include/python3.5m/pyport.h:328:0,
  1081.                  from /gpfs/home/USERNAME/anaconda3/include/python3.5m/Python.h:50,
  1082.                  from /gpfs/home/USERNAME/.theano/compiledir_Linux-2.6-el6.x86_64-x86_64-with-centos-6.6-Final-x86_64-3.5.2-64/lazylinker_ext/mod.cpp:1:
  1083. /cm/shared/apps/intel/compilers_and_libraries/2016.2.181/compiler/include/math.h:29:3: error: #error "This Intel <math.h> is for use with only the Intel compilers!"
  1084.  # error "This Intel <math.h> is for use with only the Intel compilers!"
  1085.    ^
  1086.  
  1087. Traceback (most recent call last):
  1088.   File "/gpfs/home/USERNAME/anaconda3/lib/python3.5/site-packages/theano/gof/lazylinker_c.py", line 74, in <module>
  1089.     raise ImportError()
  1090. ImportError
  1091.  
  1092. During handling of the above exception, another exception occurred:
  1093.  
  1094. Traceback (most recent call last):
  1095.   File "/gpfs/home/USERNAME/anaconda3/lib/python3.5/site-packages/theano/gof/lazylinker_c.py", line 91, in <module>
  1096.     raise ImportError()
  1097. ImportError
  1098.  
  1099. During handling of the above exception, another exception occurred:
  1100.  
  1101. Traceback (most recent call last):
  1102.   File "<stdin>", line 1, in <module>
  1103.   File "/gpfs/home/USERNAME/anaconda3/lib/python3.5/site-packages/theano/__init__.py", line 63, in <module>
  1104.     from theano.compile import (
  1105.   File "/gpfs/home/USERNAME/anaconda3/lib/python3.5/site-packages/theano/compile/__init__.py", line 9, in <module>
  1106.     from theano.compile.function_module import *
  1107.   File "/gpfs/home/USERNAME/anaconda3/lib/python3.5/site-packages/theano/compile/function_module.py", line 22, in <module>
  1108.     import theano.compile.mode
  1109.   File "/gpfs/home/USERNAME/anaconda3/lib/python3.5/site-packages/theano/compile/mode.py", line 12, in <module>
  1110.     import theano.gof.vm
  1111.   File "/gpfs/home/USERNAME/anaconda3/lib/python3.5/site-packages/theano/gof/vm.py", line 638, in <module>
  1112.     from . import lazylinker_c
  1113.   File "/gpfs/home/USERNAME/anaconda3/lib/python3.5/site-packages/theano/gof/lazylinker_c.py", line 126, in <module>
  1114.     preargs=args)
  1115.   File "/gpfs/home/USERNAME/anaconda3/lib/python3.5/site-packages/theano/gof/cmodule.py", line 2204, in compile_str
  1116.     (status, compile_stderr.replace('\n', '. ')))
  1117. Exception: Compilation failed (return status=1): In file included from /gpfs/home/USERNAME/anaconda3/include/python3.5m/pyport.h:13:0,.                  from /gpfs/home/USERNAME/anaconda3/include/python3.5m/Python.h:50,.                  from /gpfs/home/USERNAME/.theano/compiledir_Linux-2.6-el6.x86_64-x86_64-with-centos-6.6-Final-x86_64-3.5.2-64/lazylinker_ext/mod.cpp:1:. /cm/shared/apps/intel/compilers_and_libraries/2016.2.181/compiler/include/stdint.h:79:10: warning: undefining "__STDC_LIMIT_MACROS_DEFINED_BY_CLANG".  #  undef __STDC_LIMIT_MACROS_DEFINED_BY_CLANG.           ^. /cm/shared/apps/intel/compilers_and_libraries/2016.2.181/compiler/include/stdint.h:83:10: warning: undefining "__STDC_CONSTANT_MACROS_DEFINED_BY_CLANG".  #  undef __STDC_CONSTANT_MACROS_DEFINED_BY_CLANG.           ^. In file included from /gpfs/home/USERNAME/anaconda3/include/python3.5m/pyport.h:328:0,.                  from /gpfs/home/USERNAME/anaconda3/include/python3.5m/Python.h:50,.                  from /gpfs/home/USERNAME/.theano/compiledir_Linux-2.6-el6.x86_64-x86_64-with-centos-6.6-Final-x86_64-3.5.2-64/lazylinker_ext/mod.cpp:1:. /cm/shared/apps/intel/compilers_and_libraries/2016.2.181/compiler/include/math.h:29:3: error: #error "This Intel <math.h> is for use with only the Intel compilers!".  # error "This Intel <math.h> is for use with only the Intel compilers!".    ^.
  1118. >>>
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!