API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Aug 5th, 2017
133
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 7.79 KB | None | 0 0
raw download clone embed print report
  1. {
  2. "cells": [
  3. {
  4. "cell_type": "code",
  5. "execution_count": 1,
  6. "metadata": {},
  7. "outputs": [],
  8. "source": [
  9. "%load_ext cython"
  10. ]
  11. },
  12. {
  13. "cell_type": "code",
  14. "execution_count": 2,
  15. "metadata": {
  16. "collapsed": true
  17. },
  18. "outputs": [],
  19. "source": [
  20. "import pyarrow as pa\n",
  21. "import pymapd\n",
  22. "from pymapd.shm import load_buffer"
  23. ]
  24. },
  25. {
  26. "cell_type": "code",
  27. "execution_count": 3,
  28. "metadata": {},
  29. "outputs": [
  30. {
  31. "data": {
  32. "text/plain": [
  33. "TDataFrame(sm_handle=b'\\xea]\\xbbp', sm_size=272, df_handle=b'x\\xe7\\x12\\x1d', df_size=352)"
  34. ]
  35. },
  36. "execution_count": 3,
  37. "metadata": {},
  38. "output_type": "execute_result"
  39. }
  40. ],
  41. "source": [
  42. "con = pymapd.connect(user='mapd', password='HyperInteractive', host='localhost', dbname='mapd')\n",
  43. "con\n",
  44. "\n",
  45. "self = con\n",
  46. "operation = 'select depdelay, arrdelay from flights_2008_10k limit 10;'\n",
  47. "tdf = self._client.sql_execute_df(\n",
  48. " self._session, operation, device_type=0, device_id=0, first_n=-1)\n",
  49. "tdf"
  50. ]
  51. },
  52. {
  53. "cell_type": "code",
  54. "execution_count": 4,
  55. "metadata": {},
  56. "outputs": [
  57. {
  58. "data": {
  59. "text/plain": [
  60. "array([220, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 12,\n",
  61. " 0, 22, 0, 6, 0, 5, 0, 8, 0, 12, 0, 12, 0,\n",
  62. " 0, 0, 0, 3, 2, 0, 24, 0, 0, 0, 128, 0, 0,\n",
  63. " 0, 0, 0, 0, 0, 0, 0, 10, 0, 24, 0, 12, 0,\n",
  64. " 4, 0, 8, 0, 10, 0, 0, 0, 124, 0, 0, 0, 16,\n",
  65. " 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
  66. " 0, 0, 4, 0, 0, 0, 255, 255, 255, 255, 0, 0, 0,\n",
  67. " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
  68. " 0, 0, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0, 0,\n",
  69. " 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0,\n",
  70. " 0, 0, 255, 255, 255, 255, 0, 0, 0, 0, 64, 0, 0,\n",
  71. " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
  72. " 255, 255, 255, 255, 0, 0, 0, 0, 64, 0, 0, 0, 0,\n",
  73. " 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
  74. " 0, 0, 2, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0,\n",
  75. " 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0,\n",
  76. " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
  77. " 0, 0, 0, 1, 0, 12, 0, 17, 0, 252, 255, 30, 0,\n",
  78. " 10, 0, 30, 0, 12, 0, 3, 0, 253, 255, 0, 0, 0,\n",
  79. " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
  80. " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
  81. " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
  82. " 0, 0, 254, 255, 4, 0, 29, 0, 244, 255, 11, 0, 254,\n",
  83. " 255, 10, 0, 253, 255, 245, 255, 243, 255, 0, 0, 0, 0,\n",
  84. " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
  85. " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
  86. " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], dtype=uint8)"
  87. ]
  88. },
  89. "execution_count": 4,
  90. "metadata": {},
  91. "output_type": "execute_result"
  92. }
  93. ],
  94. "source": [
  95. "npbuf = load_buffer(tdf.df_handle, tdf.df_size)\n",
  96. "npbuf"
  97. ]
  98. },
  99. {
  100. "cell_type": "code",
  101. "execution_count": 5,
  102. "metadata": {},
  103. "outputs": [
  104. {
  105. "data": {
  106. "text/plain": [
  107. "<pyarrow.lib.Buffer at 0x119c1a1b8>"
  108. ]
  109. },
  110. "execution_count": 5,
  111. "metadata": {},
  112. "output_type": "execute_result"
  113. }
  114. ],
  115. "source": [
  116. "buf = pa.frombuffer(npbuf)\n",
  117. "buf"
  118. ]
  119. },
  120. {
  121. "cell_type": "code",
  122. "execution_count": 6,
  123. "metadata": {},
  124. "outputs": [
  125. {
  126. "name": "stderr",
  127. "output_type": "stream",
  128. "text": [
  129. "/Users/taugspurger/miniconda3/envs/mapd-dev/lib/python3.6/site-packages/pyarrow/__init__.py:116: FutureWarning: pyarrow.StreamReader has been renamed to RecordBatchStreamReader, will be removed in 0.5.0\n",
  130. " warnings.warn(msg, FutureWarning)\n"
  131. ]
  132. },
  133. {
  134. "ename": "ArrowIOError",
  135. "evalue": "IOError: Message not expected type: schema, was: 3",
  136. "output_type": "error",
  137. "traceback": [
  138. "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
  139. "\u001b[0;31mArrowIOError\u001b[0m Traceback (most recent call last)",
  140. "\u001b[0;32m<ipython-input-6-dfb6cb8c3239>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mStreamReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
  141. "\u001b[0;32m~/miniconda3/envs/mapd-dev/lib/python3.6/site-packages/pyarrow/__init__.py\u001b[0m in \u001b[0;36mdeprecated_factory\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdeprecated_factory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 116\u001b[0m \u001b[0mwarnings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mFutureWarning\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mklass\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 118\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mdeprecated_factory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
  142. "\u001b[0;32m~/miniconda3/envs/mapd-dev/lib/python3.6/site-packages/pyarrow/ipc.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, source)\u001b[0m\n\u001b[1;32m 56\u001b[0m \"\"\"\n\u001b[1;32m 57\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
  143. "\u001b[0;32mipc.pxi\u001b[0m in \u001b[0;36mpyarrow.lib._RecordBatchReader._open\u001b[0;34m()\u001b[0m\n",
  144. "\u001b[0;32merror.pxi\u001b[0m in \u001b[0;36mpyarrow.lib.check_status\u001b[0;34m()\u001b[0m\n",
  145. "\u001b[0;31mArrowIOError\u001b[0m: IOError: Message not expected type: schema, was: 3"
  146. ]
  147. }
  148. ],
  149. "source": [
  150. "pa.StreamReader(buf)"
  151. ]
  152. }
  153. ],
  154. "metadata": {
  155. "kernelspec": {
  156. "display_name": "Python 3",
  157. "language": "python",
  158. "name": "python3"
  159. },
  160. "language_info": {
  161. "codemirror_mode": {
  162. "name": "ipython",
  163. "version": 3
  164. },
  165. "file_extension": ".py",
  166. "mimetype": "text/x-python",
  167. "name": "python",
  168. "nbconvert_exporter": "python",
  169. "pygments_lexer": "ipython3",
  170. "version": "3.6.1"
  171. }
  172. },
  173. "nbformat": 4,
  174. "nbformat_minor": 2
  175. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!