Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- {
- "cells": [
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# iquery --afl\n",
- "AFL% stream(\n",
- " build(<x:int64 not null>[i=1:10:0:5], i),\n",
- " 'python -uc \"\n",
- "import scidbstrm\n",
- "import pyarrow\n",
- "import sys\n",
- "import tempfile\n",
- "\n",
- "def save(df):\n",
- " rb = pyarrow.RecordBatch.from_pandas(df, preserve_index=False)\n",
- "\n",
- " fn = tempfile.mkstemp()[1]\n",
- " sys.stderr.write(\\\"{}\\\".format(fn))\n",
- "\n",
- " with pyarrow.OSFile(fn, \\\"wb\\\") as sink:\n",
- " writer = pyarrow.RecordBatchStreamWriter(sink, rb.schema)\n",
- " writer.write_batch(rb)\n",
- " writer.close()\n",
- "\n",
- " return None\n",
- "\n",
- "scidbstrm.map(save)\"',\n",
- " 'format=feather',\n",
- " 'types=int64',\n",
- " 'names=unused');"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "total 12\r\n",
- "-rw-------. 1 rares rares 3787 Feb 13 20:24 read-batches.ipynb\r\n",
- "-rw-------. 1 rares rares 556 Feb 13 19:37 tmpGqyn5p\r\n",
- "-rw-------. 1 rares rares 556 Feb 13 19:37 tmpzDxek1\r\n"
- ]
- }
- ],
- "source": [
- "!ls -l"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [],
- "source": [
- "import pyarrow"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "(5, 1)"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "reader = pyarrow.open_stream('tmpGqyn5p')\n",
- "table = reader.read_all()\n",
- "table.shape"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "<pyarrow.lib.Column object at 0x7fe5281ce360>\n",
- "chunk 0: <pyarrow.lib.Int64Array object at 0x7fe5281c2c28>\n",
- "[\n",
- " 6,\n",
- " 7,\n",
- " 8,\n",
- " 9,\n",
- " 10\n",
- "]"
- ]
- },
- "execution_count": 4,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "table[0]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "(5, 1)"
- ]
- },
- "execution_count": 5,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "reader = pyarrow.open_stream('tmpzDxek1')\n",
- "table = reader.read_all()\n",
- "table.shape"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "<pyarrow.lib.Column object at 0x7fe5281ce450>\n",
- "chunk 0: <pyarrow.lib.Int64Array object at 0x7fe5281dc598>\n",
- "[\n",
- " 1,\n",
- " 2,\n",
- " 3,\n",
- " 4,\n",
- " 5\n",
- "]"
- ]
- },
- "execution_count": 6,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "table[0]"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.6.4"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
- }
Add Comment
Please, Sign In to add comment