Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- {
- "cells": [
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "data": {
- "text/plain": [
- "<Executor: scheduler=localhost:8786 workers=80 threads=80>"
- ]
- },
- "execution_count": 1,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "from distributed import Executor, progress, wait\n",
- "e = Executor('localhost:8786')\n",
- "e"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import dask.array as da\n",
- "x = da.random.randint(0, 2000, size=(10240, 1024000), \n",
- " chunks=(40, 1024000)).astype('u2')\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/opt/anaconda/lib/python3.5/site-packages/dask/imperative.py:5: UserWarning: dask.imperative has been moved to dask.delayed\n",
- " warn(\"dask.imperative has been moved to dask.delayed\")\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CPU times: user 202 ms, sys: 166 µs, total: 202 ms\n",
- "Wall time: 2.84 s\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "x = e.persist(x)\n",
- "wait(x)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CPU times: user 75.8 ms, sys: 3.98 ms, total: 79.8 ms\n",
- "Wall time: 78.8 ms\n"
- ]
- }
- ],
- "source": [
- "%%time\n",
- "y = (x.rechunk((80, 512000))\n",
- " .rechunk((160, 256000))\n",
- " .rechunk((320, 128000))\n",
- " .rechunk((640, 64000))\n",
- " .rechunk((1280, 32000))\n",
- " .rechunk((2560, 16000))\n",
- " .rechunk((5120, 8000))\n",
- " .rechunk((10240, 4000)))\n",
- "z = y.sum()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CPU times: user 528 ms, sys: 28.6 ms, total: 556 ms\n",
- "Wall time: 1min 35s\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "[10480510468080]"
- ]
- },
- "execution_count": 5,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%time\n",
- "e.get(z.dask, z._keys())"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": []
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.5.1"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 0
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement