Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # dask only version of this problem
- shape = (100000, 100, 200)
- chunks = (50, 100, 200)
- data = dsa.random.random(shape, chunks=chunks)
- data
- data_rc = data.rechunk((1000, 1, 200))
- data_rc
- da.data.ravel
- from time import sleep
- def my_custom_function(f):
- # do some computations
- g = f**2 + 2*np.sin(f)
- sleep(0.01)
- # and reduce shape
- return g.ravel()[::15]
- data_rc.map_blocks(my_custom_function, drop_axis=[1, 2], new_axis=[1], chunks=(1, 15))
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement