Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- tests/test_read_blob.py:10 (test_read_blob)
- @pytest.mark.asyncio
- async def test_read_blob():
- '''
- start_time = time.time()
- cluster = LocalCluster(dashboard_address=":10086") # Fully-featured local Dask cluster
- client = cluster.get_client()
- # ds = xr.open_dataset('/home/forestbat/usgs-streamflow-nldas_hourly.nc')
- ds = xr.open_dataset('/home/forestbat/gpm_gfs.nc')
- # new_zarr = ds.to_zarr('usgs-streamflow-nldas_hourly.zarr', mode='w')
- print(time.time() - start_time)
- '''
- zarr_start_time = time.time()
- zarr_path = await conf.FS.open_async('s3://datasets-origin/usgs_streamflow_nldas_hourly.zarr')
- > zds = xr.open_dataset(zarr_path, engine='zarr')
- test_read_blob.py:24:
- _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
- ../../../.conda/envs/test_sam/lib/python3.10/site-packages/xarray/backends/api.py:573: in open_dataset
- backend_ds = backend.open_dataset(
- ../../../.conda/envs/test_sam/lib/python3.10/site-packages/xarray/backends/zarr.py:1016: in open_dataset
- store = ZarrStore.open_group(
- ../../../.conda/envs/test_sam/lib/python3.10/site-packages/xarray/backends/zarr.py:446: in open_group
- zarr_group = zarr.open_consolidated(store, **open_kwargs)
- ../../../.conda/envs/test_sam/lib/python3.10/site-packages/zarr/convenience.py:1325: in open_consolidated
- store = normalize_store_arg(
- ../../../.conda/envs/test_sam/lib/python3.10/site-packages/zarr/storage.py:198: in normalize_store_arg
- return normalize_store(store, storage_options, mode)
- ../../../.conda/envs/test_sam/lib/python3.10/site-packages/zarr/storage.py:180: in _normalize_store_arg_v2
- store = Store._ensure_store(store)
- _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
- store = <File-like object S3FileSystem, s3://datasets-origin/usgs_streamflow_nldas_hourly.zarr>
- @staticmethod
- def _ensure_store(store: Any):
- """
- We want to make sure internally that zarr stores are always a class
- with a specific interface derived from ``BaseStore``, which is slightly
- different than ``MutableMapping``.
- We'll do this conversion in a few places automatically
- """
- from zarr.storage import KVStore # avoid circular import
- if isinstance(store, BaseStore):
- if not store._store_version == 2:
- raise ValueError(
- f"cannot initialize a v2 store with a v{store._store_version} store"
- )
- return store
- elif isinstance(store, MutableMapping):
- return KVStore(store)
- else:
- for attr in [
- "keys",
- "values",
- "get",
- "__setitem__",
- "__getitem__",
- "__delitem__",
- "__contains__",
- ]:
- if not hasattr(store, attr):
- break
- else:
- return KVStore(store)
- > raise ValueError(
- "Starting with Zarr 2.11.0, stores must be subclasses of "
- "BaseStore, if your store exposes the MutableMapping interface "
- f"wrap it in Zarr.storage.KVStore. Got {store}"
- )
- E ValueError: Starting with Zarr 2.11.0, stores must be subclasses of BaseStore, if your store exposes the MutableMapping interface wrap it in Zarr.storage.KVStore. Got <File-like object S3FileSystem, s3://datasets-origin/usgs_streamflow_nldas_hourly.zarr>
Advertisement
Add Comment
Please, Sign In to add comment