Guest User

Untitled

a guest
Feb 28th, 2024
53
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.55 KB | None | 0 0
  1. tests/test_read_blob.py:10 (test_read_blob)
  2. @pytest.mark.asyncio
  3. async def test_read_blob():
  4. '''
  5. start_time = time.time()
  6. cluster = LocalCluster(dashboard_address=":10086") # Fully-featured local Dask cluster
  7. client = cluster.get_client()
  8. # ds = xr.open_dataset('/home/forestbat/usgs-streamflow-nldas_hourly.nc')
  9. ds = xr.open_dataset('/home/forestbat/gpm_gfs.nc')
  10. # new_zarr = ds.to_zarr('usgs-streamflow-nldas_hourly.zarr', mode='w')
  11. print(time.time() - start_time)
  12. '''
  13. zarr_start_time = time.time()
  14. zarr_path = await conf.FS.open_async('s3://datasets-origin/usgs_streamflow_nldas_hourly.zarr')
  15. > zds = xr.open_dataset(zarr_path, engine='zarr')
  16.  
  17. test_read_blob.py:24:
  18. _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
  19. ../../../.conda/envs/test_sam/lib/python3.10/site-packages/xarray/backends/api.py:573: in open_dataset
  20. backend_ds = backend.open_dataset(
  21. ../../../.conda/envs/test_sam/lib/python3.10/site-packages/xarray/backends/zarr.py:1016: in open_dataset
  22. store = ZarrStore.open_group(
  23. ../../../.conda/envs/test_sam/lib/python3.10/site-packages/xarray/backends/zarr.py:446: in open_group
  24. zarr_group = zarr.open_consolidated(store, **open_kwargs)
  25. ../../../.conda/envs/test_sam/lib/python3.10/site-packages/zarr/convenience.py:1325: in open_consolidated
  26. store = normalize_store_arg(
  27. ../../../.conda/envs/test_sam/lib/python3.10/site-packages/zarr/storage.py:198: in normalize_store_arg
  28. return normalize_store(store, storage_options, mode)
  29. ../../../.conda/envs/test_sam/lib/python3.10/site-packages/zarr/storage.py:180: in _normalize_store_arg_v2
  30. store = Store._ensure_store(store)
  31. _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
  32.  
  33. store = <File-like object S3FileSystem, s3://datasets-origin/usgs_streamflow_nldas_hourly.zarr>
  34.  
  35. @staticmethod
  36. def _ensure_store(store: Any):
  37. """
  38. We want to make sure internally that zarr stores are always a class
  39. with a specific interface derived from ``BaseStore``, which is slightly
  40. different than ``MutableMapping``.
  41.  
  42. We'll do this conversion in a few places automatically
  43. """
  44. from zarr.storage import KVStore # avoid circular import
  45.  
  46. if isinstance(store, BaseStore):
  47. if not store._store_version == 2:
  48. raise ValueError(
  49. f"cannot initialize a v2 store with a v{store._store_version} store"
  50. )
  51. return store
  52. elif isinstance(store, MutableMapping):
  53. return KVStore(store)
  54. else:
  55. for attr in [
  56. "keys",
  57. "values",
  58. "get",
  59. "__setitem__",
  60. "__getitem__",
  61. "__delitem__",
  62. "__contains__",
  63. ]:
  64. if not hasattr(store, attr):
  65. break
  66. else:
  67. return KVStore(store)
  68.  
  69. > raise ValueError(
  70. "Starting with Zarr 2.11.0, stores must be subclasses of "
  71. "BaseStore, if your store exposes the MutableMapping interface "
  72. f"wrap it in Zarr.storage.KVStore. Got {store}"
  73. )
  74. E ValueError: Starting with Zarr 2.11.0, stores must be subclasses of BaseStore, if your store exposes the MutableMapping interface wrap it in Zarr.storage.KVStore. Got <File-like object S3FileSystem, s3://datasets-origin/usgs_streamflow_nldas_hourly.zarr>
  75.  
  76.  
Advertisement
Add Comment
Please, Sign In to add comment