Guest User

Untitled

a guest
Jun 28th, 2024
132
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.56 KB | None | 0 0
  1. import clearml
  2.  
  3.  
  4. def make_zipped_ds():
  5.     parent = clearml.Dataset.create(dataset_name="parent-zip", dataset_project="test")
  6.     parent.add_files("./parentfile.txt")
  7.     parent.upload()
  8.     parent.finalize()
  9.  
  10.     child = clearml.Dataset.create(
  11.         dataset_name="child-zip", dataset_project="test", parent_datasets=[parent]
  12.     )
  13.     child.add_files("./testfile.txt")
  14.     child.upload()
  15.     child.finalize()
  16.  
  17.  
  18. def make_ext_ds():
  19.     S3_URL = ""
  20.     BUCKET = ""
  21.  
  22.     parent = clearml.Dataset.create(dataset_name="parent-ext", dataset_project="test")
  23.     parent.add_external_files(f"s3://{S3_URL}/{BUCKET}/copytest.txt")
  24.     parent.upload()
  25.     parent.finalize()
  26.  
  27.     child = clearml.Dataset.create(
  28.         dataset_name="child-ext", dataset_project="test", parent_datasets=[parent]
  29.     )
  30.     child.add_files("./testfile.txt")
  31.     child.upload()
  32.     child.finalize()
  33.  
  34.  
  35. def test_zip():
  36.     parent = clearml.Dataset.get(dataset_name="parent-zip", dataset_project="test")
  37.     child = clearml.Dataset.get(dataset_name="child-zip", dataset_project="test")
  38.  
  39.     parent.get_local_copy()
  40.  
  41.     # Reuses the parent's local copy just fine
  42.     child.get_local_copy(use_soft_links=True)
  43.  
  44.  
  45. def test_ext():
  46.     parent = clearml.Dataset.get(dataset_name="parent-ext", dataset_project="test")
  47.     child = clearml.Dataset.get(dataset_name="child-ext", dataset_project="test")
  48.  
  49.     parent.get_local_copy()
  50.  
  51.     # Redownloads all the parent's external files
  52.     child.get_local_copy(use_soft_links=True)
  53.  
  54.  
  55. make_zipped_ds()
  56. make_ext_ds()
  57.  
  58. test_zip()
  59. test_ext()
Advertisement
Add Comment
Please, Sign In to add comment