Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import clearml
- def make_zipped_ds():
- parent = clearml.Dataset.create(dataset_name="parent-zip", dataset_project="test")
- parent.add_files("./parentfile.txt")
- parent.upload()
- parent.finalize()
- child = clearml.Dataset.create(
- dataset_name="child-zip", dataset_project="test", parent_datasets=[parent]
- )
- child.add_files("./testfile.txt")
- child.upload()
- child.finalize()
- def make_ext_ds():
- S3_URL = ""
- BUCKET = ""
- parent = clearml.Dataset.create(dataset_name="parent-ext", dataset_project="test")
- parent.add_external_files(f"s3://{S3_URL}/{BUCKET}/copytest.txt")
- parent.upload()
- parent.finalize()
- child = clearml.Dataset.create(
- dataset_name="child-ext", dataset_project="test", parent_datasets=[parent]
- )
- child.add_files("./testfile.txt")
- child.upload()
- child.finalize()
- def test_zip():
- parent = clearml.Dataset.get(dataset_name="parent-zip", dataset_project="test")
- child = clearml.Dataset.get(dataset_name="child-zip", dataset_project="test")
- parent.get_local_copy()
- # Reuses the parent's local copy just fine
- child.get_local_copy(use_soft_links=True)
- def test_ext():
- parent = clearml.Dataset.get(dataset_name="parent-ext", dataset_project="test")
- child = clearml.Dataset.get(dataset_name="child-ext", dataset_project="test")
- parent.get_local_copy()
- # Redownloads all the parent's external files
- child.get_local_copy(use_soft_links=True)
- make_zipped_ds()
- make_ext_ds()
- test_zip()
- test_ext()
Advertisement
Add Comment
Please, Sign In to add comment