Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import s3fs
- import sys
- # Expect aws credentials to exist in path
- def fetch_filtered_filekeys(bucket_name: str, dir_name: str, prefix_file: str="",suffix: str="") -> list:
- S3 = s3fs.S3FileSystem()
- s3_dataset_path = "/".join([bucket_name, dir_name])
- list_available_file = [p for p in S3.walk(s3_dataset_path)
- if p.endswith(suffix_file) and p.startswith(prefix_file)]
- return list_available_file
- if __name__ == "__main__":
- # usage python3 count_files_s3_dir_and_subdir.py <bucket_name> <dir_name> [prefix_file] [suffix_file]
- print(len(fetch_filtered_filekeys(**sys.argv[1:])))
Add Comment
Please, Sign In to add comment