Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #! /bin/bash
- #
- # This file will filter all s3 objects where the key/path contains a specific pattern and apply the provided tag.
- # Results are split into X number of backfground processes, each tagging 1000 objects
- #
- BUCKET_NAME="YOURBUCKET"
- function tagObjects {
- type=$1
- pattern=$2
- tagging=$3
- # get listing, save to file
- if [ ! -e objects-${type}.txt ];then
- aws s3api list-objects-v2 \
- --bucket ${BUCKET_NAME} \
- --prefix artifacts \
- --query "Contents[?contains(Key,${pattern})].[Key]" \
- --output text > objects-$type.txt
- fi
- echo "Got objects"
- # split file if >1000 to run parallel
- mkdir splitfiles-$type
- cd splitfiles-$type
- split -l 1000 ../objects-$type.txt
- cd ..
- mkdir logs-$type
- function tag_objects {
- logpath=logs-$type/$(basename ${1}).log
- while read key;do
- echo "Tagging ${key}" >> $logpath
- aws s3api put-object-tagging --bucket ${BUCKET_NAME} --key ${key} --tagging ${tagging} >> $logpath
- done < $1
- echo "Done file $1"
- }
- for file in splitfiles-${type}/*;do
- echo "Sending file ${file} o background"
- tag_objects ${file} &
- done
- wait
- }
- tagObjects "cache" "'global/cache'" "TagSet=[{Key=circleci.objectType,Value=project.cache}]"
- tagObjects "workspaces" "'workflows/workspaces'" "TagSet=[{Key=circleci.objectType,Value=workflow.workspace}]"
- tagObjects "artifact" "'/artifact/'" "TagSet=[{Key=circleci.objectType,Value=job.artifact}]"
Add Comment
Please, Sign In to add comment