API tools faq
paste
Guest User

docker resume image download

a guest
May 26th, 2019
889
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Bash 13.33 KB | None | 0 0
raw download clone embed print report
  1. #!/usr/bin/env bash
  2. BW=8
  3.  
  4. if [[ "$1" == "BW"* ]]; then
  5.     NEWBW=`echo $1|cut -d: -f2`
  6.     if [ "$NEWBW" != "" ]; then
  7.         BW=$NEWBW
  8.         echo using bw limit $BW
  9.     fi
  10.     shift
  11. fi
  12.  
  13. unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY
  14. set -eo pipefail
  15.  
  16. # hello-world                      latest              ef872312fe1b        3 months ago        910 B
  17. # hello-world                      latest              ef872312fe1bbc5e05aae626791a47ee9b032efa8f3bda39cc0be7b56bfe59b9   3 months ago        910 B
  18.  
  19. # debian                           latest              f6fab3b798be        10 weeks ago        85.1 MB
  20. # debian                           latest              f6fab3b798be3174f45aa1eb731f8182705555f89c9026d8c1ef230cbf8301dd   10 weeks ago        85.1 MB
  21.  
  22. # check if essential commands are in our PATH
  23. for cmd in curl jq go; do
  24.     if ! command -v $cmd &> /dev/null; then
  25.         echo >&2 "error: \"$cmd\" not found!"
  26.         exit 1
  27.     fi
  28. done
  29.  
  30. usage() {
  31.     echo "usage: $0 dir image[:tag][@digest] ..."
  32.     echo "       $0 /tmp/old-hello-world hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7"
  33.     [ -z "$1" ] || exit "$1"
  34. }
  35.  
  36. dir="$1" # dir for building tar in
  37. shift || usage 1 >&2
  38.  
  39. [ $# -gt 0 -a "$dir" ] || usage 2 >&2
  40. mkdir -p "$dir"
  41.  
  42. # hacky workarounds for Bash 3 support (no associative arrays)
  43. images=()
  44. rm -f "$dir"/tags-*.tmp
  45. manifestJsonEntries=()
  46. doNotGenerateManifestJson=
  47. # repositories[busybox]='"latest": "...", "ubuntu-14.04": "..."'
  48.  
  49. # bash v4 on Windows CI requires CRLF separator
  50. newlineIFS=$'\n'
  51. if [ "$(go env GOHOSTOS)" = 'windows' ]; then
  52.     major=$(echo ${BASH_VERSION%%[^0.9]} | cut -d. -f1)
  53.     if [ "$major" -ge 4 ]; then
  54.         newlineIFS=$'\r\n'
  55.     fi
  56. fi
  57.  
  58. registryBase='https://registry-1.docker.io'
  59. authBase='https://auth.docker.io'
  60. authService='registry.docker.io'
  61.  
  62. # https://github.com/moby/moby/issues/33700
  63. fetch_blob() {
  64.     FULL_FILE=0
  65.     local token="$1"; shift
  66.     local image="$1"; shift
  67.     local digest="$1"; shift
  68.     local targetFile="$1"; shift
  69.     local curlArgs=( "$@" )
  70.  
  71.     local curlHeaders="$(
  72.         curl -S "${curlArgs[@]}" \
  73.             -H "Authorization: Bearer $token" \
  74.             "$registryBase/v2/$image/blobs/$digest" \
  75.             -o "$targetFile.headers" \
  76.             -D-
  77.     )"
  78.     curlHeaders="$(echo "$curlHeaders" | tr -d '\r')"
  79.     if grep -qE "^HTTP/[0-9].[0-9] 3" <<<"$curlHeaders"; then
  80.         local blobRedirect="$(echo "$curlHeaders" | awk -F ': ' 'tolower($1) == "location" { print $2; exit }')"
  81.         if [ -z "$blobRedirect" ]; then
  82.             echo >&2 "error: failed fetching '$image' blob '$digest'"
  83.             echo "$curlHeaders" | head -1 >&2
  84.             return 1
  85.         fi
  86.         #layer.tar gets handeled differently than small json files...
  87.         if [[ "$targetFile" == *"layer.tar"* ]]; then
  88.             #turn off the scripts cancel on any error settings..
  89.             set +eo pipefail
  90.  
  91.             #loop until we get a 416 (server cannot accomodate resume byte range due to us having full file)
  92.             while :; do
  93.                 #if the file already exists.. we will be resuming..
  94.                 if [ -f "$targetFile" ];then
  95.                     #getting current size of file we are resuming
  96.                     CUR=`stat --printf="%s" $targetFile`
  97.                     #use curl to get headers to find content-length of the full file
  98.                     LEN=`curl -I -fL "${curlArgs[@]}" "$blobRedirect"|grep content-length|cut -d" " -f2`
  99.  
  100.                     #if we already have the entire file... lets stop curl from erroring with 416
  101.                     if [ "$CUR" == "${LEN//[!0-9]/}" ]; then
  102.                         FULL_FILE=1
  103.                         break
  104.                     fi
  105.                 fi
  106.  
  107.                 HTTP_CODE=`curl -w %{http_code} -C - --tr-encoding --compressed --progress-bar -fL "${curlArgs[@]}" "$blobRedirect" -o "$targetFile"`
  108.                 if [ "$HTTP_CODE" == "403" ]; then
  109.                     #token expired so the server stopped allowing us to resume, lets return without setting FULL_FILE and itll restart this func w new token
  110.                     FULL_FILE=0
  111.                     break
  112.                 fi
  113.  
  114.                 if [ "$HTTP_CODE" == "416" ]; then
  115.                     FULL_FILE=1
  116.                     break
  117.                 fi
  118.  
  119.                 sleep 1
  120.             done
  121.         else
  122.             #small file.. needs no resume
  123.             curl -fSL "${curlArgs[@]}" "$blobRedirect" -o "$targetFile"
  124.             FULL_FILE=1
  125.         fi
  126.         #this could fail if we nested a call to this same function and it deletes .headers before it returns, and we try again..
  127.         rm -f "$targetFile.headers"
  128.         #turn back on scripts error checking
  129.         set -eo pipefail
  130.     fi
  131. }
  132.  
  133. # handle 'application/vnd.docker.distribution.manifest.v2+json' manifest
  134. handle_single_manifest_v2() {
  135.     local manifestJson="$1"; shift
  136.  
  137.     local configDigest="$(echo "$manifestJson" | jq --raw-output '.config.digest')"
  138.     local imageId="${configDigest#*:}" # strip off "sha256:"
  139.  
  140.     local configFile="$imageId.json"
  141.     fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s
  142.  
  143.     local layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')"
  144.     local IFS="$newlineIFS"
  145.     local layers=( $layersFs )
  146.     unset IFS
  147.  
  148.     echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..."
  149.     local layerId=
  150.     local layerFiles=()
  151.     for i in "${!layers[@]}"; do
  152.         local layerMeta="${layers[$i]}"
  153.  
  154.         local layerMediaType="$(echo "$layerMeta" | jq --raw-output '.mediaType')"
  155.         local layerDigest="$(echo "$layerMeta" | jq --raw-output '.digest')"
  156.  
  157.         # save the previous layer's ID
  158.         local parentId="$layerId"
  159.         # create a new fake layer ID based on this layer's digest and the previous layer's fake ID
  160.         layerId="$(echo "$parentId"$'\n'"$layerDigest" | sha256sum | cut -d' ' -f1)"
  161.         # this accounts for the possibility that an image contains the same layer twice (and thus has a duplicate digest value)
  162.  
  163.         mkdir -p "$dir/$layerId"
  164.         echo '1.0' > "$dir/$layerId/VERSION"
  165.  
  166.         if [ ! -s "$dir/$layerId/json" ]; then
  167.             local parentJson="$(printf ', parent: "%s"' "$parentId")"
  168.             local addJson="$(printf '{ id: "%s"%s }' "$layerId" "${parentId:+$parentJson}")"
  169.             # this starter JSON is taken directly from Docker's own "docker save" output for unimportant layers
  170.             jq "$addJson + ." > "$dir/$layerId/json" <<-'EOJSON'
  171.                 {
  172.                     "created": "0001-01-01T00:00:00Z",
  173.                     "container_config": {
  174.                         "Hostname": "",
  175.                         "Domainname": "",
  176.                         "User": "",
  177.                         "AttachStdin": false,
  178.                         "AttachStdout": false,
  179.                         "AttachStderr": false,
  180.                         "Tty": false,
  181.                         "OpenStdin": false,
  182.                         "StdinOnce": false,
  183.                         "Env": null,
  184.                         "Cmd": null,
  185.                         "Image": "",
  186.                         "Volumes": null,
  187.                         "WorkingDir": "",
  188.                         "Entrypoint": null,
  189.                         "OnBuild": null,
  190.                         "Labels": null
  191.                     }
  192.                 }
  193.             EOJSON
  194.         fi
  195.  
  196.         case "$layerMediaType" in
  197.             application/vnd.docker.image.rootfs.diff.tar.gzip)
  198.                 local layerTar="$layerId/layer.tar"
  199.                 layerFiles=( "${layerFiles[@]}" "$layerTar" )
  200.                 FULL_FILE=0
  201.                 #loop until FULL_FILE is set in fetch_blob.. this is for bad/slow connections
  202.                 while [ "$FULL_FILE" != "1" ];do
  203.                     local token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
  204.                     fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress
  205.                     sleep 1
  206.                 done
  207.  
  208.                 ;;
  209.  
  210.             *)
  211.                 echo >&2 "error: unknown layer mediaType ($imageIdentifier, $layerDigest): '$layerMediaType'"
  212.                 exit 1
  213.                 ;;
  214.         esac
  215.     done
  216.  
  217.     # change "$imageId" to be the ID of the last layer we added (needed for old-style "repositories" file which is created later -- specifically for older Docker daemons)
  218.     imageId="$layerId"
  219.  
  220.     # munge the top layer image manifest to have the appropriate image configuration for older daemons
  221.     local imageOldConfig="$(jq --raw-output --compact-output '{ id: .id } + if .parent then { parent: .parent } else {} end' "$dir/$imageId/json")"
  222.     jq --raw-output "$imageOldConfig + del(.history, .rootfs)" "$dir/$configFile" > "$dir/$imageId/json"
  223.  
  224.     local manifestJsonEntry="$(
  225.         echo '{}' | jq --raw-output '. + {
  226.             Config: "'"$configFile"'",
  227.             RepoTags: ["'"${image#library\/}:$tag"'"],
  228.             Layers: '"$(echo '[]' | jq --raw-output ".$(for layerFile in "${layerFiles[@]}"; do echo " + [ \"$layerFile\" ]"; done)")"'
  229.         }'
  230.     )"
  231.     manifestJsonEntries=( "${manifestJsonEntries[@]}" "$manifestJsonEntry" )
  232. }
  233.  
  234. while [ $# -gt 0 ]; do
  235.     imageTag="$1"
  236.     shift
  237.     image="${imageTag%%[:@]*}"
  238.     imageTag="${imageTag#*:}"
  239.     if [ "$imageTag" == "$image" ]; then
  240.         imageTag="latest"
  241.     fi
  242.     digest="${imageTag##*@}"
  243.     tag="${imageTag%%@*}"
  244.  
  245.     # add prefix library if passed official image
  246.     if [[ "$image" != *"/"* ]]; then
  247.         image="library/$image"
  248.     fi
  249.  
  250.     imageFile="${image//\//_}" # "/" can't be in filenames :)
  251.  
  252.     token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
  253.  
  254.     manifestJson="$(
  255.         curl -fsSL \
  256.             -H "Authorization: Bearer $token" \
  257.             -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
  258.             -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
  259.             -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
  260.             "$registryBase/v2/$image/manifests/$digest"
  261.     )"
  262.     if [ "${manifestJson:0:1}" != '{' ]; then
  263.         echo >&2 "error: /v2/$image/manifests/$digest returned something unexpected:"
  264.         echo >&2 "  $manifestJson"
  265.         exit 1
  266.     fi
  267.  
  268.     imageIdentifier="$image:$tag@$digest"
  269.  
  270.     schemaVersion="$(echo "$manifestJson" | jq --raw-output '.schemaVersion')"
  271.     case "$schemaVersion" in
  272.         2)
  273.             mediaType="$(echo "$manifestJson" | jq --raw-output '.mediaType')"
  274.  
  275.             case "$mediaType" in
  276.                 application/vnd.docker.distribution.manifest.v2+json)
  277.                     handle_single_manifest_v2 "$manifestJson"
  278.                     ;;
  279.                 application/vnd.docker.distribution.manifest.list.v2+json)
  280.                     layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.manifests[]')"
  281.                     IFS="$newlineIFS"
  282.                     layers=( $layersFs )
  283.                     unset IFS
  284.  
  285.                     found=""
  286.                     # parse first level multi-arch manifest
  287.                     for i in "${!layers[@]}"; do
  288.                         layerMeta="${layers[$i]}"
  289.                         maniArch="$(echo "$layerMeta" | jq --raw-output '.platform.architecture')"
  290.                         if [ "$maniArch" = "$(go env GOARCH)" ]; then
  291.                             digest="$(echo "$layerMeta" | jq --raw-output '.digest')"
  292.                             # get second level single manifest
  293.                             submanifestJson="$(
  294.                                 curl -fsSL \
  295.                                     -H "Authorization: Bearer $token" \
  296.                                     -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
  297.                                     -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
  298.                                     -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
  299.                                     "$registryBase/v2/$image/manifests/$digest"
  300.                             )"
  301.                             handle_single_manifest_v2 "$submanifestJson"
  302.                             found="found"
  303.                             break
  304.                         fi
  305.                     done
  306.                     if [ -z "$found" ]; then
  307.                         echo >&2 "error: manifest for $maniArch is not found"
  308.                         exit 1
  309.                     fi
  310.                     ;;
  311.                 *)
  312.                     echo >&2 "error: unknown manifest mediaType ($imageIdentifier): '$mediaType'"
  313.                     exit 1
  314.                     ;;
  315.             esac
  316.             ;;
  317.  
  318.         1)
  319.             if [ -z "$doNotGenerateManifestJson" ]; then
  320.                 echo >&2 "warning: '$imageIdentifier' uses schemaVersion '$schemaVersion'"
  321.                 echo >&2 "  this script cannot (currently) recreate the 'image config' to put in a 'manifest.json' (thus any schemaVersion 2+ images will be imported in the old way, and their 'docker history' will suffer)"
  322.                 echo >&2
  323.                 doNotGenerateManifestJson=1
  324.             fi
  325.  
  326.             layersFs="$(echo "$manifestJson" | jq --raw-output '.fsLayers | .[] | .blobSum')"
  327.             IFS="$newlineIFS"
  328.             layers=( $layersFs )
  329.             unset IFS
  330.  
  331.             history="$(echo "$manifestJson" | jq '.history | [.[] | .v1Compatibility]')"
  332.             imageId="$(echo "$history" | jq --raw-output '.[0]' | jq --raw-output '.id')"
  333.  
  334.             echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..."
  335.             for i in "${!layers[@]}"; do
  336.                 imageJson="$(echo "$history" | jq --raw-output ".[${i}]")"
  337.                 layerId="$(echo "$imageJson" | jq --raw-output '.id')"
  338.                 imageLayer="${layers[$i]}"
  339.  
  340.                 mkdir -p "$dir/$layerId"
  341.                 echo '1.0' > "$dir/$layerId/VERSION"
  342.  
  343.                 echo "$imageJson" > "$dir/$layerId/json"
  344.  
  345.                 FULL_FILE=0
  346.                 #loop until FULL_FILE is set in fetch_blob.. this is for bad/slow connections
  347.                 while [ "$FULL_FILE" != "1" ];do
  348.                                         token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
  349.                         fetch_blob "$token" "$image" "$imageLayer" "$dir/$layerId/layer.tar" --progress
  350.                         echo after fetch full file $FULL_FILE
  351.                         sleep 1
  352.                     done
  353.             done
  354.             ;;
  355.  
  356.         *)
  357.             echo >&2 "error: unknown manifest schemaVersion ($imageIdentifier): '$schemaVersion'"
  358.             exit 1
  359.             ;;
  360.     esac
  361.  
  362.     echo
  363.  
  364.     if [ -s "$dir/tags-$imageFile.tmp" ]; then
  365.         echo -n ', ' >> "$dir/tags-$imageFile.tmp"
  366.     else
  367.         images=( "${images[@]}" "$image" )
  368.     fi
  369.     echo -n '"'"$tag"'": "'"$imageId"'"' >> "$dir/tags-$imageFile.tmp"
  370. done
  371.  
  372. echo -n '{' > "$dir/repositories"
  373. firstImage=1
  374. for image in "${images[@]}"; do
  375.     imageFile="${image//\//_}" # "/" can't be in filenames :)
  376.     image="${image#library\/}"
  377.  
  378.     [ "$firstImage" ] || echo -n ',' >> "$dir/repositories"
  379.     firstImage=
  380.     echo -n $'\n\t' >> "$dir/repositories"
  381.     echo -n '"'"$image"'": { '"$(cat "$dir/tags-$imageFile.tmp")"' }' >> "$dir/repositories"
  382. done
  383. echo -n $'\n}\n' >> "$dir/repositories"
  384.  
  385. rm -f "$dir"/tags-*.tmp
  386.  
  387. if [ -z "$doNotGenerateManifestJson" ] && [ "${#manifestJsonEntries[@]}" -gt 0 ]; then
  388.     echo '[]' | jq --raw-output ".$(for entry in "${manifestJsonEntries[@]}"; do echo " + [ $entry ]"; done)" > "$dir/manifest.json"
  389. else
  390.     rm -f "$dir/manifest.json"
  391. fi
  392.  
  393. echo "Download of images into '$dir' complete."
  394. echo "Use something like the following to load the result into a Docker daemon:"
  395. echo "  tar -cC '$dir' . | docker load"
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!