Guest User

docker resume image download

a guest
May 26th, 2019
524
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #!/usr/bin/env bash
  2. BW=8
  3.  
  4. if [[ "$1" == "BW"* ]]; then
  5.     NEWBW=`echo $1|cut -d: -f2`
  6.     if [ "$NEWBW" != "" ]; then
  7.         BW=$NEWBW
  8.         echo using bw limit $BW
  9.     fi
  10.     shift
  11. fi
  12.  
  13. unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY
  14. set -eo pipefail
  15.  
  16. # hello-world                      latest              ef872312fe1b        3 months ago        910 B
  17. # hello-world                      latest              ef872312fe1bbc5e05aae626791a47ee9b032efa8f3bda39cc0be7b56bfe59b9   3 months ago        910 B
  18.  
  19. # debian                           latest              f6fab3b798be        10 weeks ago        85.1 MB
  20. # debian                           latest              f6fab3b798be3174f45aa1eb731f8182705555f89c9026d8c1ef230cbf8301dd   10 weeks ago        85.1 MB
  21.  
  22. # check if essential commands are in our PATH
  23. for cmd in curl jq go; do
  24.     if ! command -v $cmd &> /dev/null; then
  25.         echo >&2 "error: \"$cmd\" not found!"
  26.         exit 1
  27.     fi
  28. done
  29.  
  30. usage() {
  31.     echo "usage: $0 dir image[:tag][@digest] ..."
  32.     echo "       $0 /tmp/old-hello-world hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7"
  33.     [ -z "$1" ] || exit "$1"
  34. }
  35.  
  36. dir="$1" # dir for building tar in
  37. shift || usage 1 >&2
  38.  
  39. [ $# -gt 0 -a "$dir" ] || usage 2 >&2
  40. mkdir -p "$dir"
  41.  
  42. # hacky workarounds for Bash 3 support (no associative arrays)
  43. images=()
  44. rm -f "$dir"/tags-*.tmp
  45. manifestJsonEntries=()
  46. doNotGenerateManifestJson=
  47. # repositories[busybox]='"latest": "...", "ubuntu-14.04": "..."'
  48.  
  49. # bash v4 on Windows CI requires CRLF separator
  50. newlineIFS=$'\n'
  51. if [ "$(go env GOHOSTOS)" = 'windows' ]; then
  52.     major=$(echo ${BASH_VERSION%%[^0.9]} | cut -d. -f1)
  53.     if [ "$major" -ge 4 ]; then
  54.         newlineIFS=$'\r\n'
  55.     fi
  56. fi
  57.  
  58. registryBase='https://registry-1.docker.io'
  59. authBase='https://auth.docker.io'
  60. authService='registry.docker.io'
  61.  
  62. # https://github.com/moby/moby/issues/33700
  63. fetch_blob() {
  64.     FULL_FILE=0
  65.     local token="$1"; shift
  66.     local image="$1"; shift
  67.     local digest="$1"; shift
  68.     local targetFile="$1"; shift
  69.     local curlArgs=( "$@" )
  70.  
  71.     local curlHeaders="$(
  72.         curl -S "${curlArgs[@]}" \
  73.             -H "Authorization: Bearer $token" \
  74.             "$registryBase/v2/$image/blobs/$digest" \
  75.             -o "$targetFile.headers" \
  76.             -D-
  77.     )"
  78.     curlHeaders="$(echo "$curlHeaders" | tr -d '\r')"
  79.     if grep -qE "^HTTP/[0-9].[0-9] 3" <<<"$curlHeaders"; then
  80.         local blobRedirect="$(echo "$curlHeaders" | awk -F ': ' 'tolower($1) == "location" { print $2; exit }')"
  81.         if [ -z "$blobRedirect" ]; then
  82.             echo >&2 "error: failed fetching '$image' blob '$digest'"
  83.             echo "$curlHeaders" | head -1 >&2
  84.             return 1
  85.         fi
  86.         #layer.tar gets handeled differently than small json files...
  87.         if [[ "$targetFile" == *"layer.tar"* ]]; then
  88.             #turn off the scripts cancel on any error settings..
  89.             set +eo pipefail
  90.  
  91.             #loop until we get a 416 (server cannot accomodate resume byte range due to us having full file)
  92.             while :; do
  93.                 #if the file already exists.. we will be resuming..
  94.                 if [ -f "$targetFile" ];then
  95.                     #getting current size of file we are resuming
  96.                     CUR=`stat --printf="%s" $targetFile`
  97.                     #use curl to get headers to find content-length of the full file
  98.                     LEN=`curl -I -fL "${curlArgs[@]}" "$blobRedirect"|grep content-length|cut -d" " -f2`
  99.  
  100.                     #if we already have the entire file... lets stop curl from erroring with 416
  101.                     if [ "$CUR" == "${LEN//[!0-9]/}" ]; then
  102.                         FULL_FILE=1
  103.                         break
  104.                     fi
  105.                 fi
  106.  
  107.                 HTTP_CODE=`curl -w %{http_code} -C - --tr-encoding --compressed --progress-bar -fL "${curlArgs[@]}" "$blobRedirect" -o "$targetFile"`
  108.                 if [ "$HTTP_CODE" == "403" ]; then
  109.                     #token expired so the server stopped allowing us to resume, lets return without setting FULL_FILE and itll restart this func w new token
  110.                     FULL_FILE=0
  111.                     break
  112.                 fi
  113.  
  114.                 if [ "$HTTP_CODE" == "416" ]; then
  115.                     FULL_FILE=1
  116.                     break
  117.                 fi
  118.  
  119.                 sleep 1
  120.             done
  121.         else
  122.             #small file.. needs no resume
  123.             curl -fSL "${curlArgs[@]}" "$blobRedirect" -o "$targetFile"
  124.             FULL_FILE=1
  125.         fi
  126.         #this could fail if we nested a call to this same function and it deletes .headers before it returns, and we try again..
  127.         rm -f "$targetFile.headers"
  128.         #turn back on scripts error checking
  129.         set -eo pipefail
  130.     fi
  131. }
  132.  
  133. # handle 'application/vnd.docker.distribution.manifest.v2+json' manifest
  134. handle_single_manifest_v2() {
  135.     local manifestJson="$1"; shift
  136.  
  137.     local configDigest="$(echo "$manifestJson" | jq --raw-output '.config.digest')"
  138.     local imageId="${configDigest#*:}" # strip off "sha256:"
  139.  
  140.     local configFile="$imageId.json"
  141.     fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s
  142.  
  143.     local layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')"
  144.     local IFS="$newlineIFS"
  145.     local layers=( $layersFs )
  146.     unset IFS
  147.  
  148.     echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..."
  149.     local layerId=
  150.     local layerFiles=()
  151.     for i in "${!layers[@]}"; do
  152.         local layerMeta="${layers[$i]}"
  153.  
  154.         local layerMediaType="$(echo "$layerMeta" | jq --raw-output '.mediaType')"
  155.         local layerDigest="$(echo "$layerMeta" | jq --raw-output '.digest')"
  156.  
  157.         # save the previous layer's ID
  158.         local parentId="$layerId"
  159.         # create a new fake layer ID based on this layer's digest and the previous layer's fake ID
  160.         layerId="$(echo "$parentId"$'\n'"$layerDigest" | sha256sum | cut -d' ' -f1)"
  161.         # this accounts for the possibility that an image contains the same layer twice (and thus has a duplicate digest value)
  162.  
  163.         mkdir -p "$dir/$layerId"
  164.         echo '1.0' > "$dir/$layerId/VERSION"
  165.  
  166.         if [ ! -s "$dir/$layerId/json" ]; then
  167.             local parentJson="$(printf ', parent: "%s"' "$parentId")"
  168.             local addJson="$(printf '{ id: "%s"%s }' "$layerId" "${parentId:+$parentJson}")"
  169.             # this starter JSON is taken directly from Docker's own "docker save" output for unimportant layers
  170.             jq "$addJson + ." > "$dir/$layerId/json" <<-'EOJSON'
  171.                 {
  172.                     "created": "0001-01-01T00:00:00Z",
  173.                     "container_config": {
  174.                         "Hostname": "",
  175.                         "Domainname": "",
  176.                         "User": "",
  177.                         "AttachStdin": false,
  178.                         "AttachStdout": false,
  179.                         "AttachStderr": false,
  180.                         "Tty": false,
  181.                         "OpenStdin": false,
  182.                         "StdinOnce": false,
  183.                         "Env": null,
  184.                         "Cmd": null,
  185.                         "Image": "",
  186.                         "Volumes": null,
  187.                         "WorkingDir": "",
  188.                         "Entrypoint": null,
  189.                         "OnBuild": null,
  190.                         "Labels": null
  191.                     }
  192.                 }
  193.             EOJSON
  194.         fi
  195.  
  196.         case "$layerMediaType" in
  197.             application/vnd.docker.image.rootfs.diff.tar.gzip)
  198.                 local layerTar="$layerId/layer.tar"
  199.                 layerFiles=( "${layerFiles[@]}" "$layerTar" )
  200.                 FULL_FILE=0
  201.                 #loop until FULL_FILE is set in fetch_blob.. this is for bad/slow connections
  202.                 while [ "$FULL_FILE" != "1" ];do
  203.                     local token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
  204.                     fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress
  205.                     sleep 1
  206.                 done
  207.  
  208.                 ;;
  209.  
  210.             *)
  211.                 echo >&2 "error: unknown layer mediaType ($imageIdentifier, $layerDigest): '$layerMediaType'"
  212.                 exit 1
  213.                 ;;
  214.         esac
  215.     done
  216.  
  217.     # change "$imageId" to be the ID of the last layer we added (needed for old-style "repositories" file which is created later -- specifically for older Docker daemons)
  218.     imageId="$layerId"
  219.  
  220.     # munge the top layer image manifest to have the appropriate image configuration for older daemons
  221.     local imageOldConfig="$(jq --raw-output --compact-output '{ id: .id } + if .parent then { parent: .parent } else {} end' "$dir/$imageId/json")"
  222.     jq --raw-output "$imageOldConfig + del(.history, .rootfs)" "$dir/$configFile" > "$dir/$imageId/json"
  223.  
  224.     local manifestJsonEntry="$(
  225.         echo '{}' | jq --raw-output '. + {
  226.             Config: "'"$configFile"'",
  227.             RepoTags: ["'"${image#library\/}:$tag"'"],
  228.             Layers: '"$(echo '[]' | jq --raw-output ".$(for layerFile in "${layerFiles[@]}"; do echo " + [ \"$layerFile\" ]"; done)")"'
  229.         }'
  230.     )"
  231.     manifestJsonEntries=( "${manifestJsonEntries[@]}" "$manifestJsonEntry" )
  232. }
  233.  
  234. while [ $# -gt 0 ]; do
  235.     imageTag="$1"
  236.     shift
  237.     image="${imageTag%%[:@]*}"
  238.     imageTag="${imageTag#*:}"
  239.     if [ "$imageTag" == "$image" ]; then
  240.         imageTag="latest"
  241.     fi
  242.     digest="${imageTag##*@}"
  243.     tag="${imageTag%%@*}"
  244.  
  245.     # add prefix library if passed official image
  246.     if [[ "$image" != *"/"* ]]; then
  247.         image="library/$image"
  248.     fi
  249.  
  250.     imageFile="${image//\//_}" # "/" can't be in filenames :)
  251.  
  252.     token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
  253.  
  254.     manifestJson="$(
  255.         curl -fsSL \
  256.             -H "Authorization: Bearer $token" \
  257.             -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
  258.             -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
  259.             -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
  260.             "$registryBase/v2/$image/manifests/$digest"
  261.     )"
  262.     if [ "${manifestJson:0:1}" != '{' ]; then
  263.         echo >&2 "error: /v2/$image/manifests/$digest returned something unexpected:"
  264.         echo >&2 "  $manifestJson"
  265.         exit 1
  266.     fi
  267.  
  268.     imageIdentifier="$image:$tag@$digest"
  269.  
  270.     schemaVersion="$(echo "$manifestJson" | jq --raw-output '.schemaVersion')"
  271.     case "$schemaVersion" in
  272.         2)
  273.             mediaType="$(echo "$manifestJson" | jq --raw-output '.mediaType')"
  274.  
  275.             case "$mediaType" in
  276.                 application/vnd.docker.distribution.manifest.v2+json)
  277.                     handle_single_manifest_v2 "$manifestJson"
  278.                     ;;
  279.                 application/vnd.docker.distribution.manifest.list.v2+json)
  280.                     layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.manifests[]')"
  281.                     IFS="$newlineIFS"
  282.                     layers=( $layersFs )
  283.                     unset IFS
  284.  
  285.                     found=""
  286.                     # parse first level multi-arch manifest
  287.                     for i in "${!layers[@]}"; do
  288.                         layerMeta="${layers[$i]}"
  289.                         maniArch="$(echo "$layerMeta" | jq --raw-output '.platform.architecture')"
  290.                         if [ "$maniArch" = "$(go env GOARCH)" ]; then
  291.                             digest="$(echo "$layerMeta" | jq --raw-output '.digest')"
  292.                             # get second level single manifest
  293.                             submanifestJson="$(
  294.                                 curl -fsSL \
  295.                                     -H "Authorization: Bearer $token" \
  296.                                     -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
  297.                                     -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
  298.                                     -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
  299.                                     "$registryBase/v2/$image/manifests/$digest"
  300.                             )"
  301.                             handle_single_manifest_v2 "$submanifestJson"
  302.                             found="found"
  303.                             break
  304.                         fi
  305.                     done
  306.                     if [ -z "$found" ]; then
  307.                         echo >&2 "error: manifest for $maniArch is not found"
  308.                         exit 1
  309.                     fi
  310.                     ;;
  311.                 *)
  312.                     echo >&2 "error: unknown manifest mediaType ($imageIdentifier): '$mediaType'"
  313.                     exit 1
  314.                     ;;
  315.             esac
  316.             ;;
  317.  
  318.         1)
  319.             if [ -z "$doNotGenerateManifestJson" ]; then
  320.                 echo >&2 "warning: '$imageIdentifier' uses schemaVersion '$schemaVersion'"
  321.                 echo >&2 "  this script cannot (currently) recreate the 'image config' to put in a 'manifest.json' (thus any schemaVersion 2+ images will be imported in the old way, and their 'docker history' will suffer)"
  322.                 echo >&2
  323.                 doNotGenerateManifestJson=1
  324.             fi
  325.  
  326.             layersFs="$(echo "$manifestJson" | jq --raw-output '.fsLayers | .[] | .blobSum')"
  327.             IFS="$newlineIFS"
  328.             layers=( $layersFs )
  329.             unset IFS
  330.  
  331.             history="$(echo "$manifestJson" | jq '.history | [.[] | .v1Compatibility]')"
  332.             imageId="$(echo "$history" | jq --raw-output '.[0]' | jq --raw-output '.id')"
  333.  
  334.             echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..."
  335.             for i in "${!layers[@]}"; do
  336.                 imageJson="$(echo "$history" | jq --raw-output ".[${i}]")"
  337.                 layerId="$(echo "$imageJson" | jq --raw-output '.id')"
  338.                 imageLayer="${layers[$i]}"
  339.  
  340.                 mkdir -p "$dir/$layerId"
  341.                 echo '1.0' > "$dir/$layerId/VERSION"
  342.  
  343.                 echo "$imageJson" > "$dir/$layerId/json"
  344.  
  345.                 FULL_FILE=0
  346.                 #loop until FULL_FILE is set in fetch_blob.. this is for bad/slow connections
  347.                 while [ "$FULL_FILE" != "1" ];do
  348.                                         token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
  349.                         fetch_blob "$token" "$image" "$imageLayer" "$dir/$layerId/layer.tar" --progress
  350.                         echo after fetch full file $FULL_FILE
  351.                         sleep 1
  352.                     done
  353.             done
  354.             ;;
  355.  
  356.         *)
  357.             echo >&2 "error: unknown manifest schemaVersion ($imageIdentifier): '$schemaVersion'"
  358.             exit 1
  359.             ;;
  360.     esac
  361.  
  362.     echo
  363.  
  364.     if [ -s "$dir/tags-$imageFile.tmp" ]; then
  365.         echo -n ', ' >> "$dir/tags-$imageFile.tmp"
  366.     else
  367.         images=( "${images[@]}" "$image" )
  368.     fi
  369.     echo -n '"'"$tag"'": "'"$imageId"'"' >> "$dir/tags-$imageFile.tmp"
  370. done
  371.  
  372. echo -n '{' > "$dir/repositories"
  373. firstImage=1
  374. for image in "${images[@]}"; do
  375.     imageFile="${image//\//_}" # "/" can't be in filenames :)
  376.     image="${image#library\/}"
  377.  
  378.     [ "$firstImage" ] || echo -n ',' >> "$dir/repositories"
  379.     firstImage=
  380.     echo -n $'\n\t' >> "$dir/repositories"
  381.     echo -n '"'"$image"'": { '"$(cat "$dir/tags-$imageFile.tmp")"' }' >> "$dir/repositories"
  382. done
  383. echo -n $'\n}\n' >> "$dir/repositories"
  384.  
  385. rm -f "$dir"/tags-*.tmp
  386.  
  387. if [ -z "$doNotGenerateManifestJson" ] && [ "${#manifestJsonEntries[@]}" -gt 0 ]; then
  388.     echo '[]' | jq --raw-output ".$(for entry in "${manifestJsonEntries[@]}"; do echo " + [ $entry ]"; done)" > "$dir/manifest.json"
  389. else
  390.     rm -f "$dir/manifest.json"
  391. fi
  392.  
  393. echo "Download of images into '$dir' complete."
  394. echo "Use something like the following to load the result into a Docker daemon:"
  395. echo "  tar -cC '$dir' . | docker load"
RAW Paste Data