Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/env bash
- getStringValue(){
- local JSON_FILE=$1
- local KEY=$2
- # echo "KEY $JSON_FILE $KEY"
- cat $JSON_FILE | egrep "$KEY" | egrep -o '\"([^"]|\\\")+\"$' | sed 's/^\"//' | sed 's/\"$//'
- }
- getIntegerValue(){
- local JSON_FILE=$1
- local KEY=$2
- # echo "KEY $JSON_FILE $KEY"
- cat $JSON_FILE | egrep "$KEY" | egrep -o '[0-9]+$'
- }
- echo "Queue owner section start"
- if [ -z $1 ] || [ -z $2 ] || [ -z $3 ] ; then
- echo "converter need 3 attr WEB_SERVER_CONV_API_URL TOKEN PRIVATE_TOKEN."
- exit
- fi
- DATA_FILE=$(mktemp -t data_temp.XXXXXX)
- ERROR_FILE=$(mktemp -t data_temp.XXXXXX)
- FTP_LOG=$(mktemp -t data_temp.XXXXXX)
- #ITEM_LOG=$(mktemp -t data_temp.XXXXXX)
- FTP_GET_TOOL="ftp -o "
- type -P wget &>/dev/null >/dev/null 2>/dev/null && HTTP_GET_TOOL="wget -O -" && FTP_GET_TOOL="wget -O "
- type -P wget &>/dev/null >/dev/null 2>/dev/null && FTP_GET_TOOL="wget -O "
- type -P fetch &>/dev/null >/dev/null 2>/dev/null && HTTP_GET_TOOL="fetch -o - "
- type -P curl &>/dev/null >/dev/null 2>/dev/null && HTTP_GET_TOOL="curl "
- WEB_SERVER_CONV_API_URL="$1"
- TOKEN="$2"
- PRIVATE_TOKEN="$3"
- echo "send request $WEB_SERVER_CONV_API_URL/getConfig/token/$TOKEN "
- OUT=$($HTTP_GET_TOOL "$WEB_SERVER_CONV_API_URL/getConfig/token/$TOKEN" )
- # get exit code
- RET=$?
- if [[ $RET -ne 0 ]] ; then
- echo "ERROR $HTTP_GET_TOOL $RET"
- exit
- fi
- echo "data $OUT"
- # парсинг данных
- echo "$OUT" | ./scripts/JSON.sh/JSON.sh -b 1> $DATA_FILE 2> $ERROR_FILE
- # проверка ошибки парсинга
- ERROR=$(< $ERROR_FILE)
- if [[ "$ERROR" != "" ]] ; then
- echo "Error parsing $ERROR"
- exit
- fi
- #configs paths
- PATH_BASE=$(getStringValue $DATA_FILE '\["tmp_path"\]')
- PATH_BASE_SOURCE=$PATH_BASE/source
- PATH_BASE_CONV=$PATH_BASE/conv
- PATH_BASE_PREVIEW=$PATH_BASE/preview
- URL_BASE_SOURCE_URL=$(getStringValue $DATA_FILE '\["source_url"\]')
- FTP_SERVER=$(getStringValue $DATA_FILE '\["conv_ftp_server"\]')
- FTP_PATH=$(getStringValue $DATA_FILE '\["conv_ftp_path"\]')
- FTP_USER=$(getStringValue $DATA_FILE '\["conv_ftp_user"\]')
- FTP_PASSWORD=$(getStringValue $DATA_FILE '\["conv_ftp_password"\]')
- RESERVE_FTP_SERVER=$(getStringValue $DATA_FILE '\["reserve_conv_ftp_server"\]')
- RESERVE_FTP_PATH=$(getStringValue $DATA_FILE '\["reserve_conv_ftp_path"\]')
- RESERVE_FTP_USER=$(getStringValue $DATA_FILE '\["reserve_conv_ftp_user"\]')
- RESERVE_FTP_PASSWORD=$(getStringValue $DATA_FILE '\["reserve_conv_ftp_password"\]')
- if [ ! -d "$PATH_BASE" ]; then
- mkdir -p $PATH_BASE
- fi
- if [ ! -d "$PATH_BASE_SOURCE" ]; then
- mkdir -p $PATH_BASE_SOURCE
- fi
- if [ ! -d "$PATH_BASE_CONV" ]; then
- mkdir -p $PATH_BASE_CONV
- fi
- if [ ! -d "$PATH_BASE_PREVIEW" ]; then
- mkdir -p $PATH_BASE_PREVIEW
- fi
- while true
- do
- # сохраняем
- echo "send request $WEB_SERVER_CONV_API_URL/gettasks/token/$TOKEN "
- OUT=$($HTTP_GET_TOOL $WEB_SERVER_CONV_API_URL/gettasks/token/$TOKEN )
- # get exit code
- RET=$?
- if [[ $RET -ne 0 ]] ; then
- # if error exit code, print exit code
- echo "ERROR $HTTP_GET_TOOL $RET"
- sleep 60
- continue
- fi
- echo "data $OUT"
- # парсинг данных
- echo "$OUT" | ./scripts/JSON.sh/JSON.sh -b 1> $DATA_FILE 2> $ERROR_FILE
- # проверка ошибки парсинга
- ERROR=$(< $ERROR_FILE)
- if [[ "$ERROR" != "" ]] ; then
- echo "Error parsing $ERROR"
- echo "output:"
- echo " $OUT"
- sleep 60
- continue
- fi
- STATUS=$(getStringValue $DATA_FILE '\["status"\]')
- echo "Status $STATUS"
- if [ "$STATUS" != "success" ]; then
- sleep 30
- continue
- fi
- LENGTH=$(getIntegerValue $DATA_FILE '\["data","length"\]')
- if [ "$LENGTH" == 0 ]; then
- sleep 60
- continue
- fi
- # Close STDOUT file descriptor
- #exec 1<&-
- # Close STDERR FD
- #exec 2<&-
- # Open STDOUT as $LOG_FILE file for read and write.
- #exec 1<>$ITEM_LOG
- # Redirect STDERR to STDOUT
- #exec 2>&1
- echo "Length $LENGTH"
- echo "Removing local conv files version"
- find "$PATH_BASE_CONV" -type f -delete
- echo "Removing local source files version"
- find "$PATH_BASE_SOURCE" -type f -delete
- echo "Removing local preview files version"
- find "$PATH_BASE_PREVIEW" -type f -delete
- DURATION=0
- FILEPNG=0
- FILENAME_SET=0
- MIDPOINTS=()
- FRAGMENTS=()
- file=$(getStringValue $DATA_FILE "\[\"data\",\"items\",0,\"file\"\]")
- file_name=$(getStringValue $DATA_FILE "\[\"data\",\"items\",0,\"file_name\"\]")
- picture_name_on_disk=$(getStringValue $DATA_FILE "\[\"data\",\"items\",0,\"picture_name_on_disk\"\]")
- video_id=$(getIntegerValue $DATA_FILE "\[\"data\",\"items\",0,\"video_id\"\]")
- user_id=$(getIntegerValue $DATA_FILE "\[\"data\",\"items\",0,\"user_id\"\]")
- queue_id=$(getIntegerValue $DATA_FILE "\[\"data\",\"items\",0,\"id\"\]")
- if ! [ -f "$PATH_BASE_SOURCE/$file" ];
- then
- # скачать если сервер
- $FTP_GET_TOOL $PATH_BASE_SOURCE/$file $URL_BASE_SOURCE_URL/$file
- fi
- if ! [ -f "$PATH_BASE_SOURCE/$file" ]; then
- echo "No file to encode: $PATH_BASE_SOURCE/$file"
- $HTTP_GET_TOOL $WEB_SERVER_CONV_API_URL/setstate/id/$queue_id/token/$TOKEN/state/error/errno/2/?error=ERROR_DOWNLOAD_SOURCE
- else
- $HTTP_GET_TOOL $WEB_SERVER_CONV_API_URL/setstate/id/$queue_id/token/$TOKEN/state/downloaded
- #если переданы параметры фрагментации видео
- fragmentLENGTH=$(getIntegerValue $DATA_FILE "\[\"data\",\"items\",0,\"fragmentation\",\"length\"\]")
- if (( $fragmentLENGTH > 0 )); then
- echo "Start fragmentation"
- CONCAT_STRING=""
- #переменная сдвига мидпоинтов после обрезки роликов
- MIDPOINTS_STEPS=0
- for fragmentINDEX in `seq 0 $(($fragmentLENGTH-1))`;
- do
- FRAGMENT_START=$(getStringValue $DATA_FILE "\[\"data\",\"items\",0,\"fragmentation\",\"fragments\",$fragmentINDEX,\"start\"\]")
- FRAGMENT_LENGTH=$(getStringValue $DATA_FILE "\[\"data\",\"items\",0,\"fragmentation\",\"fragments\",$fragmentINDEX,\"length\"\]")
- if [ "$FRAGMENT_LENGTH" > 0 ]; then
- echo "Start crop[$FRAGMENT_START:$FRAGMENT_LENGTH]"
- fragment_filename="part$fragmentINDEX.ts"
- #AVCONV_STR="$PATH_BASE_SOURCE/$file -ss $FRAGMENT_START -t $FRAGMENT_LENGTH -vcodec libx264 -acodec aac -bsf:v h264_mp4toannexb -f mpegts -strict experimental -y -threads auto $PATH_BASE_SOURCE/$fragment_filename"
- AVCONV_STR=""
- AVCONV_STR=$( cat <<EOF
- -ss $FRAGMENT_START \
- -t $FRAGMENT_LENGTH \
- -vcodec libx264 -acodec aac -bsf:v h264_mp4toannexb \
- -f mpegts -strict experimental -y -threads auto \
- $PATH_BASE_SOURCE/$fragment_filename
- EOF
- )
- avconv -i "$PATH_BASE_SOURCE/$file" $AVCONV_STR
- echo "End crop[$FRAGMENT_START:$FRAGMENT_LENGTH]"
- if [ -f "$PATH_BASE_SOURCE/$fragment_filename" ]; then
- MIDPOINTS=("${MIDPOINTS[@]}" "$MIDPOINTS_STEPS")
- #сдвигаем мидпоинт на длину фрагмента
- MIDPOINTS_STEPS="$(echo "$MIDPOINTS_STEPS+$FRAGMENT_LENGTH" | bc)"
- FRAGMENTS=("${FRAGMENTS[@]}" "$PATH_BASE_SOURCE/$fragment_filename")
- if (( "$fragmentINDEX" > 0 )); then
- CONCAT_STRING="$CONCAT_STRING|$PATH_BASE_SOURCE/$fragment_filename"
- else
- CONCAT_STRING="$PATH_BASE_SOURCE/$fragment_filename"
- fi
- fi
- fi
- done
- #MIDPOINTS_STRING=$(IFS=,; echo "${MIDPOINTS[*]}")
- SPLIT_FILENAME="split.mp4"
- #Если созданы фрагменты
- if (( ${#FRAGMENTS[@]} > 0 )); then
- FRAGMENTS_JOIN=$(IFS='|'; echo "${FRAGMENTS[*]}")
- echo "Split fragments: $FRAGMENTS_JOIN"
- COMMAND_SPLIT=$( cat <<EOF
- -c copy \
- -bsf:a aac_adtstoasc \
- -y $PATH_BASE_SOURCE/$SPLIT_FILENAME
- EOF
- )
- avconv -i "concat:$FRAGMENTS_JOIN" $COMMAND_SPLIT
- #проверка, что новый файл собран
- if [ -f "$PATH_BASE_SOURCE/$SPLIT_FILENAME" ]; then
- file="$SPLIT_FILENAME"
- fi
- #отправляем новые midpoints
- if (( ${#MIDPOINTS[@]} > 0)); then
- MIDPOINTS_CONCAT=$(IFS='|'; echo "${MIDPOINTS[*]}")
- $HTTP_GET_TOOL $WEB_SERVER_CONV_API_URL/setmidpoints/video_id/$video_id/token/$TOKEN/?midpoints=$MIDPOINTS_CONCAT
- fi
- fi
- fi
- echo "Start encode. $file "
- AVPROBE_STR=$( avprobe -v 0 -of ini -show_format -show_streams "$PATH_BASE_SOURCE/$file" )
- if ! echo "$AVPROBE_STR" | grep -q "width" ; then
- echo "$AVPROBE_STR"
- $HTTP_GET_TOOL $WEB_SERVER_CONV_API_URL/setstate/id/$queue_id/token/$TOKEN/state/error/errno/3/?error=ERROR_AVPROBE_ENCODE
- continue
- fi
- if echo "$AVPROBE_STR" | grep -q "duration" ; then
- if [ $DURATION == "0" ]
- then
- DURATION=$(echo "$AVPROBE_STR" | grep -m 1 "duration" | sed -n 's/duration=\([0-9]*\).*/\1/p')
- $HTTP_GET_TOOL $WEB_SERVER_CONV_API_URL/setduration/video_id/$video_id/token/$TOKEN/duration/$DURATION
- fi
- if [ -z "$picture_name_on_disk" ] && [ $FILEPNG == "0" ]
- then
- FILEPNG=$(basename $file).jpeg
- FILEPNG2=$(basename $file)_2.jpeg
- FILEPNG3=$(basename $file)_3.jpeg
- FILEPNG4=$(basename $file)_4.jpeg
- FILEPNG5=$(basename $file)_5.jpeg
- avconv -noaccurate_seek -ss $(( (DURATION * 25) / 100 )) -i "$PATH_BASE_SOURCE/$file" -f image2 -vframes 1 $PATH_BASE_PREVIEW/$FILEPNG
- if ! [ -f "$PATH_BASE_PREVIEW/$FILEPNG" ]; then
- avconv -noaccurate_seek -ss $(( (DURATION * 10) / 100 )) -i "$PATH_BASE_SOURCE/$file" -f image2 -vframes 1 $PATH_BASE_PREVIEW/$FILEPNG
- fi
- #A/B TEST допилка by ashmits 11.02.2015
- #avconv -noaccurate_seek -ss $(( (DURATION * 11) / 100 )) -i "$PATH_BASE_SOURCE/$file" -f image2 -vframes 1 $PATH_BASE_PREVIEW/$FILEPNG2
- #avconv -noaccurate_seek -ss $(( (DURATION * 12) / 100 )) -i "$PATH_BASE_SOURCE/$file" -f image2 -vframes 1 $PATH_BASE_PREVIEW/$FILEPNG3
- #avconv -noaccurate_seek -ss $(( (DURATION * 13) / 100 )) -i "$PATH_BASE_SOURCE/$file" -f image2 -vframes 1 $PATH_BASE_PREVIEW/$FILEPNG4
- #avconv -noaccurate_seek -ss $(( (DURATION * 14) / 100 )) -i "$PATH_BASE_SOURCE/$file" -f image2 -vframes 1 $PATH_BASE_PREVIEW/$FILEPNG5
- #curl -F "file[]=@$PATH_BASE_PREVIEW/$FILEPNG" -F "file[]=@$PATH_BASE_PREVIEW/$FILEPNG2" -F "file[]=@$PATH_BASE_PREVIEW/$FILEPNG3" -F "file[]=@$PATH_BASE_PREVIEW/$FILEPNG4" -F "file[]=@$PATH_BASE_PREVIEW/$FILEPNG5" $WEB_SERVER_CONV_API_URL/setVideoPriview/token/$TOKEN/uploader_id/file/video_id/$video_id
- curl -F "file=@$PATH_BASE_PREVIEW/$FILEPNG" $WEB_SERVER_CONV_API_URL/setVideoPriview/token/$TOKEN/uploader_id/file/video_id/$video_id
- echo "file : $file"
- echo "VIDEO_ID : $video_id"
- fi
- fi
- fi
- AVCONV_STR=""
- for INDEX in `seq 0 $(($LENGTH-1))`;
- do
- echo "index = $INDEX"
- queue_id=$(getIntegerValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"id\"\]")
- vcodec=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"vcodec\"\]")
- acodec=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"acodec\"\]")
- width=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"width\"\]")
- height=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"height\"\]")
- format=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"format\"\]")
- vbitrate=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"vbitrate\"\]")
- abitrate=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"abitrate\"\]")
- crf=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"crf\"\]")
- maxrate=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"maxrate\"\]")
- bufsize=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"bufsize\"\]")
- ac=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"ac\"\]")
- freq=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"freq\"\]")
- fps=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"fps\"\]")
- ext=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"ext\"\]")
- vpreset=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"vpreset\"\]")
- custom_args_str=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"custom_args_str\"\]")
- FILENAME=${file_name}_${height}p.${ext}
- AVCONV_STR=$( cat <<EOF
- -vcodec libx264 $custom_args_str -preset:v $vpreset -threads auto \
- -filter:v scale=w=trunc((iw*min($height\,ih)/ih)/2)*2:h=trunc(min($height\,ih)/2)*2 \
- -maxrate $maxrate -bufsize:v $bufsize -r $fps -crf $crf \
- -acodec libfdk_aac -ab $abitrate -ac $ac \
- $PATH_BASE_CONV/$FILENAME \
- $AVCONV_STR
- EOF
- )
- done
- avconv -i "$PATH_BASE_SOURCE/$file" \
- $AVCONV_STR
- echo $AVCONV_STR
- echo "Done encode"
- for INDEX in `seq 0 $(($LENGTH-1))`;
- do
- echo "index = $INDEX"
- queue_id=$(getIntegerValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"id\"\]")
- video_id=$(getIntegerValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"video_id\"\]")
- user_id=$(getIntegerValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"user_id\"\]")
- vcodec=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"vcodec\"\]")
- acodec=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"acodec\"\]")
- width=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"width\"\]")
- height=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"height\"\]")
- format=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"format\"\]")
- vbitrate=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"vbitrate\"\]")
- abitrate=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"abitrate\"\]")
- ac=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"ac\"\]")
- freq=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"freq\"\]")
- fps=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"fps\"\]")
- ext=$(getStringValue $DATA_FILE "\[\"data\",\"items\",$INDEX,\"ext\"\]")
- FILENAME=${file_name}_${height}p.${ext}
- if ! [ -f "$PATH_BASE_CONV/$FILENAME" ]; then
- echo "file not encoded: $PATH_BASE_SOURCE/$file"
- $HTTP_GET_TOOL $WEB_SERVER_CONV_API_URL/setstate/id/$queue_id/token/$TOKEN/state/error/errno/4/?error=ERROR_WHILE_ENCODE
- sleep 10
- continue
- else
- $HTTP_GET_TOOL $WEB_SERVER_CONV_API_URL/setstate/id/$queue_id/token/$TOKEN/state/converted/?filename=$FILENAME
- fi
- # fix permision
- chmod 755 "$PATH_BASE_CONV/$FILENAME"
- FTP_FILE_PATH="$FTP_PATH/$user_id"
- echo "Uploading to FTP"
- ftp -inv $FTP_SERVER >$FTP_LOG <<END_FTP
- quote USER $FTP_USER
- quote PASS $FTP_PASSWORD
- passive mode on
- binary
- mkdir "$FTP_FILE_PATH"
- put "$PATH_BASE_CONV/$FILENAME" "$FTP_FILE_PATH/$FILENAME"
- quit
- END_FTP
- FTP_SUCCESS_MSG="226" # 226 Закрытие канала, обмен завершен успешно
- if fgrep "$FTP_SUCCESS_MSG" $FTP_LOG ;then
- $HTTP_GET_TOOL $WEB_SERVER_CONV_API_URL/setstate/id/$queue_id/token/$TOKEN/state/uploaded
- echo "ftp OK"
- else
- echo "ftp Error:"
- $HTTP_GET_TOOL $WEB_SERVER_CONV_API_URL/setstate/id/$queue_id/token/$TOKEN/state/error/errno/5?error=CDN_ERROR
- cat $FTP_LOG
- continue
- fi
- if [ $RESERVE_FTP_SERVER != "off" ] ;then
- RESERVE_FTP_FILE_PATH="$RESERVE_FTP_PATH/$user_id"
- echo "Uploading to reserve FTP"
- ftp -inv $RESERVE_FTP_SERVER >$FTP_LOG <<END_FTP
- quote USER $RESERVE_FTP_USER
- quote PASS $RESERVE_FTP_PASSWORD
- passive mode on
- binary
- mkdir "$RESERVE_FTP_FILE_PATH"
- put "$PATH_BASE_CONV/$FILENAME" "$RESERVE_FTP_FILE_PATH/$FILENAME"
- quit
- END_FTP
- FTP_SUCCESS_MSG="226" # 226 Закрытие канала, обмен завершен успешно
- if fgrep "$FTP_SUCCESS_MSG" $FTP_LOG ;then
- $HTTP_GET_TOOL $WEB_SERVER_CONV_API_URL/setstate/id/$queue_id/token/$TOKEN/state/uploaded
- echo "reserve ftp OK"
- else
- echo "reserve ftp Error:"
- $HTTP_GET_TOOL $WEB_SERVER_CONV_API_URL/setstate/id/$queue_id/token/$TOKEN/state/error/errno/5?error=CDN_ERROR
- cat $FTP_LOG
- continue
- fi
- fi
- done
- sleep 5
- done
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement