#!/bin/bash
#SBATCH -D /imk/imk-bignas/developers/czotti/training/brain_tumor/work/3f/a303188b4d73d7132d770303d1f04b
#SBATCH -J nf-gpu_output_(2)
#SBATCH -o /imk/imk-bignas/developers/czotti/training/brain_tumor/work/3f/a303188b4d73d7132d770303d1f04b/.command.log
#SBATCH --no-requeue
#SBATCH -p gpu
#SBATCH --gres=gpu:gtx:1

# NEXTFLOW TASK: gpu_output (2)
set -e
set -u
NXF_DEBUG=${NXF_DEBUG:=0}; [[ $NXF_DEBUG > 1 ]] && set -x

nxf_env() {
    echo '============= task environment ============='
    env | sort | sed "s/\(.*\)AWS\(.*\)=\(.\{6\}\).*/\1AWS\2=\3xxxxxxxxxxxxx/"
    echo '============= task output =================='
}

nxf_kill() {
    declare -a ALL_CHILD
    while read P PP;do
        ALL_CHILD[$PP]+=" $P"
    done < <(ps -e -o pid= -o ppid=)

    walk() {
        [[ $1 != $$ ]] && kill $1 2>/dev/null || true
        for i in ${ALL_CHILD[$1]:=}; do walk $i; done
    }

    walk $1
}

nxf_mktemp() {
    local base=${1:-/tmp}
    if [[ $(uname) = Darwin ]]; then mktemp -d $base/nxf.XXXXXXXXXX
    else TMPDIR="$base" mktemp -d -t nxf.XXXXXXXXXX
    fi
}

on_exit() {
  exit_status=${ret:=$?}
  printf $exit_status > /imk/imk-bignas/developers/czotti/training/brain_tumor/work/3f/a303188b4d73d7132d770303d1f04b/.exitcode
  set +u
  [[ "$tee1" ]] && kill $tee1 2>/dev/null
  [[ "$tee2" ]] && kill $tee2 2>/dev/null
  [[ "$ctmp" ]] && rm -rf $ctmp || true
  exit $exit_status
}

on_term() {
    set +e
    [[ "$pid" ]] && nxf_kill $pid
}

trap on_exit EXIT
trap on_term TERM INT USR1 USR2

export NXF_BOXID="nxf-$(dd bs=18 count=1 if=/dev/urandom 2>/dev/null | base64 | tr +/ 0A)"
NXF_SCRATCH=''
[[ $NXF_DEBUG > 0 ]] && nxf_env
touch /imk/imk-bignas/developers/czotti/training/brain_tumor/work/3f/a303188b4d73d7132d770303d1f04b/.command.begin
# task environment
nxf_taskenv() {
cat << EOF
export CUDA_VISIBLE_DEVICES="\$CUDA_VISIBLE_DEVICES"
export PATH="/imk/imk-bignas/developers/czotti/training/brain_tumor/bin:\$PATH"
EOF
}

[[ $NXF_SCRATCH ]] && echo "nxf-scratch-dir $HOSTNAME:$NXF_SCRATCH" && cd $NXF_SCRATCH

set +e
ctmp=$(set +u; nxf_mktemp /dev/shm 2>/dev/null || nxf_mktemp $TMPDIR)
cout=$ctmp/.command.out; mkfifo $cout
cerr=$ctmp/.command.err; mkfifo $cerr
tee .command.out < $cout &
tee1=$!
tee .command.err < $cerr >&2 &
tee2=$!
(
set +u; env - PATH="$PATH" SINGULARITYENV_TMP="$TMP" SINGULARITYENV_TMPDIR="$TMPDIR" singularity exec --nv /imk/imk-bignas/developers/czotti/containers/pytorch_20180629.img /bin/bash -c "cd $PWD; eval $(nxf_taskenv); /bin/bash -ue /imk/imk-bignas/developers/czotti/training/brain_tumor/work/3f/a303188b4d73d7132d770303d1f04b/.command.sh"
) >$cout 2>$cerr &
pid=$!
wait $pid || ret=$?
wait $tee1 $tee2