#!/bin/bash
#SBATCH -n 1
#SBATCH -c 36
#SBATCH --gres=gpu:4
#SBATCH --mem=120G
#SBATCH --mail-type=END

## The following sends a SIGHUP signal 900 seconds before the job
## gets sent a kill signal on TimeOut. We can thus premptively clean
## up and export any data saved automatically.

## It is possible to use SIGUSR1, SIGUSR2 as well.

#SBATCH --signal=B:HUP@900


function _export {
    ssh $USER@ada "mkdir -p $REMOTE_DIR/"
    rsync -rz $CHECKPOINTS/checkpoint_{best,last}.pt ada:$REMOTE_DIR/
}

## We trap the SIGHUP signal and register a handler, which saves computed checkpoint
## In my case.

trap "_export" SIGHUP