extract.pl

#!/usr/bin/perl
my $limit = $ARGV[0];
my $curr_base = 0;
my $end = 0;
my $tot_reads = 0;

while(<STDIN>) {
    chomp;
    my $line = $_;
    my @q = split(/\s+/, $line);
    print STDERR "Log > File: $q[3]\n";

    $file = $q[3];
    my $command = "aws s3 cp s3://giab/$q[3] - | pigz -dc";
    open my $cmd_fh, "$command |";

    my $ind = 0;
    my $chunk_reads = 0;
    my $t1 = "";
    my $t2 = "";
    my $t3 = "";
    my $t4 = "";
    while (<$cmd_fh>) {
    chomp;
    my $line2 = $_;
    $ind++;
    if ($ind % 4 == 2) {
        $curr_base += length($line2);
        $t2 = $line2;
    } elsif ($ind % 4 == 1) {
        $t1 = $line2;
    } elsif ($ind % 4 == 3) {
        $t3 = $line2;
    } else {
        if ($curr_base < $limit) {
            print "$t1\n$t2\n$t3\n$line2\n";
            $tot_reads += 1;
            $chunk_reads += 1;
            print STDERR "Log > progress: $chunk_reads / $tot_reads $q[3]\n" if ($tot_reads % 100000 == 0);
        } else {
            $end = 1;
            last;
        }
    }
    }
    close $cmd_fh;
    last if ($end == 1);
}

#my $t1 = "";
#my $t2 = "";
#my $t3 = "";
#my $t4 = "";
#while(<>) {
#   chomp;
#   my $line = $_;
#   $ind++;
#   if ($ind % 4 == 2) {
#       $curr_base += length($line);
#       $t2 = $line;
#   } elsif ($ind % 4 == 1) {
#       $t1 = $line;
#   } elsif ($ind % 4 == 3) {
#       $t3 = $line;
#   } else {
#       if ($curr_base < $limit) {
#           print "$t1\n$t2\n$t3\n$line\n";
#       } else {
#           last;
#       }
#   }
#}