#!/usr/bin/perl -w

use strict;

use Getopt::Std;
use Cwd;

use IO::File;

# ------------------------------------------------------


my $ecldir       = "/prog/res/ecl/grid";
my $eclolddir    = "/prog/ecl/grid";
my $ecltestdir   = "/prog/res/ecl/testgrid";
my $mpiintelroot = "/prog/res/ecl/grid/tools/linux_x86_64/intel/mpi/2018.1.163";

my $lisens = "7321\@eclipse-lic-no.statoil.no";

my $extconfig   = "macros/CONFIG.ECL";
my $extmpirun64 = "tools/linux_x86_64/mpich_x86_64/bin/mpirun";

my $config = "$ecldir/$extconfig";

my $mpirun64 = "$ecldir/$extmpirun64";

# -----------------------------------------------------

my $arch64 = "x86_64Linux";    # bsub resource string for 64bit

# ---------------------------------------------------

my $que_name  = "daily";
my $default_version = "2023.1";
my $version   = `$ecldir/macros/eclrun --report-versions eclipse | xargs -n1 | sort -gr | head -1`;
chomp($version);
if (!$version) {$version = $default_version}
my $memoryreq = 512;

my $progname    = "eclipse";
my $f64bit      = 0;
my $span        = 0;
my $interactive = 0;
my $parallel    = 0;

my $testversion = 0;

my $scali       = 0;
my $intelmpi    = 0;
my $mpiextchmpi = "mpi";
my $mpiextscali = "scampi";
my $mpiextintel = "ilmpi";

my $mpirunscali = "/opt/scali/bin/mpirun";
my $mpirunintel = "$mpiintelroot/bin64/mpirun";

my $mpiext = $mpiextchmpi;
my $mpirun = $mpirun64;

# ------------------------------------------------------



sub find_number_of_processors;
sub find_allocated_processors;
sub build_batch;
sub build_parallel_batch;
sub submit_batch;
sub in_error;

my $workdir = cwd;

my $scriptname = "runeclipse";

my $usage =
  "usage $scriptname  [-p program] [-q que_name] [-v version] [-x]  [-i] root";

#
# Get command line options:
#

my %options = ();

getopts( "p:q:v:m:s:c:iLhx", \%options );

if ( $options{h} ) {
    print "$usage\n\n";
    print "   program  is either eclipse or e300, default value: $progname\n";
    print "   que_name is a LSF que name, default value: $que_name\n";
    print "   version  is a valid ECLIPSE version, default: $version\n";
    print "   -x       Force execution on single host for parallell runs\n";
    print "   -i       Selects interactive execution\n";
    exit;
}

if ( $options{p} )         { $progname  = $options{p}; }
if ( $progname eq "e100" ) { $progname  = "eclipse" }
if ( $options{q} )         { $que_name  = $options{q}; }
if ( $options{v} )         { $version   = $options{v}; }
if ( $options{m} )         { $memoryreq = $options{m}; }
my $swapreq = $memoryreq;
if ( $options{s} ) { $swapreq     = $options{s}; }
if ( $options{x} ) { $span        = 1 }
if ( $options{i} ) { $interactive = 1 }
if ( $options{L} ) { $f64bit      = 1 }

# Extract datafile name (and possibly working directory)
#  from command line argument:

my $rootname;

my $numargs = @ARGV;
if ( $numargs == 0 ) {
    in_error("Missing root name,\n$usage\nFor help:$scriptname -h");
}
else {
    my ($argname) = @ARGV;
    $argname =~ s/\.DATA$//;
    if ( $argname =~ /^(\/\S+\/)(\S+)$/ ) {
        $workdir  = $1;
        $rootname = $2;
    }
    elsif ( $argname =~ /^(\S+\/)(\S+)$/ ) {
        $workdir  = $workdir . "/" . $1;
        $rootname = $2;
    }
    else {
        $rootname = $argname;
    }
}

if ( !-f "$workdir/$rootname\.DATA" ) {
    in_error("Datafile $workdir/$rootname\.DATA does not exist");
}

# Remove the files that marks the end of simulation (*.ECLEND and *.job.done):

my $eclendfile  = "$workdir/$rootname\.ECLEND";
my $eclendfile2 = "$workdir/$rootname\.job\.done";
if ( -f $eclendfile ) {
    unlink $eclendfile;
}
if ( -f $eclendfile2 ) {
    unlink $eclendfile2;
}

#
# Extract version year to be able to cater to changes in directory structure:
#

my $version_year = substr( $version, 0, 4 );

# Check for parallel job

$parallel = find_number_of_processors($rootname);
my $local = 0;    # Local parallelinteractive job

if ($parallel) {
    if ( $version_year > 2010 ) {    # Use Intel MPI
        $intelmpi = 1;
        $mpirun   = $mpirunintel;
        $mpiext   = $mpiextintel;
    }
    elsif ( $version_year > 2006 ) {    # Use SCALI
	in_error("Parallel runs for version < 2011 is no longer supported, sorry");
        $scali  = 1;
        $mpirun = $mpirunscali;
        $mpiext = $mpiextscali;
    }
    if ($interactive) {
        my $allocproc = find_allocated_processors;
        if ( $allocproc < $parallel ) {
            if ($allocproc) {
                in_error(
"Mismath between LSF allocated processors ($allocproc) and processors needed by parallel Eclipse ($parallel)"
                );
            }
            else {
                $local = 1;
                print "** Interactive parallel job: $parallel processors\n";

        #                in_error("Unable to run parallel job as interactive") ;
            }
        }
    }
    $progname = "$progname" . "_" . $mpiext;
}

#
# Test if executable exists. If not; try old then test installation:
#

my $exe64;

$exe64 = "$ecldir/$version/bin/linux_x86_64/$progname.exe" ;

if ( !-f $exe64 ) {
    # Trying the old installation
    $ecldir = $eclolddir;
    $exe64  = "$ecldir/$version/bin/linux_x86_64/$progname.exe";
    $config = "$ecldir/$extconfig";

    if ($version_year < 2006) {
	$exe64 = "$ecldir/$version/bin/linux/$progname.exe" ;
    } else {
	$exe64 = "$ecldir/$version/bin/linux_x86_64/$progname.exe" ;
} ;

}

if ( !-f $exe64 ) {
    # Trying the test installation
    $ecldir = $ecltestdir;
    $exe64  = "$ecldir/$version/bin/linux_x86_64/$progname.exe";
    $config = "$ecldir/$extconfig";

    if ( !-f $exe64 ) {
        in_error("Executable $version does not exist");
    }

    $testversion = 1;
}

#
# Forward stdout to rootname.OUT for versions 2019.3 and younger for consistency with eclrun
# Symlink rootname.LOG -> rootname.OUT for backwards compatibility
#

my $stdout_file = "$rootname.LOG";
if (($version_year > 2019 ) || ($version eq "2019.3" )) {
    $stdout_file = "$rootname.OUT";
    if ($interactive != 1){
	symlink ($stdout_file, "$rootname.LOG");
    }
}

#
# Build batchfile:
#

my $batchfilename = "$workdir/$rootname" . ".job";

if ($parallel) {
    build_parallel_batch;
}
else {
    build_batch;
}

#
# Submit batchfile to que:
#

submit_batch();

sub submit_batch {
    my $bsub;
    my $line;
    my $nErrStr;

    if ($interactive) {
        $bsub = $batchfilename;
        system($bsub);

        if ( !open( ECLENDFILE, "<$eclendfile" ) ) {
            in_error("Cannot open ECLEND file $eclendfile");
        }
        while ( $line = <ECLENDFILE> ) {
            if ( index( $line, "Errors" ) != -1 ) {
                $nErrStr = substr( $line, 20, 50 );
            }
        }
        if ( $nErrStr > 0 ) {
            exit 1;
        }
        else {
            exit 0;
        }
    }
    else {
        my $architectures;
        if ($f64bit) {
            $architectures = $arch64;
        }
        else {
            $architectures = "$arch64";
        }
        if ($parallel) {    # Submit parallel batch job:
            if ($span) {
                $bsub =
"bsub -L /bin/csh -q $que_name -n $parallel -o $workdir/$stdout_file -J \"$progname $rootname\" -R \"select[$architectures] rusage[mem=$memoryreq:swap=$swapreq] same[type:model] span[hosts=1]\"  $batchfilename";
            }
            else {
                $bsub =
"bsub -L /bin/csh -q $que_name -n $parallel -o $workdir/$stdout_file -J \"$progname $rootname\" -R \"select[$architectures] rusage[mem=$memoryreq:swap=$swapreq] same[type:model]\"  $batchfilename";
            }
        }
        else {              # Submit serial job:
            $bsub =
"bsub  -L /bin/csh -q $que_name -o $workdir/$stdout_file -J \"$progname $rootname\" -R \"select[$architectures] rusage[mem=$memoryreq:swap=$swapreq]\"  $batchfilename";
        }
        my $bsubmess = `$bsub`;

        if ( $bsubmess =~ /Job <(\d+)>/ ) {
            print "$1\n";
        }
        else {
            print "-1\n";
        }
    }
}

sub find_number_of_processors {
    my ($rootname) = @_;

    my $datafilename = $workdir . "\/" . $rootname . ".DATA";
    my $line;
    my $found = 0;

    if ( !open( DATAFIL, "<$datafilename" ) ) {
        in_error("$scriptname: Cannot open DATA file $datafilename");
    }
    while ( $line = <DATAFIL> ) {
        if ( $line =~ /^\s*PARALLEL/ ) {
            $found = 1;
            last;
        }
    }
    if ($found) {
        my $numproc;
        $line = <DATAFIL>;
        while ( $line =~ /^--/ ) {    # skip comment lines
            $line = <DATAFIL>;
        }
        $line =~ s/^\s+//;            # Remove initial spaces
        ($numproc) = split /\s+/, $line;    # Read first token
        close(DATAFIL);
        return $numproc;
    }
    else {
        return find_number_of_slaves($datafilename)
          ;                                 # Check for reservoir coupling

    }
}

sub find_number_of_slaves {
    my ($datafilename) = @_;
    my $datafile = new IO::File;
    if ( !$datafile->open("< $datafilename") ) {
        in_error("Cannot open DATA file $datafilename");
    }

    my $line;
    my $found = 0;

    while ( !$found && ( $line = <$datafile> ) ) {

        if ( $line =~ /^--/ ) { next; }
        ;    # Skip comments

        if ( $line =~ /^\s*SLAVES/ ) { $found = 1; }

    }
    if ($found) {
        my $nslaves = 1;    # The master uses one processor
        for ( $line = <$datafile> ; $line ; $line = <$datafile> ) {
            if ( $line =~ /^\s*\// ) { last; }
            if ( $line =~ /^\s*\S+\s+\S+\s+\S+\s+\S+\s+(\d+)\s*\// ) {
                $nslaves = $nslaves + $1;
            }
            elsif ( $line =~ /^\s*\S+\s+\S+\s+\S+\s+\S+\s*\// ) {
                $nslaves = $nslaves + 1;
            }
        }
        return $nslaves;
    }
    else {
        return 0;
    }
}

sub find_allocated_processors {
    my $numproc = 0;

    if ( exists( $ENV{'LSB_MCPU_HOSTS'} ) ) {
        my $allocstring = $ENV{'LSB_MCPU_HOSTS'};
        while ( $allocstring =~ s/^\s*(\S+)\s+(\d+)// ) {
            $numproc += $2;
        }
    }
    return $numproc;
}

sub build_batch {
    open( BATCHFIL, ">$batchfilename" )
      or die "$scriptname: Cannot create batch file $batchfilename\n";

    print BATCHFIL "#!/bin/sh \n";
    if ($testversion) {
        print BATCHFIL "echo \"*** RUNNING TEST VERSION $version ****\"\n";
    }
    print BATCHFIL "export LM_LICENSE_FILE=$lisens\n";
    print BATCHFIL "export F_UFMTENDIAN=big\n";
    print BATCHFIL "export ARCH=\`uname -m\`\n";
    print BATCHFIL "cd $workdir\n";
    print BATCHFIL "/bin/rm -f ECL.CFG\n";
    print BATCHFIL "/bin/cp $config ./ECL.CFG 2>> $rootname" . ".ERR\n";
    print BATCHFIL "executable=$exe64\n";
    print BATCHFIL "\$executable <<END ";

    if ($interactive) {
        print BATCHFIL "\n";
    }
    else {
        print BATCHFIL "> $stdout_file 2> $rootname.ERR\n";
    }
    print BATCHFIL "$rootname\n";
    print BATCHFIL "$rootname\n";
    print BATCHFIL "30000000\n";
    print BATCHFIL "30000000\n";
    print BATCHFIL "\$executable\n";
    print BATCHFIL "END\n";
    print BATCHFIL "mv -f $batchfilename ${batchfilename}.done\n";

    if ($testversion) {
        print BATCHFIL
"echo \"*** NOTE THAT $version IS STILL A TEST VERSION IN Equinor ****\"\n";
    }

    close(BATCHFIL);
    chmod( 0770, $batchfilename );
}

sub build_parallel_batch {
    open( BATCHFIL, ">$batchfilename" )
      or die "$scriptname: Cannot create batch file $batchfilename\n";

    print BATCHFIL "#!/bin/sh \n";
    print BATCHFIL "export LM_LICENSE_FILE=$lisens\n";

    #    print BATCHFIL
    #        "export FLEXLM_NO_CKOUT_INSTALL_LIC=1\n" ;

    print BATCHFIL "export ECLARCH=$ecldir\n";
    print BATCHFIL "export F_UFMTENDIAN=big\n";
    print BATCHFIL "export P4_RSHCOMMAND=ssh\n";    # Variable used by mpi
    print BATCHFIL "export ARCH=\`uname -m\`\n";
    print BATCHFIL "/bin/rm -f ECL.CFG\n";
    print BATCHFIL "cd $workdir\n";
    print BATCHFIL "/bin/cp $config ./ECL.CFG 2>> $rootname" . ".ERR\n";
    print BATCHFIL "executable=$exe64\n";
    print BATCHFIL "mpicommand=$mpirun\n";

    if ($intelmpi) {
        print BATCHFIL "export I_MPI_ROOT=$mpiintelroot/\n";
        print BATCHFIL "export PATH=$mpiintelroot/bin64:\$PATH\n";
        print BATCHFIL
"export LD_LIBRARY_PATH=$mpiintelroot/lib64:$ecldir/$version/lib/linux_x86_64:\$LD_LIBRARY_PATH\n";
    }
    elsif ($scali) {
        print BATCHFIL
"export LD_LIBRARY_PATH=$ecldir/$version/lib/linux_x86_64:/opt/scali/lib64\n";
    }
    else {
        print BATCHFIL
          "export LD_LIBRARY_PATH=$ecldir/$version/lib/linux_x86_64\n";
    }
    print BATCHFIL "machinefile=$rootname.mpi.\$\$\n";

    if ( !$local ) {

        print BATCHFIL "((tnum=0))\n";
        print BATCHFIL "for ((num=1 ; tnum<$parallel ; num++)) ; do\n";
        print BATCHFIL "   ((nhost = 2*num - 1))\n";
        print BATCHFIL "   ((nnum  = 2*num))\n";
        print BATCHFIL
          "   thishost=\`echo \$LSB_MCPU_HOSTS | cut -f\$nhost -d\" \"\`\n";
        print BATCHFIL
          "   thisnum=\`echo \$LSB_MCPU_HOSTS | cut -f\$nnum -d\" \"\`\n";
        print BATCHFIL "   for ((mnum=0 ; mnum<\$thisnum ; mnum++)) ; do\n";
        print BATCHFIL "      if ((num==1 && mnum==0)) ; then\n";
        print BATCHFIL "         echo \$thishost > \$machinefile\n";
        print BATCHFIL "      else\n";
        print BATCHFIL "         echo \$thishost >> \$machinefile\n";
        print BATCHFIL "      fi\n";
        print BATCHFIL "   done\n";
        print BATCHFIL "   ((tnum=tnum+thisnum))\n";
        print BATCHFIL "done\n";
    }

    if ($local) {
        print BATCHFIL "\$mpicommand -np $parallel \$executable $rootname \n";
    }
    elsif ($interactive) {
        print BATCHFIL
"\$mpicommand -machinefile \$machinefile -np $parallel \$executable $rootname \n";
    }
    else {
        print BATCHFIL
"\$mpicommand -machinefile \$machinefile -np $parallel \$executable $rootname > $stdout_file 2> $rootname.ERR \n";
    }
    print BATCHFIL "rm -f \$machinefile\n";
    print BATCHFIL "if [ -f ECL.CFG ] ; then\n";
    print BATCHFIL "   rm ECL.CFG\n";
    print BATCHFIL "fi\n";
    print BATCHFIL "mv -f $batchfilename ${batchfilename}.done\n";

    close(BATCHFIL);
    chmod( 0770, $batchfilename );
}

sub in_error {
    my ($message) = @_;

    if ($interactive) {
        print "ERROR $scriptname: $message\n";
    }
    else {
        print "-1\n";
    }
    exit 1;
}
