source: trunk/LMDZ.COMMON/libf/evolution/deftank/lib_launchPEM.sh @ 3363

#!/bin/bash
########################################################################
######## Library of bash functions for the PEM launching script ########
########################################################################

# To end the launching script
endlaunch() {
    # Restore the previous value of LC_NUMERIC
    LC_NUMERIC=$OLD_LC_NUMERIC

    date
    echo "Successful end of the launching script for the PEM simulation."
    exit 0
}

# To end the launching script with error
errlaunch() {
    # Restore the previous value of LC_NUMERIC
    LC_NUMERIC=$OLD_LC_NUMERIC

    date
    echo "End with error of the launching script for the PEM."
    exit 1
}

# To check if SLURM is the job scheduler
function is_slurm() {
    if [ ! -x $(command -v squeue) ]; then
        echo "Error: the job scheduler is not SLURM on $machine!"
        echo "You need to adapt the script to your case."
        errlaunch
    fi
}

# To check if everything necessary for the launching script is ok
checklaunch() {
    # Save the current value of LC_NUMERIC and set it to a locale that uses a dot as the decimal separator
    OLD_LC_NUMERIC=$LC_NUMERIC
    LC_NUMERIC=en_US.UTF-8

    is_slurm
    if [ -v n_mars_years ] && [ ! -z "$n_mars_years" ]; then
        if [ $n_mars_years -lt 1 ]; then
            echo "Error: the value of 'n_mars_years' must be >0!"
            errlaunch
        fi
    elif [ -v n_earth_years ] && [ ! -z "$n_earth_years" ]; then
        if [ $n_earth_years -lt 1 ]; then
            echo "Error: the value of 'n_earth_years' must be >0!"
            errlaunch
        fi
    else
        echo "Error: no number of years to be simulated has been set!"
        errlaunch
    fi
    if [ $nPCM_ini -lt 2 ] || [ -z "$nPCM_ini" ]; then
        echo "Error: the value of 'nPCM_ini' must be >1!"
        errlaunch
    fi
    if [ $nPCM -lt 2 ] || [ -z "$nPCM" ]; then
        echo "Error: the value of 'nPCM' must be >1!"
        errlaunch
    fi
    if [ ! -f "jobPCM.slurm" ]; then
        echo "Error: file \"jobPCM.slurm\" does not exist in $dir!"
        errlaunch
    fi
    if [ ! -f "jobPEM.slurm" ]; then
        echo "Error: file \"$jobPEM.slurm\" does not exist in $dir!"
        errlaunch
    fi
    if [ ! -f "run_PCM.def" ]; then
        echo "Error: file \"run_PCM.def\" does not exist in $dir!"
        errlaunch
    fi
    if [ ! -f "run_PEM.def" ]; then
        echo "Error: file \"run_PEM.def\" does not exist in $dir!"
        errlaunch
    fi
    if [ ! -f "context_lmdz_physics.xml" ]; then
        echo "Error: file \"context_lmdz_physics.xml\" does not exist in $dir!"
        errlaunch
    fi
    if [ ! -f "field_def_physics_mars.xml" ]; then
        echo "Error: file \"field_def_physics_mars.xml\" does not exist in $dir!"
        errlaunch
    fi
    if [ ! -f "file_def_physics_mars.xml" ]; then
        echo "Error: file \"file_def_physics_mars.xml\" does not exist in $dir!"
        errlaunch
    fi
    if [ ! -f "iodef.xml" ]; then
        echo "Error: file \"iodef.xml\" does not exist in $dir!"
        errlaunch
    fi
    if [ ! -d "out_PCM" ]; then
        mkdir out_PCM
    fi
    if [ ! -d "out_PEM" ]; then
        mkdir out_PEM
    fi
    if [ ! -d "starts" ]; then
        mkdir starts
    fi
    if [ ! -d "diags" ]; then
        mkdir diags
    fi
}

# To initialize the launching script
initlaunch() {
    echo "This is a chained simulation for PEM and PCM runs in $dir on $machine by $user."
    myear=686.9725      # Number of Earth days in Martian year
    eyear=365.256363004 # Number of days in Earth year
    convert_years=$(echo "$myear/$eyear" | bc -l)
    convert_years=$(printf "%.4f" $convert_years) # Rounding to the 4th decimal to respect the precision of Martian year
    if [ -v n_mars_years ]; then
        n_myear=$n_mars_years
        echo "Number of years to be simulated: $n_myear Martian years."
    elif [ -v n_earth_years ]; then
        n_myear=$(echo "($n_earth_years/$convert_years + 0.999999)/1" | bc) # Ceiling of n_earth_years/convert_years
        echo "Number of years to be simulated: $n_earth_years Earth years = $n_myear Martian years."
    fi
    i_myear=0
    iPEM=1
    iPCM=1
    cp startfi.nc starts/
    if [ -f "start.nc" ]; then
        cp start.nc starts/
    elif [ -f "star1D.nc" ]; then
        cp star1D.txt starts/
    fi

    # Create a file to manage years of the chained simulation and store some info from the PEM runs
    echo $i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini > info_PEM.txt
}

# To submit the PCM runs
submitPCM() {
    find . -type f -name "jobPCM*.slurm" ! -name "jobPCM.slurm" -delete
    ii=1
    if [ ! -z $2 ]; then
        ii=$2
    fi
    if [ $i_myear -lt $n_myear ]; then
        echo "Run PCM $iPCM: call $ii/$1..."
        cp jobPCM.slurm jobPCM${iPCM}.slurm
        sed -i "s/#SBATCH --job-name=jobPCM.*/#SBATCH --job-name=jobPCM${iPCM}/" jobPCM${iPCM}.slurm
        sed -i "s/^k=[0-9]\+$/k=$(echo "3 - $nPCM_ini" | bc -l)/" jobPCM${iPCM}.slurm
        jobID=$(sbatch --parsable jobPCM${iPCM}.slurm)
        # Create a file to cancel the dependent jobs of the cycle
        echo "#!/bin/bash" > kill_launchPEM.sh
        chmod +x kill_launchPEM.sh
        echo "scancel" $jobID >> kill_launchPEM.sh
        ((iPCM++))
        ((i_myear++))
        ((ii++))
    else
        endlaunch
    fi
    for ((i = $ii; i <= $1; i++)); do
        if [ $i_myear -lt $n_myear ]; then
            echo "Run PCM $iPCM: call $i/$1..."
            cp jobPCM.slurm jobPCM${iPCM}.slurm
            sed -i "s/#SBATCH --job-name=jobPCM.*/#SBATCH --job-name=jobPCM${iPCM}/" jobPCM${iPCM}.slurm
            sed -i "s/^k=[0-9]\+$/k=$(echo "$i + 2 - $nPCM_ini" | bc -l)/" jobPCM${iPCM}.slurm
            jobID=$(sbatch --parsable --dependency=afterok:${jobID} jobPCM${iPCM}.slurm)
            echo "scancel" $jobID >> kill_launchPEM.sh
            ((iPCM++))
            ((i_myear++))
        else
            endlaunch
        fi
    done
}

# To submit the PEM run
submitPEM() {
    if [ $i_myear -lt $n_myear ]; then
        echo "Run PEM $iPEM"
        sed -i "s/#SBATCH --job-name=jobPEM.*/#SBATCH --job-name=jobPEM${iPEM}/" jobPEM.slurm
        jobID=$(sbatch --parsable jobPEM.slurm)
        # Create a file to cancel the dependent jobs of the cycle
        echo "#!/bin/bash" > kill_launchPEM.sh
        chmod +x kill_launchPEM.sh
        echo "scancel" $jobID >> kill_launchPEM.sh
    else
        endlaunch
    fi
}

# To make one cycle of PCM and PEM runs
cyclelaunch() {
    # PCM runs
    submitPCM $1 $2

    # PEM run
    if [ $i_myear -lt $n_myear ]; then
        echo "Run PEM $iPEM"
        sed -i "s/#SBATCH --job-name=jobPEM.*/#SBATCH --job-name=jobPEM${iPEM}/" jobPEM.slurm
        jobID=$(sbatch --parsable --dependency=afterok:${jobID} jobPEM.slurm)
        echo "scancel" $jobID >> kill_launchPEM.sh
    else
        endlaunch
    fi
}

# To relaunch from PCM run
relaunchPCM() {
    iPCM=$(($irelaunch + 1))
    cp starts/restartfi${irelaunch}.nc startfi.nc
    if [ -f "starts/restart${irelaunch}.nc" ]; then
        cp starts/restart${irelaunch}.nc start.nc
    elif [ -f "starts/restart1D${irelaunch}.txt" ]; then
        cp starts/restart1D${irelaunch}.txt start1D.txt
    fi
    if [ $irelaunch -le $nPCM_ini ]; then
        # PCM relaunch during the initialization cycle
        iPEM=1
        i_myear=$irelaunch
        sed -i "1s/.*/$i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini/" info_PEM.txt
        if [ $irelaunch -eq $(($nPCM_ini - 1)) ]; then
            cp diags/data2reshape${irelaunch}.nc data2reshape_Y1.nc
            cyclelaunch $nPCM_ini $irelaunch
        elif [ $irelaunch -eq $nPCM_ini ]; then
            cp diags/data2reshape$(($irelaunch - 1)).nc data2reshape_Y1.nc
            cp diags/data2reshape${irelaunch}.nc data2reshape_Y2.nc
            submitPEM # The next job is a PEM run
        else
            cyclelaunch $nPCM_ini $iPCM
        fi
    else
        # PCM relaunch during a cycle
        iPEM=$(($irelaunch/$nPCM + 1))
        il=$(echo "($irelaunch - $nPCM_ini)%$nPCM" | bc -l)
        i_myear=$(($(awk "NR==$(($iPEM + 1)) {print \$1}" "info_PEM.txt") + $il))
        sed -i "1s/.*/$i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini/" info_PEM.txt
        cp starts/restartpem${iPEM}.nc startpem.nc
        if [ $il -eq $(($nPCM - 1)) ]; then
            cp diags/data2reshape${irelaunch}.nc data2reshape_Y1.nc
            cyclelaunch $nPCM $il
        elif [ $il -eq $nPCM ]; then
            cp diags/data2reshape$(($irelaunch - 1)).nc data2reshape_Y1.nc
            cp diags/data2reshape${irelaunch}.nc data2reshape_Y2.nc
            submitPEM # The next job is a PEM run
        else
            cyclelaunch $nPCM $il
        fi
    fi
}

# To relaunch from PEM run
relaunchPEM() {
    iPEM=$irelaunch
    iPCM=$(($nPCM_ini + ($nPCM - 1)*$irelaunch + 1))
    i_myear=$(awk "NR==$(($iPEM + 1)) {print \$1}" "info_PEM.txt")
    sed -i "1s/.*/$i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini/" info_PEM.txt
    cp starts/restartpem${irelaunch}.nc startpem.nc
    cp starts/restartfi_postPEM${irelaunch}.nc startfi.nc
    if [ -f "starts/restart_postPEM${irelaunch}.nc" ]; then
        cp starts/restart_postPEM${irelaunch}.nc start.nc
    elif [ -f "starts/restart1D_postPEM${irelaunch}.txt" ]; then
        cp starts/restart1D_postPEM${irelaunch}.txt start1D.txt
    fi
    cyclelaunch $nPCM
}