source: trunk/LMDZ.COMMON/libf/evolution/deftank/pem_workflow.sh @ 4091

#!/bin/bash
########################################################################
#### Workflow script for a chained simulation of PEM and PCM runs  #####
########################################################################
# This script can take an argument:
#     1) None: to start a simulation from scratch;
#     2) 're': to resume a simulation from a starting point (interactive prompt).
########################################################################
set -e
trap 'echo -e "\033[31mError: an issue occurred in the script on line $LINENO! Please review the command and try again.\033[0m"' ERR


########################################################################
# Modify here the parameters for the simulation
###############################################
# Set the number of years to be simulated, either Planetary or Earth years (> 0):
n_planetary_years=100.
#n_earth_years=300.

# Set the number of initial PCM years (>= 2):
n_pcm_runs_ini=3

# Set the number of PCM years between each PEM run (>= 2):
n_pcm_runs=2

# Set the execution mode (0 = "processing scripts"; any other values = "submitting jobs"). The former option is usually used to process the script on a local machine while the latter is used to submit jobs on a supercomputer with SLURM or PBS/TORQUE:
exec_mode=1
########################################################################


dir=`pwd`
machine=`hostname`
user=`whoami`
if [ ! -f "pem_workflow_lib.sh" ]; then
    echo "Error: file \"pem_workflow_lib.sh\" does not exist in $dir!"
    echo "It can be found in the PEM deftank."
    exit 1
fi

source pem_workflow_lib.sh
export exec_mode

if [ $# -eq 0 ]; then
    # Starting from scratch
    echo "The PEM workflow is starting!"
    echo "The output file is \"pem_workflow.log\"."
    exec > pem_workflow.log 2>&1
    echo "Beginning of log file for the PEM workflow."
    date
    check_workflow
    ini_workflow
    submit_cycle $exec_mode $n_pcm_runs_ini

else
    # Starting a new cycle
    if [ $1 = "new" ]; then
        exec >> pem_workflow.log 2>&1
        echo
        echo "This is a new cycle of the PEM worflow."
        date
        if [ $exec_mode -ne 0 ]; then
            detect_scheduler
            if command -v squeue &> /dev/null; then
                unset SLURM_MEM_PER_CPU SLURM_MEM_PER_GPU SLURM_MEM_PER_NODE
            fi
        fi
        read n_yr_sim ntot_yr_sim r_plnt2earth_yr i_pcm_run i_pem_run n_pcm_runs n_pcm_runs_ini < pem_workflow.sts
        submit_cycle $exec_mode $n_pcm_runs

    # Starting a resumption of the workflow
    elif [ $1 = "re" ]; then
        if [ ! -f "pem_workflow.sts" ]; then
            echo "Error: file \"pem_workflow.sts\" does not exist in $dir!"
            echo "It is necessary to resume the PEM workflow."
            abort_workflow
        fi
        echo "The resumption is initialized with a previous successful run to be specified."
        while true; do
            echo "Do you want to resume from a 'PCM' or 'PEM' run?"
            read run_resume
            if [ $run_resume = "PCM" ] || [ $run_resume = "PEM" ]; then
                break
            else
                echo "Invalid input. Please enter 'PCM' or 'PEM'."
            fi
        done
        read n_yr_sim n_year_old r_plnt2earth_yr i_pcm_run i_pem_run nPCM_old nPCM_ini_old < pem_workflow.sts
        while true; do
            if [ $run_resume = "PCM" ]; then
                echo "What is the number of the PCM run?"
                echo "It should be between 1 and $(( $((i_pcm_run - 1)) > 1 ? $((i_pcm_run - 1)) : 1 ))."
                read i_resume
                if [ 1 -le $i_resume ] && [ $i_resume -le $(( $((i_pcm_run - 1)) > 1 ? $((i_pcm_run - 1)) : 1 )) ]; then
                    break
                else
                    echo "Invalid input. Please enter a valid PCM run number."
                fi
            else
                echo "What is the number of the PEM run?"
                echo "It should be between 1 and $(( $((i_pem_run - 1)) > 1 ? $((i_pem_run - 1)) : 1 ))."
                read i_resume
                if [ 1 -le $i_resume ] && [ $i_resume -le $(( $((i_pem_run - 1)) > 1 ? $((i_pem_run - 1)) : 1 )) ]; then
                    break
                else
                    echo "Invalid input. Please enter a valid PEM run number."
                fi
            fi
        done
        exec >> pem_workflow.log 2>&1
        echo
        echo "This is a resumption for the PEM workflow from the run \"$run_resume $i_resume\"."
        date
        check_workflow
        convert_earth2plnt_years
        if [ $n_pcm_runs_ini -ne $nPCM_ini_old ]; then
            echo "The number of initial PCM years has been modified from $nPCM_ini_old to $n_pcm_runs_ini."
        fi
        if [ $n_pcm_runs -ne $nPCM_old ]; then
            echo "The number of PCM years between each PEM run has been modified from $nPCM_old to $n_pcm_runs."
        fi
        if [ "$(echo "$ntot_yr_sim != $n_year_old" | bc)" -eq 1 ]; then
            echo "The number of initial PCM years has been modified from $n_year_old to $ntot_yr_sim."
        fi
        sed -i "1s/.*/$n_yr_sim $ntot_yr_sim $r_plnt2earth_yr $i_pcm_run $i_pem_run $n_pcm_runs $n_pcm_runs_ini/" pem_workflow.sts
        if [ -f "kill_pem_workflow.sh" ]; then
            ./kill_pem_workflow.sh
        fi
        if [ $run_resume = "PCM" ]; then
            resume_from_pcm_run $exec_mode
        else
            resume_from_pem_run $exec_mode
        fi

    # Continuing a PEM run
    # CANNOT BE DONE FOR NOW BECAUSE THE PEM DOES NOT SAVE ITS STATE TO BE ABLE TO RECOVER
    #elif [ $1 = "cont" ]; then
    #    exec >> pem_workflow.log 2>&1
    #    echo
    #    echo "This is a continuation of the previous PEM run."
    #    date
    #    submit_pem_phase $exec_mode

    # Default case: error
    else
        echo "Error: given argument '$1' for the PEM workflow script is unknown!"
        abort_workflow
    fi
fi