source: trunk/LMDZ.COMMON/libf/evolution/deftank/lib_launchPEM.sh @ 3458

Last change on this file since 3458 was 3446, checked in by jbclement, 15 months ago

PEM:
Correction of the launching script for a relaunch situation.
JBC

  • Property svn:executable set to *
File size: 13.7 KB
RevLine 
[3349]1#!/bin/bash
2########################################################################
3######## Library of bash functions for the PEM launching script ########
4########################################################################
5
6# To end the launching script
7endlaunch() {
8    # Restore the previous value of LC_NUMERIC
9    LC_NUMERIC=$OLD_LC_NUMERIC
10
11    date
12    echo "Successful end of the launching script for the PEM simulation."
13    exit 0
14}
15
16# To end the launching script with error
17errlaunch() {
18    # Restore the previous value of LC_NUMERIC
19    LC_NUMERIC=$OLD_LC_NUMERIC
20
21    date
22    echo "End with error of the launching script for the PEM."
23    exit 1
24}
25
[3403]26# To check what is the job scheduler
27function job_scheduler() {
28    if command -v squeue &> /dev/null; then
29        echo "SLURM is installed on $machine."
30        name_job="#SBATCH --job-name="
31        kill_job="scancel"
32        submit_job="sbatch --parsable"
33        submit_dependjob="sbatch --parsable --dependency"
34        sed -i 's/\$PBS_JOBID/\$SLURM_JOB_ID/g' PEMrun.job
35    elif command -v qstat &> /dev/null; then
36        echo "PBS/TORQUE is installed on $machine."
37        name_job="#PBS -N "
38        kill_job="qdel"
39        submit_job="qsub"
40        submit_dependjob="qsub -W depend"
41        sed -i 's/\$SLURM_JOB_ID/\$PBS_JOBID/g' PEMrun.job
42    else
43        echo "Error: neither SLURM nor TORQUE/PBS is installed on $machine!"
44        echo "You need to adapt the script to your job scheduler."
[3349]45        errlaunch
46    fi
47}
48
49# To check if everything necessary for the launching script is ok
50checklaunch() {
51    # Save the current value of LC_NUMERIC and set it to a locale that uses a dot as the decimal separator
52    OLD_LC_NUMERIC=$LC_NUMERIC
53    LC_NUMERIC=en_US.UTF-8
54
55    if [ -v n_mars_years ] && [ ! -z "$n_mars_years" ]; then
56        if [ $n_mars_years -lt 1 ]; then
57            echo "Error: the value of 'n_mars_years' must be >0!"
58            errlaunch
59        fi
60    elif [ -v n_earth_years ] && [ ! -z "$n_earth_years" ]; then
61        if [ $n_earth_years -lt 1 ]; then
62            echo "Error: the value of 'n_earth_years' must be >0!"
63            errlaunch
64        fi
65    else
66        echo "Error: no number of years to be simulated has been set!"
67        errlaunch
68    fi
69    if [ $nPCM_ini -lt 2 ] || [ -z "$nPCM_ini" ]; then
70        echo "Error: the value of 'nPCM_ini' must be >1!"
71        errlaunch
72    fi
73    if [ $nPCM -lt 2 ] || [ -z "$nPCM" ]; then
74        echo "Error: the value of 'nPCM' must be >1!"
75        errlaunch
76    fi
[3391]77    if [ ! -f "PCMrun.job" ]; then
78        echo "Error: file \"PCMrun.job\" does not exist in $dir!"
[3349]79        errlaunch
80    fi
[3391]81    if [ ! -f "PEMrun.job" ]; then
82        echo "Error: file \"PEMrun.job\" does not exist in $dir!"
[3349]83        errlaunch
84    fi
85    if [ ! -f "run_PCM.def" ]; then
86        echo "Error: file \"run_PCM.def\" does not exist in $dir!"
87        errlaunch
88    fi
89    if [ ! -f "run_PEM.def" ]; then
90        echo "Error: file \"run_PEM.def\" does not exist in $dir!"
91        errlaunch
92    fi
93    if [ ! -f "context_lmdz_physics.xml" ]; then
94        echo "Error: file \"context_lmdz_physics.xml\" does not exist in $dir!"
95        errlaunch
96    fi
97    if [ ! -f "field_def_physics_mars.xml" ]; then
98        echo "Error: file \"field_def_physics_mars.xml\" does not exist in $dir!"
99        errlaunch
100    fi
101    if [ ! -f "file_def_physics_mars.xml" ]; then
102        echo "Error: file \"file_def_physics_mars.xml\" does not exist in $dir!"
103        errlaunch
104    fi
105    if [ ! -f "iodef.xml" ]; then
106        echo "Error: file \"iodef.xml\" does not exist in $dir!"
107        errlaunch
108    fi
109    if [ ! -d "out_PCM" ]; then
110        mkdir out_PCM
111    fi
112    if [ ! -d "out_PEM" ]; then
113        mkdir out_PEM
114    fi
115    if [ ! -d "starts" ]; then
116        mkdir starts
117    fi
118    if [ ! -d "diags" ]; then
119        mkdir diags
120    fi
[3403]121    if [ $dim -ne 1 ]; then
122        job_scheduler
123    fi
[3349]124}
125
[3386]126# To convert Earth years into Mars years
127convertyears() {
[3349]128    myear=686.9725      # Number of Earth days in Martian year
129    eyear=365.256363004 # Number of days in Earth year
130    convert_years=$(echo "$myear/$eyear" | bc -l)
131    convert_years=$(printf "%.4f" $convert_years) # Rounding to the 4th decimal to respect the precision of Martian year
132    if [ -v n_mars_years ]; then
133        n_myear=$n_mars_years
134        echo "Number of years to be simulated: $n_myear Martian years."
135    elif [ -v n_earth_years ]; then
136        n_myear=$(echo "($n_earth_years/$convert_years + 0.999999)/1" | bc) # Ceiling of n_earth_years/convert_years
137        echo "Number of years to be simulated: $n_earth_years Earth years = $n_myear Martian years."
138    fi
[3386]139}
140
141# To initialize the launching script
142initlaunch() {
143    echo "This is a chained simulation for PEM and PCM runs in $dir on $machine by $user."
144    convertyears
[3349]145    i_myear=0
146    iPEM=1
[3355]147    iPCM=1
[3349]148    cp startfi.nc starts/
149    if [ -f "start.nc" ]; then
150        cp start.nc starts/
151    elif [ -f "star1D.nc" ]; then
152        cp star1D.txt starts/
153    fi
[3428]154    if [ -f "startpem.nc" ]; then
155        cp startpem.nc starts/
156    fi
[3349]157
158    # Create a file to manage years of the chained simulation and store some info from the PEM runs
159    echo $i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini > info_PEM.txt
160}
161
162# To submit the PCM runs
[3391]163# arg1: model dimension
164# arg2: number of PCM runs to launch
165# arg3: local number of the PCM run from which to start (optional)
[3349]166submitPCM() {
[3391]167    find . -type f -name "PCMrun*.job" ! -name "PCMrun.job" -delete
[3355]168    ii=1
[3391]169    if [ ! -z $3 ]; then
170        ii=$3
[3355]171    fi
[3349]172    if [ $i_myear -lt $n_myear ]; then
[3391]173        echo "Run PCM $iPCM: call $ii/$2..."
174        if [ $1 -eq 1 ]; then # 1D model
[3432]175            sed -i "s/^k=[0-9]\+$/k=$(echo "3 - $nPCM_ini" | bc)/" PCMrun.job
[3391]176            ./PCMrun.job
[3430]177            if [ $? -ne 0 ]; then
178                errlaunch
179            fi
[3391]180        else # 3D model
181            cp PCMrun.job PCMrun${iPCM}.job
[3403]182            sed -i -E "s/($name_job[^0-9]*[0-9]*[^0-9]*)[0-9]+$/\1${iPCM}/" PCMrun${iPCM}.job
[3432]183            sed -i "s/^k=[0-9]\+$/k=$(echo "3 - $nPCM_ini" | bc)/" PCMrun${iPCM}.job
[3403]184            jobID=$(eval "$submit_job PCMrun${iPCM}.job")
[3391]185            # Create a file to cancel the dependent jobs of the cycle
186            echo "#!/bin/bash" > kill_launchPEM.sh
187            chmod +x kill_launchPEM.sh
[3403]188            echo $kill_job $jobID >> kill_launchPEM.sh
[3391]189        fi
[3349]190        ((iPCM++))
191        ((i_myear++))
[3355]192        ((ii++))
[3349]193    else
194        endlaunch
195    fi
[3391]196    for ((i = $ii; i <= $2; i++)); do
[3349]197        if [ $i_myear -lt $n_myear ]; then
[3391]198            echo "Run PCM $iPCM: call $i/$2..."
199            if [ $1 -eq 1 ]; then # 1D model
[3432]200                sed -i "s/^k=[0-9]\+$/k=$(echo "$i + 2 - $nPCM_ini" | bc)/" PCMrun.job
[3391]201                ./PCMrun.job
[3430]202                if [ $? -ne 0 ]; then
203                    errlaunch
204                fi
[3391]205            else # 3D model
206                cp PCMrun.job PCMrun${iPCM}.job
[3403]207                sed -i -E "s/($name_job[^0-9]*[0-9]*[^0-9]*)[0-9]+$/\1${iPCM}/" PCMrun${iPCM}.job
[3432]208                sed -i "s/^k=[0-9]\+$/k=$(echo "$i + 2 - $nPCM_ini" | bc)/" PCMrun${iPCM}.job
[3403]209                jobID=$(eval "$submit_dependjob=afterok:${jobID} PCMrun${iPCM}.job")
210                echo $kill_job $jobID >> kill_launchPEM.sh
[3391]211            fi
[3349]212            ((iPCM++))
213            ((i_myear++))
214        else
215            endlaunch
216        fi
217    done
218}
219
220# To submit the PEM run
[3391]221# arg1: model dimension
[3349]222submitPEM() {
223    if [ $i_myear -lt $n_myear ]; then
[3355]224        echo "Run PEM $iPEM"
[3391]225        if [ $1 -eq 1 ]; then # 1D model
226            ./PEMrun.job
[3430]227            if [ $? -ne 0 ]; then
228                errlaunch
229            fi
[3391]230        else # 3D model
[3403]231            sed -i -E "s/($name_job[^0-9]*[0-9]*[^0-9]*)[0-9]+$/\1${iPEM}/" PEMrun.job
232            jobID=$(eval "$submit_job PEMrun.job")
[3391]233            # Create a file to cancel the dependent jobs of the cycle
234            echo "#!/bin/bash" > kill_launchPEM.sh
235            chmod +x kill_launchPEM.sh
[3403]236            echo $kill_job $jobID >> kill_launchPEM.sh
[3391]237        fi
[3349]238    else
239        endlaunch
240    fi
241}
242
243# To make one cycle of PCM and PEM runs
[3391]244# arg1: model dimension
245# arg2: number of PCM runs to launch
246# arg3: local number of the PCM run from which to start (optional)
[3349]247cyclelaunch() {
248    # PCM runs
[3391]249    submitPCM $1 $2 $3
[3349]250
251    # PEM run
[3355]252    if [ $i_myear -lt $n_myear ]; then
253        echo "Run PEM $iPEM"
[3391]254        if [ $1 -eq 1 ]; then # 1D model
255            ./PEMrun.job
[3430]256            if [ $? -ne 0 ]; then
257                errlaunch
258            fi
[3391]259        else # 3D model
[3403]260            sed -i -E "s/($name_job[^0-9]*[0-9]*[^0-9]*)[0-9]+$/\1${iPEM}/" PEMrun.job
261            jobID=$(eval "$submit_dependjob=afterok:${jobID} PEMrun.job")
262            echo $kill_job $jobID >> kill_launchPEM.sh
[3391]263        fi
[3355]264    else
265        endlaunch
266    fi
[3349]267}
[3355]268
[3365]269# To clean files after the starting run of the relaunch
[3386]270# arg1: file name prefix to clean
271# arg2: file name extension to clean
272# arg3: file number from which to clean
[3365]273cleanfiles() {
274    prefix=$1
275    extension=$2
276    if [ -z "$extension" ]; then
277        for file in ${prefix}*; do
278            num=${file#$prefix}
279            if [[ $num =~ ^[0-9]+$ ]] && [ $num -gt $3 ]; then
280                rm $file
281            fi
282        done
283    else
284        for file in ${prefix}*${extension}; do
285            num=${file#$prefix}
286            num=${num%$extension}
287            if [[ $num =~ ^[0-9]+$ ]] && [ $num -gt $3 ]; then
288                rm $file
289            fi
290        done
291    fi
292}
293
[3355]294# To relaunch from PCM run
[3391]295# arg1: model dimension
[3355]296relaunchPCM() {
297    iPCM=$(($irelaunch + 1))
[3365]298    cleanfiles diags/diagfi .nc $irelaunch
[3386]299    cleanfiles diags/data2reshape .nc $irelaunch
[3365]300    cleanfiles "out_PCM/run" "" $irelaunch
301    cleanfiles starts/restart1D .txt $irelaunch
302    cleanfiles starts/restart .nc $irelaunch
303    cleanfiles starts/restartfi .nc $irelaunch
[3355]304    cp starts/restartfi${irelaunch}.nc startfi.nc
305    if [ -f "starts/restart${irelaunch}.nc" ]; then
306        cp starts/restart${irelaunch}.nc start.nc
307    elif [ -f "starts/restart1D${irelaunch}.txt" ]; then
308        cp starts/restart1D${irelaunch}.txt start1D.txt
309    fi
310    if [ $irelaunch -le $nPCM_ini ]; then
311        # PCM relaunch during the initialization cycle
312        iPEM=1
[3365]313        cleanfiles diags/diagpem .nc $iPEM
314        cleanfiles "out_PEM/run" "" $iPEM
315        cleanfiles starts/restart1D_postPEM .txt $iPEM
316        cleanfiles starts/restart_postPEM .nc $iPEM
317        cleanfiles starts/restartfi_postPEM .nc $iPEM
318        cleanfiles starts/restartpem .nc $iPEM
[3355]319        i_myear=$irelaunch
320        sed -i "1s/.*/$i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini/" info_PEM.txt
[3367]321        rm -f startpem.nc
[3355]322        if [ $irelaunch -eq $(($nPCM_ini - 1)) ]; then
323            cp diags/data2reshape${irelaunch}.nc data2reshape_Y1.nc
[3391]324            cyclelaunch $1 $nPCM_ini $iPCM
[3383]325        elif [ $irelaunch -eq $nPCM_ini ]; then
[3355]326            cp diags/data2reshape$(($irelaunch - 1)).nc data2reshape_Y1.nc
327            cp diags/data2reshape${irelaunch}.nc data2reshape_Y2.nc
[3391]328            submitPEM $1 # The next job is a PEM run
[3355]329        else
[3391]330            cyclelaunch $1 $nPCM_ini $iPCM
[3355]331        fi
332    else
333        # PCM relaunch during a cycle
[3446]334        iPEM=$(echo "($iPCM - $nPCM_ini)/$nPCM + 1" | bc)
335        il=$(echo "($irelaunch - $nPCM_ini + 1)%$nPCM + 1" | bc)
[3365]336        cleanfiles diags/diagpem .nc $iPEM
337        cleanfiles "out_PEM/run" "" $iPEM
338        cleanfiles starts/restart1D_postPEM .txt $iPEM
339        cleanfiles starts/restart_postPEM .nc $iPEM
340        cleanfiles starts/restartfi_postPEM .nc $iPEM
341        cleanfiles starts/restartpem .nc $iPEM
[3355]342        cp starts/restartpem${iPEM}.nc startpem.nc
[3428]343        if [ $il -eq $(($nPCM - 1)) ]; then # Second to last PCM run
[3365]344            i_myear=$(($(awk "NR==$iPEM {print \$1}" "info_PEM.txt") + $il))
345            sed -i "1s/.*/$i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini/" info_PEM.txt
[3355]346            cp diags/data2reshape${irelaunch}.nc data2reshape_Y1.nc
[3391]347            cyclelaunch $1 $nPCM $il
[3428]348        elif [ $il -eq $nPCM ]; then # Last PCM run so the next job is a PEM run
[3365]349            i_myear=$(($(awk "NR==$iPEM {print \$1}" "info_PEM.txt") + $nPCM))
350            sed -i "1s/.*/$i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini/" info_PEM.txt
[3355]351            cp diags/data2reshape$(($irelaunch - 1)).nc data2reshape_Y1.nc
352            cp diags/data2reshape${irelaunch}.nc data2reshape_Y2.nc
[3428]353            submitPEM $1
[3355]354        else
[3365]355            i_myear=$(($(awk "NR==$iPEM {print \$1}" "info_PEM.txt") + $il))
356            sed -i "1s/.*/$i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini/" info_PEM.txt
[3391]357            cyclelaunch $1 $nPCM $il
[3355]358        fi
359    fi
360}
361
362# To relaunch from PEM run
[3391]363# arg1: model dimension
[3355]364relaunchPEM() {
[3432]365    iPEM=$(echo "$irelaunch + 1" | bc)
366    iPCM=$(echo "$nPCM_ini + $nPCM*($irelaunch - 1) + 1" | bc)
[3355]367    i_myear=$(awk "NR==$(($iPEM + 1)) {print \$1}" "info_PEM.txt")
368    sed -i "1s/.*/$i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini/" info_PEM.txt
[3365]369    cleanfiles diags/diagfi .nc $(($iPCM - 1))
370    cleanfiles "out_PCM/run" "" $(($iPCM - 1))
371    cleanfiles starts/restart1D .txt $(($iPCM - 1))
372    cleanfiles starts/restart .nc $(($iPCM - 1))
373    cleanfiles starts/restartfi .nc $(($iPCM - 1))
374    cleanfiles diags/data2reshape .nc $(($iPCM - 1))
375    cleanfiles diags/diagpem .nc $irelaunch
376    cleanfiles "out_PEM/run" "" $irelaunch
377    cleanfiles starts/restart1D_postPEM .txt $irelaunch
378    cleanfiles starts/restart_postPEM .nc $irelaunch
379    cleanfiles starts/restartfi_postPEM .nc $irelaunch
380    cleanfiles starts/restartpem .nc $irelaunch
[3355]381    cp starts/restartpem${irelaunch}.nc startpem.nc
382    cp starts/restartfi_postPEM${irelaunch}.nc startfi.nc
383    if [ -f "starts/restart_postPEM${irelaunch}.nc" ]; then
384        cp starts/restart_postPEM${irelaunch}.nc start.nc
385    elif [ -f "starts/restart1D_postPEM${irelaunch}.txt" ]; then
386        cp starts/restart1D_postPEM${irelaunch}.txt start1D.txt
387    fi
[3391]388    cyclelaunch $1 $nPCM
[3355]389}
Note: See TracBrowser for help on using the repository browser.