source: trunk/LMDZ.COMMON/libf/evolution/deftank/lib_launchPEM.sh @ 3961

Last change on this file since 3961 was 3958, checked in by jbclement, 11 days ago

PEM:
Checking if a PCM run length is one Martian year + Changing instruction from "run" to "year" to be clearer.
JBC

  • Property svn:executable set to *
File size: 19.3 KB
Line 
1#!/bin/bash
2########################################################################
3######## Library of bash functions for the PEM launching script ########
4########################################################################
5
6# To end the launching script
7endlaunch() {
8    # Restore the previous value of LC_NUMERIC
9    LC_NUMERIC=$OLD_LC_NUMERIC
10
11    date
12    echo "Success: the launching script for the PEM simulation completed normally!"
13    exit 0
14}
15
16# To end the launching script with error
17errlaunch() {
18    # Restore the previous value of LC_NUMERIC
19    LC_NUMERIC=$OLD_LC_NUMERIC
20
21    date
22    echo "Error: an issue occured in the launching script for the PEM simulation!"
23    exit 1
24}
25
26# To check what is the job scheduler
27function job_scheduler() {
28    if command -v squeue &> /dev/null; then
29        echo "SLURM is installed on $machine."
30        scheduler="SLURM"
31        name_job="#SBATCH --job-name="
32        kill_job="scancel"
33    elif command -v qstat &> /dev/null; then
34        echo "PBS/TORQUE is installed on $machine."
35        scheduler="PBS"
36        name_job="#PBS -N "
37        kill_job="qdel"
38    else
39        echo "Error: neither SLURM nor TORQUE/PBS is installed on $machine!"
40        echo "You need to adapt the script to your job scheduler or set 'mode' to 0."
41        errlaunch
42    fi
43}
44
45# To get the number of slopes for the simulation
46get_nslope() {
47    ns=1
48    if [ -f "startfi.nc" ]; then
49        ns=$(ncdump -h startfi.nc | sed -n 's/.*nslope = \([0-9]*\) ;.*/\1/p')
50    else
51        for f in run_PCM.def callphys.def; do
52            if [[ -f "$f" ]]; then
53                while IFS= read -r line; do
54                    # Remove leading whitespace
55                    trimmed=$(echo "$line" | sed 's/^[[:space:]]*//')
56                    # Skip lines that are commented out
57                    if [[ "$trimmed" == \#* ]]; then
58                        continue
59                    fi
60                    # Check if line contains 'nslope = N'
61                    if [[ "$trimmed" =~ ^nslope[[:space:]]*=[[:space:]]*([0-9]+) ]]; then
62                        ns="${BASH_REMATCH[1]}"
63                        break
64                    fi
65                done < "$f"
66                [[ -n "$ns" ]] && break
67            fi
68        done
69    fi
70}
71
72# To modify the xml file according nslope
73modify_xml() {
74    tmp="tmp_file_def.xml"
75    in_diurnalave=false
76    in_diurnalave_s=false
77
78    sed -i 's/enabled="\.true\.\">/enabled=".false.">/g' file_def_physics_mars.xml
79
80    while IFS= read -r line; do
81        case "$line" in
82            *'<file id="diurnalave"'*)
83                in_diurnalave=true
84                ;;
85            *'<file id="diurnalave_s"'*)
86                in_diurnalave_s=true
87                ;;
88        esac
89
90        # Handle enabled attribute
91        if [[ $line == *'enabled="'* ]]; then
92            if $in_diurnalave; then
93                if [[ $ns -eq 1 ]]; then
94                    line='              enabled=".true.">'
95                else
96                    line='              enabled=".false.">'
97                fi
98            elif $in_diurnalave_s; then
99                if [[ $ns -eq 1 ]]; then
100                    line='              enabled=".false.">'
101                else
102                    line='              enabled=".true.">'
103                fi
104            fi
105        fi
106
107        # Detect slope variables
108        if $in_diurnalave_s && [[ $line =~ slope([0-9]+) ]]; then
109            slope_id="${BASH_REMATCH[1]}"
110            if (( 10#$slope_id > ns )); then
111                # Ensure the line is commented
112                if [[ $line != "<!--"* ]]; then
113                    line="<!-- $line -->"
114                fi
115            else
116                # Ensure the line is uncommented
117                if [[ $line == "<!--"* ]]; then
118                    line="${line#<!-- }" # remove leading <!--
119                    line="${line% -->}"  # remove trailing -->
120                fi
121            fi
122        fi
123
124        case "$line" in
125            *'</file>'*)
126                in_diurnalave=false
127                in_diurnalave_s=false
128                ;;
129        esac
130
131        echo "$line" >> "$tmp"
132    done < file_def_physics_mars.xml
133
134    mv "$tmp" file_def_physics_mars.xml
135}
136
137# To check if a PCM run is one year
138check_runyear() {
139    if [ -f "run_PCM.def" ]; then
140        year_sol=$(ncdump -v controle startfi.nc 2>/dev/null | \
141                   sed -n '/controle =/,/;/p' | tr -d '[:space:]' | \
142                   sed 's/.*=//; s/;//' | tr ',' '\n' | sed -n '14p')
143    else
144        year_sol=669 # Length of Martian year (sols)
145    fi
146    sol_in_file=$(awk -F'=' '/^[[:space:]]*(nday|ndt)[[:space:]]*=/ {
147                  val=$2
148                  gsub(/^[[:space:]]+|[[:space:]]+$/,"",val)
149                  print val
150                  exit
151                  }' run_PCM.def)
152
153    if [ -z "$sol_in_file" ]; then
154        echo "Error: no length of year found in \"run_PCM.def\"!"
155        errlaunch
156    elif [ "$sol_in_file" -eq "$year_sol" ]; then
157        # Good: we do nothing
158        :
159    else
160        echo "Error: length of year mismatch between \"run_PCM.def\" ($sol_in_file) and \"startfi.nc\" ($year_sol)!"
161        errlaunch
162    fi
163}
164
165# To check if everything necessary for the launching script is ok
166checklaunch() {
167    # Save the current value of LC_NUMERIC and set it to a locale that uses a dot as the decimal separator
168    OLD_LC_NUMERIC=$LC_NUMERIC
169    LC_NUMERIC=en_US.UTF-8
170
171    if [ -v n_mars_years ] && [ ! -z "$n_mars_years" ]; then
172        if [ $(echo "$n_mars_years <= 0." | bc -l) -eq 1 ]; then
173            echo "Error: 'n_mars_years' must be > 0!"
174            errlaunch
175        fi
176    elif [ -v n_earth_years ] && [ ! -z "$n_earth_years" ]; then
177        if [ $(echo "$n_earth_years <= 0." | bc -l) -eq 1 ]; then
178            echo "Error: 'n_earth_years' must be > 0!"
179            errlaunch
180        fi
181    else
182        echo "Error: the number of years to be simulated is not set!"
183        errlaunch
184    fi
185    if [ $nPCM_ini -lt 2 ] || [ -z "$nPCM_ini" ]; then
186        echo "Error: 'nPCM_ini' must be >= 2!"
187        errlaunch
188    fi
189    if [ $nPCM -lt 2 ] || [ -z "$nPCM" ]; then
190        echo "Error: 'nPCM' must be >= 2!"
191        errlaunch
192    fi
193    if [ ! -f "PCMrun.job" ]; then
194        echo "Error: file \"PCMrun.job\" does not exist in $dir!"
195        errlaunch
196    fi
197    if [ ! -f "PEMrun.job" ]; then
198        echo "Error: file \"PEMrun.job\" does not exist in $dir!"
199        errlaunch
200    fi
201    if [ ! -f "run_PCM.def" ]; then
202        echo "Error: file \"run_PCM.def\" does not exist in $dir!"
203        errlaunch
204    fi
205    if [ ! -f "run_PEM.def" ]; then
206        echo "Error: file \"run_PEM.def\" does not exist in $dir!"
207        errlaunch
208    fi
209    if [ ! -f "context_lmdz_physics.xml" ]; then
210        echo "Error: file \"context_lmdz_physics.xml\" does not exist in $dir!"
211        errlaunch
212    fi
213    if [ ! -f "field_def_physics_mars.xml" ]; then
214        echo "Error: file \"field_def_physics_mars.xml\" does not exist in $dir!"
215        errlaunch
216    fi
217    if [ ! -f "file_def_physics_mars.xml" ]; then
218        echo "Error: file \"file_def_physics_mars.xml\" does not exist in $dir!"
219        errlaunch
220    fi
221    if [ ! -f "iodef.xml" ]; then
222        echo "Error: file \"iodef.xml\" does not exist in $dir!"
223        errlaunch
224    fi
225    if [ ! -d "logs" ]; then
226        mkdir logs
227    fi
228    if [ ! -d "starts" ]; then
229        mkdir starts
230    fi
231    if [ ! -d "diags" ]; then
232        mkdir diags
233    fi
234    if [ $mode -ne 0 ]; then
235        job_scheduler
236    fi
237    # Set automatically the XIOS output file for the PEM according to the number of slopes
238    get_nslope
239    modify_xml
240    # Check if a PCM run is one year
241    check_runyear
242}
243
244# To convert Earth years into Mars years
245convertyears() {
246    myear=686.9725      # Number of Earth days in Martian year
247    eyear=365.256363004 # Number of days in Earth year
248    convert_years=$(echo "$myear/$eyear" | bc -l)
249    convert_years=$(printf "%.4f" $convert_years) # Rounding to the 4th decimal to respect the precision of Martian year
250    if [ -v n_mars_years ]; then
251        n_myear=$n_mars_years
252        echo "Number of years to be simulated: $n_myear Martian years."
253    elif [ -v n_earth_years ]; then
254        n_myear=$(echo "$n_earth_years/$convert_years" | bc -l)
255        echo "Number of years to be simulated: $n_earth_years Earth years = $n_myear Martian years."
256    fi
257}
258
259# To initialize the launching script
260initlaunch() {
261    echo "This is a chained simulation for PEM and PCM runs in $dir on $machine by $user."
262    convertyears
263    i_myear=0.
264    iPEM=1
265    iPCM=1
266    if [ -f "startfi.nc" ]; then
267        cp startfi.nc starts/
268    fi
269    if [ -f "start.nc" ]; then
270        cp start.nc starts/
271    elif [ -f "start1D.txt" ]; then
272        cp start1D.txt starts/
273    fi
274    if [ -f "startpem.nc" ]; then
275        cp startpem.nc starts/
276    fi
277
278    # Create a file to manage years of the chained simulation and store some info from the PEM runs
279    echo $i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini > info_PEM.txt
280}
281
282# To submit the PCM runs
283# arg1: launching mode
284# arg2: counting method
285# arg3: number of PCM runs to launch
286# arg4: local number of the PCM run from which to start (optional)
287submitPCM() {
288    find . -type f -name "PCMrun*.job" ! -name "PCMrun.job" -delete
289    ii=1
290    if [ ! -z $4 ]; then
291        ii=$4
292    fi
293    if [ $(echo "$i_myear < $n_myear" | bc -l) -eq 1 ]; then
294        echo "Run \"PCM $iPCM\" ($ii/$3)"
295        if [ $1 -eq 0 ]; then # Mode: processing scripts
296            sed -i "s/^k=-\?[0-9]\+$/k=$(echo "$ii - $3 + 2" | bc)/" PCMrun.job
297            ./PCMrun.job
298            if [ $? -ne 0 ]; then
299                errlaunch
300            fi
301        else # Mode: submitting jobs
302            cp PCMrun.job PCMrun${iPCM}.job
303            sed -i -E "/^$name_job/s/(.*[^0-9])([0-9]+)(_[^0-9]*)?$/\1${iPCM}\3/" PCMrun${iPCM}.job
304            sed -i "s/^k=-\?[0-9]\+$/k=$(echo "$ii - $3 + 2" | bc)/" PCMrun${iPCM}.job
305            if [[ "$scheduler" == "SLURM" ]]; then
306                jobID=$(sbatch --parsable PCMrun${iPCM}.job)
307            elif [[ "$scheduler" == "PBS" ]]; then
308                jobID=$(qsub PCMrun${iPCM}.job | cut -d. -f1)
309            fi
310            # Create a file to cancel the dependent jobs of the cycle
311            echo "#!/bin/bash" > kill_launchPEM.sh
312            chmod +x kill_launchPEM.sh
313            echo $kill_job $jobID >> kill_launchPEM.sh
314        fi
315        ((iPCM++))
316        if [ $2 -ne 0 ]; then # Counting: PCM years taken into account
317            i_myear=$(echo "$i_myear + 1." | bc -l)
318        fi
319        ((ii++))
320    else
321        endlaunch
322    fi
323    for ((i = $ii; i <= $3; i++)); do
324        if [ $(echo "$i_myear < $n_myear" | bc -l) -eq 1 ]; then
325            echo "Run \"PCM $iPCM\" ($i/$3)"
326            if [ $1 -eq 0 ]; then # Mode: processing scripts
327                sed -i "s/^k=-\?[0-9]\+$/k=$(echo "$i - $3 + 2" | bc)/" PCMrun.job
328                ./PCMrun.job
329                if [ $? -ne 0 ]; then
330                    errlaunch
331                fi
332            else # Mode: submitting jobs
333                cp PCMrun.job PCMrun${iPCM}.job
334                sed -i -E "/^$name_job/s/(.*[^0-9])([0-9]+)(_[^0-9]*)?$/\1${iPCM}\3/" PCMrun${iPCM}.job
335                sed -i "s/^k=-\?[0-9]\+$/k=$(echo "$i - $3 + 2" | bc)/" PCMrun${iPCM}.job
336                if [[ "$scheduler" == "SLURM" ]]; then
337                    jobID=$(sbatch --parsable --dependency=afterok:${jobID} PCMrun${iPCM}.job)
338                elif [[ "$scheduler" == "PBS" ]]; then
339                    jobID=$(qsub -W depend=afterok:${jobID} PCMrun${iPCM}.job | cut -d. -f1)
340                fi
341                echo $kill_job $jobID >> kill_launchPEM.sh
342            fi
343            ((iPCM++))
344            if [ $2 -ne 0 ]; then # Counting: PCM years taken into account
345                i_myear=$(echo "$i_myear + 1." | bc -l)
346            fi
347        else
348            endlaunch
349        fi
350    done
351}
352
353# To submit the PEM run
354# arg1: launching mode
355submitPEM() {
356    if [ $(echo "$i_myear < $n_myear" | bc -l) -eq 1 ]; then
357        echo "Run \"PEM $iPEM\""
358        if [ $1 -eq 0 ]; then # Mode: processing scripts
359            ./PEMrun.job
360            if [ $? -ne 0 ]; then
361                errlaunch
362            fi
363        else # Mode: submitting jobs
364            sed -i -E "/^$name_job/s/(.*[^0-9])([0-9]+)(_[^0-9]*)?$/\1${iPEM}\3/" PEMrun.job
365            if [[ "$scheduler" == "SLURM" ]]; then
366                jobID=$(sbatch --parsable PEMrun.job)
367            elif [[ "$scheduler" == "PBS" ]]; then
368                jobID=$(qsub PEMrun.job | cut -d. -f1)
369            fi
370            # Create a file to cancel the dependent jobs of the cycle
371            echo "#!/bin/bash" > kill_launchPEM.sh
372            chmod +x kill_launchPEM.sh
373            echo $kill_job $jobID >> kill_launchPEM.sh
374        fi
375    else
376        endlaunch
377    fi
378}
379
380# To make one cycle of PCM and PEM runs
381# arg1: launching mode
382# arg2: counting method
383# arg3: number of PCM runs to launch
384# arg4: local number of the PCM run from which to start (optional)
385cyclelaunch() {
386    # PCM runs
387    submitPCM $1 $2 $3 $4
388
389    # PEM run
390    if [ $(echo "$i_myear < $n_myear" | bc -l) -eq 1 ]; then
391        echo "Run \"PEM $iPEM\""
392        if [ $1 -eq 0 ]; then # Mode: processing scripts
393            ./PEMrun.job
394            if [ $? -ne 0 ]; then
395                errlaunch
396            fi
397        else # Mode: submitting jobs
398            sed -i -E "/^$name_job/s/(.*[^0-9])([0-9]+)(_[^0-9]*)?$/\1${iPEM}\3/" PEMrun.job
399            if [[ "$scheduler" == "SLURM" ]]; then
400                jobID=$(sbatch --parsable --dependency=afterok:${jobID} PEMrun.job)
401            elif [[ "$scheduler" == "PBS" ]]; then
402                jobID=$(qsub -W depend=afterok:${jobID} PEMrun.job | cut -d. -f1)
403            fi
404            echo $kill_job $jobID >> kill_launchPEM.sh
405        fi
406    else
407        endlaunch
408    fi
409}
410
411# To clean files after the starting run of the relaunch
412# arg1: file name prefix to clean
413# arg2: file name extension to clean
414# arg3: file number from which to clean
415cleanfiles() {
416    prefix=$1
417    extension=$2
418    if [ -z "$extension" ]; then
419        for file in ${prefix}*; do
420            num=${file#$prefix}
421            if [[ $num =~ ^[0-9]+$ ]] && [ $num -gt $3 ]; then
422                rm $file
423            fi
424        done
425    else
426        for file in ${prefix}*${extension}; do
427            num=${file#$prefix}
428            num=${num%$extension}
429            if [[ $num =~ ^[0-9]+$ ]] && [ $num -gt $3 ]; then
430                rm $file
431            fi
432        done
433    fi
434}
435
436# To relaunch from PCM run
437# arg1: launching mode
438# arg2: counting method
439relaunchPCM() {
440    iPCM=$(($irelaunch + 1))
441    cleanfiles diags/diagfi .nc $irelaunch
442    cleanfiles diags/diagsoil .nc $irelaunch
443    cleanfiles diags/data2reshape .nc $irelaunch
444    cleanfiles logs/runPCM .log $irelaunch
445    cleanfiles starts/restart1D .txt $irelaunch
446    cleanfiles starts/restart .nc $irelaunch
447    cleanfiles starts/restartfi .nc $irelaunch
448    cp starts/restartfi${irelaunch}.nc startfi.nc
449    if [ -f "starts/restart${irelaunch}.nc" ]; then
450        cp starts/restart${irelaunch}.nc start.nc
451    elif [ -f "starts/restart1D${irelaunch}.txt" ]; then
452        cp starts/restart1D${irelaunch}.txt start1D.txt
453    fi
454    if [ $irelaunch -le $nPCM_ini ]; then
455        # PCM relaunch during the initialization cycle
456        iPEM=1
457        if [ $2 -ne 0 ]; then # Counting: PCM years taken into account
458            i_myear=$irelaunch
459        else # Counting: only PEM years count
460            i_myear=0
461        fi
462        sed -i "1s/.*/$i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini/" info_PEM.txt
463        cleanfiles diags/diagpem .nc $(($iPEM - 1))
464        cleanfiles diags/diagsoilpem .nc $(($iPEM - 1))
465        cleanfiles logs/runPEM .log $(($iPEM - 1))
466        cleanfiles starts/restart1D_postPEM .txt $(($iPEM - 1))
467        cleanfiles starts/restart_postPEM .nc $(($iPEM - 1))
468        cleanfiles starts/restartfi_postPEM .nc $(($iPEM - 1))
469        cleanfiles starts/restartpem .nc $(($iPEM - 1))
470        rm -f startpem.nc
471        if [ -f "starts/startpem.nc" ]; then
472            cp starts/startpem.nc .
473        fi
474        if [ $irelaunch -eq $(($nPCM_ini - 1)) ]; then
475            cp diags/data2reshape${irelaunch}.nc data2reshape_Y1.nc
476            cyclelaunch $1 $2 $nPCM_ini $iPCM
477        elif [ $irelaunch -eq $nPCM_ini ]; then
478            cp diags/data2reshape$(($irelaunch - 1)).nc data2reshape_Y1.nc
479            cp diags/data2reshape${irelaunch}.nc data2reshape_Y2.nc
480            submitPEM $1 # The next job is a PEM run
481        else
482            cyclelaunch $1 $2 $nPCM_ini $iPCM
483        fi
484    else
485        # PCM relaunch during a cycle
486        iPEM=$(echo "($iPCM - $nPCM_ini)/$nPCM + 1" | bc)
487        il=$(echo "($irelaunch - $nPCM_ini + 1)%$nPCM + 1" | bc)
488        if [ $2 -ne 0 ]; then # Counting: PCM years taken into account
489            i_myear=$(echo "$(awk "NR==$iPEM {printf \"%s\n\", \$3}" "info_PEM.txt") + $il" | bc -l)
490        else # Counting: only PEM years count
491            i_myear=$(awk "NR==$iPEM {printf \"%s\n\", \$3}" "info_PEM.txt")
492        fi
493        sed -i "1s/.*/$i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini/" info_PEM.txt
494        cleanfiles diags/diagpem .nc $(($iPEM - 1))
495        cleanfiles diags/diagsoilpem .nc $(($iPEM - 1))
496        cleanfiles logs/runPEM .log $(($iPEM - 1))
497        cleanfiles starts/restart1D_postPEM .txt $(($iPEM - 1))
498        cleanfiles starts/restart_postPEM .nc $(($iPEM - 1))
499        cleanfiles starts/restartfi_postPEM .nc $(($iPEM - 1))
500        cleanfiles starts/restartpem .nc $(($iPEM - 1))
501        cp starts/restartpem$(($iPEM - 1)).nc startpem.nc
502        if [ $il -eq $(($nPCM - 1)) ]; then # Second to last PCM run
503            cp diags/data2reshape${irelaunch}.nc data2reshape_Y1.nc
504            cyclelaunch $1 $2 $nPCM $(($il + 1))
505        elif [ $il -eq $nPCM ]; then # Last PCM run so the next job is a PEM run
506            cp diags/data2reshape$(($irelaunch - 1)).nc data2reshape_Y1.nc
507            cp diags/data2reshape${irelaunch}.nc data2reshape_Y2.nc
508            submitPEM $1
509        else
510            cyclelaunch $1 $2 $nPCM $(($il + 1))
511        fi
512    fi
513}
514
515# To relaunch from PEM run
516# arg1: launching mode
517# arg2: counting method
518relaunchPEM() {
519    iPEM=$(echo "$irelaunch + 1" | bc)
520    iPCM=$(echo "$nPCM_ini + $nPCM*($irelaunch - 1) + 1" | bc)
521    i_myear=$(awk "NR==$iPEM {printf \"%s\n\", \$3}" "info_PEM.txt")
522    sed -i "1s/.*/$i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini/" info_PEM.txt
523    cleanfiles diags/diagfi .nc $(($iPCM - 1))
524    cleanfiles diags/diagsoil .nc $(($iPCM - 1))
525    cleanfiles logs/runPCM .log $(($iPCM - 1))
526    cleanfiles starts/restart1D .txt $(($iPCM - 1))
527    cleanfiles starts/restart .nc $(($iPCM - 1))
528    cleanfiles starts/restartfi .nc $(($iPCM - 1))
529    cleanfiles diags/data2reshape .nc $(($iPCM - 1))
530    cleanfiles diags/diagpem .nc $irelaunch
531    cleanfiles diags/diagsoilpem .nc $irelaunch
532    cleanfiles logs/runPEM .log $irelaunch
533    cleanfiles starts/restart1D_postPEM .txt $irelaunch
534    cleanfiles starts/restart_postPEM .nc $irelaunch
535    cleanfiles starts/restartfi_postPEM .nc $irelaunch
536    cleanfiles starts/restartpem .nc $irelaunch
537    cp starts/restartpem${irelaunch}.nc startpem.nc
538    cp starts/restartfi_postPEM${irelaunch}.nc startfi.nc
539    if [ -f "starts/restart_postPEM${irelaunch}.nc" ]; then
540        cp starts/restart_postPEM${irelaunch}.nc start.nc
541    elif [ -f "starts/restart1D_postPEM${irelaunch}.txt" ]; then
542        cp starts/restart1D_postPEM${irelaunch}.txt start1D.txt
543    fi
544    cyclelaunch $1 $2 $nPCM
545}
Note: See TracBrowser for help on using the repository browser.