source: trunk/LMDZ.COMMON/libf/evolution/deftank/pem_workflow_lib.sh @ 4117

Last change on this file since 4117 was 4117, checked in by jbclement, 9 days ago

PEM:

  • Remove 'd_h2oice' and 'd_co2ice' arguments from adsorption and soil thermal inertia routines (not needed and possible bugs).
  • Move 'read_callphys' call from 'read_start1D' to 'read_rundef' so that 'CO2cond_ps' is initialized no matter what.
  • Explicit initialization of 'delta_h2o_ads'/'delta_co2_ads' before 'read_startpem'.
  • Add time index argument to 'put_var_nc' calls to write "restart.nc".
  • Increase temporary string buffer size for 'real2str_qp'.
  • Fixed some typos.

JBC
File size: 22.0 KB
Line 
1#!/bin/bash
2########################################################################
3######## Library of bash functions for the PEM workflow script #########
4########################################################################
5
6# To end the workflow script
7end_workflow() {
8    # Restore the previous value of LC_NUMERIC
9    LC_NUMERIC=$OLD_LC_NUMERIC
10    date
11    echo "Success: the PEM workflow script completed normally!"
12    exit 0
13}
14
15# To abort the workflow script with error
16abort_workflow() {
17    # Restore the previous value of LC_NUMERIC
18    LC_NUMERIC=$OLD_LC_NUMERIC
19
20    date
21    echo "Error: an issue occured in the PEM workflow script!"
22    exit 1
23}
24
25# To check if the command exists
26# arg1: required command
27require_cmd() {
28    if ! command -v "$1" >/dev/null 2>&1; then
29        echo "Error: required command '$1' not found."
30        echo "Please install or load the corresponding library."
31        abort_workflow
32    fi
33}
34
35# To check what is the job scheduler
36function detect_scheduler() {
37    if command -v squeue &> /dev/null; then
38        echo "SLURM is installed on $machine."
39        job_scheduler="SLURM"
40        job_name="#SBATCH --job-name="
41        kill_job="scancel"
42    elif command -v qstat &> /dev/null; then
43        echo "PBS/TORQUE is installed on $machine."
44        job_scheduler="PBS"
45        job_name="#PBS -N "
46        kill_job="qdel"
47    else
48        echo "Error: neither SLURM nor TORQUE/PBS is installed on $machine!"
49        echo "Please adapt the script to your job scheduler or set 'mode' to 0."
50        abort_workflow
51    fi
52}
53
54# To get the number of slopes for the simulation
55get_nslope() {
56    ns=1
57    if [ -f "startfi.nc" ]; then
58        ns=$(ncdump -h startfi.nc | sed -n 's/.*nslope = \([0-9]*\) ;.*/\1/p')
59    else
60        for f in run_pcm.def callphys.def; do
61            if [[ -f "$f" ]]; then
62                while IFS= read -r line; do
63                    # Remove leading whitespace
64                    trimmed=$(echo "$line" | sed 's/^[[:space:]]*//')
65                    # Skip lines that are commented out
66                    if [[ "$trimmed" == \#* ]]; then
67                        continue
68                    fi
69                    # Check if line contains 'nslope = N'
70                    if [[ "$trimmed" =~ ^nslope[[:space:]]*=[[:space:]]*([0-9]+) ]]; then
71                        ns="${BASH_REMATCH[1]}"
72                        break
73                    fi
74                done < "$f"
75                [[ -n "$ns" ]] && break
76            fi
77        done
78    fi
79}
80
81# To modify the xml file according nslope
82config_xios_outputs() {
83    tmp="tmp_file_def.xml"
84    in_outdaily4pem=false
85    in_outyearly4pem=false
86    in_outdaily4pem_s=false
87    in_outyearly4pem_s=false
88
89    sed -i 's/enabled="\.true\.\">/enabled=".false.">/g' file_def_physics_mars.xml
90
91    while IFS= read -r line; do
92        # Detect file blocks
93        case "$line" in
94            *'<file id="outdaily4pem"'*)
95                in_outdaily4pem=true
96                ;;
97            *'<file id="outyearly4pem"'*)
98                in_outyearly4pem=true
99                ;;
100            *'<file id="outdaily4pem_s"'*)
101                in_outdaily4pem_s=true
102                ;;
103            *'<file id="outyearly4pem_s"'*)
104                in_outyearly4pem_s=true
105                ;;
106        esac
107
108        # Handle enabled attribute
109        if [[ $line == *'enabled="'* ]]; then
110            if $in_outdaily4pem || $in_outyearly4pem; then
111                if [[ $ns -eq 1 ]]; then
112                    line='              enabled=".true.">'
113                else
114                    line='              enabled=".false.">'
115                fi
116            elif $in_outdaily4pem_s || $in_outyearly4pem_s; then
117                if [[ $ns -eq 1 ]]; then
118                    line='              enabled=".false.">'
119                else
120                    line='              enabled=".true.">'
121                fi
122            fi
123        fi
124
125        # Handle slope variables
126        if ( $in_outdaily4pem_s || $in_outyearly4pem_s ) && [[ $line =~ slope([0-9]+) ]]; then
127            slope_id="${BASH_REMATCH[1]}"
128            if (( 10#$slope_id > ns )); then
129                # Ensure the line is commented
130                if [[ $line != "<!--"* ]]; then
131                    line="<!-- $line -->"
132                fi
133            else
134                # Ensure the line is uncommented
135                if [[ $line == "<!--"* ]]; then
136                    line="${line#<!-- }" # remove leading <!--
137                    line="${line% -->}"  # remove trailing -->
138                fi
139            fi
140        fi
141
142
143        # Leaving the file block
144        case "$line" in
145            *'</file>'*)
146                in_outdaily4pem=false
147                in_outyearly4pem=false
148                in_outdaily4pem_s=false
149                in_outyearly4pem_s=false
150                ;;
151        esac
152
153        echo "$line" >> "$tmp"
154    done < file_def_physics_mars.xml
155
156    mv "$tmp" file_def_physics_mars.xml
157}
158
159# To check if a PCM run is one year
160check_run_yearly() {
161    if [ -f "startfi.nc" ]; then
162        year_sol=$(ncdump -v controle startfi.nc 2>/dev/null | \
163                   sed -n '/controle =/,/;/p' | tr -d '[:space:]' | \
164                   sed 's/.*=//; s/;//' | tr ',' '\n' | sed -n '14p')
165    else
166        echo "Warning: no \"startfi.nc\" found! So default year_sol=669 (Mars year) is taken..."
167        year_sol=669 # Length of Martian year (sols)
168    fi
169    sol_in_file=$(awk -F'=' '/^[[:space:]]*(nday|ndt)[[:space:]]*=/ {
170                  val=$2
171                  gsub(/^[[:space:]]+|[[:space:]]+$/,"",val)
172                  print val
173                  exit
174                  }' run_pcm.def)
175
176    if [ -z "$sol_in_file" ]; then
177        echo "Error: no length of year found in \"run_pcm.def\"!"
178        abort_workflow
179    elif [ "$sol_in_file" -eq "$year_sol" ]; then
180        # Good: we do nothing
181        :
182    else
183        echo "Error: length of year mismatch between \"run_pcm.def\" ($sol_in_file) and \"startfi.nc\" ($year_sol)!"
184        abort_workflow
185    fi
186}
187
188# To check the "callphys.def" compatibility to run with the PEM
189# arg1: callphys key
190check_callphys_key() {
191    key="$1"
192    line=$(grep -E "^[[:space:]]*${key}[[:space:]]*=" callphys.def \
193           | grep -v '^[[:space:]]*#' \
194           | tail -n 1)
195    if [[ -z "$line" ]]; then
196        echo "Error: the key '$key' is missing in \"callphys.def\"!"
197        abort_workflow
198    fi
199    # Remove inline comments, extract value, normalize
200    value=$(echo "$line" \
201            | cut -d'#' -f1 \
202            | cut -d'=' -f2 \
203            | tr -d '[:space:]' \
204            | tr '[:upper:]' '[:lower:]')
205    if [[ "$value" == "true" || "$value" == ".true." ]]; then
206        return 0
207    else
208        echo "Error: the key '$key' must be true in \"callphys.def\"!"
209        abort_workflow
210    fi
211}
212
213# To check if everything necessary for the workflow script is ok
214check_workflow() {
215    # Save the current value of LC_NUMERIC and set it to a locale that uses a dot as the decimal separator
216    OLD_LC_NUMERIC=$LC_NUMERIC
217    LC_NUMERIC=en_US.UTF-8
218
219    if [ -v n_planetary_years ] && [ ! -z "$n_planetary_years" ]; then
220        if [ $(echo "$n_planetary_years <= 0." | bc -l) -eq 1 ]; then
221            echo "Error: 'n_planetary_years' must be > 0!"
222            abort_workflow
223        fi
224    elif [ -v n_earth_years ] && [ ! -z "$n_earth_years" ]; then
225        if [ $(echo "$n_earth_years <= 0." | bc -l) -eq 1 ]; then
226            echo "Error: 'n_earth_years' must be > 0!"
227            abort_workflow
228        fi
229    else
230        echo "Error: the number of years to be simulated is not set!"
231        abort_workflow
232    fi
233    if [ $n_pcm_runs_ini -lt 2 ] || [ -z "$n_pcm_runs_ini" ]; then
234        echo "Error: 'n_pcm_runs_ini' must be >= 2!"
235        abort_workflow
236    fi
237    if [ $n_pcm_runs -lt 2 ] || [ -z "$n_pcm_runs" ]; then
238        echo "Error: 'n_pcm_runs' must be >= 2!"
239        abort_workflow
240    fi
241    if [ ! -f "pcm_run.job" ]; then
242        echo "Error: file \"pcm_run.job\" does not exist in $dir!"
243        abort_workflow
244    fi
245    if [ ! -f "pem_run.job" ]; then
246        echo "Error: file \"pem_run.job\" does not exist in $dir!"
247        abort_workflow
248    fi
249    if [ ! -f "run_pcm.def" ]; then
250        echo "Error: file \"run_pcm.def\" does not exist in $dir!"
251        abort_workflow
252    fi
253    if [ ! -f "run_pem.def" ]; then
254        echo "Error: file \"run_pem.def\" does not exist in $dir!"
255        abort_workflow
256    fi
257    if [ ! -f "callphys.def" ]; then
258        echo "Error: file \"callphys.def\" does not exist in $dir!"
259        abort_workflow
260    fi
261    if [ ! -f "z2sig.def" ]; then
262        echo "Error: file \"z2sig.def\" does not exist in $dir!"
263        abort_workflow
264    fi
265    if [ ! -f "traceur.def" ]; then
266        echo "Error: file \"traceur.def\" does not exist in $dir!"
267        abort_workflow
268    fi
269    if [ ! -f "context_lmdz_physics.xml" ]; then
270        echo "Error: file \"context_lmdz_physics.xml\" does not exist in $dir!"
271        abort_workflow
272    fi
273    if [ ! -f "field_def_physics_mars.xml" ]; then
274        echo "Error: file \"field_def_physics_mars.xml\" does not exist in $dir!"
275        abort_workflow
276    fi
277    if [ ! -f "file_def_physics_mars.xml" ]; then
278        echo "Error: file \"file_def_physics_mars.xml\" does not exist in $dir!"
279        abort_workflow
280    fi
281    if [ ! -f "iodef.xml" ]; then
282        echo "Error: file \"iodef.xml\" does not exist in $dir!"
283        abort_workflow
284    fi
285    if [ ! -f "obl_ecc_lsp.asc" ]; then
286        echo "Warning: file \"obl_ecc_lsp.asc\" has not been found in $dir!"
287    fi
288    if [ ! -d "logs" ]; then
289        mkdir logs
290    fi
291    if [ ! -d "starts" ]; then
292        mkdir starts
293    fi
294    if [ ! -d "diags" ]; then
295        mkdir diags
296    fi
297    if [ $exec_mode -ne 0 ]; then
298        detect_scheduler
299    fi
300    require_cmd ncdump
301    # Check if the "callphys.def" is compatible with the PEM
302    check_callphys_key paleoclimate
303    # Set automatically the XIOS output file for the PEM according to the number of slopes
304    get_nslope
305    config_xios_outputs
306    # Check if a PCM run is one year
307    check_run_yearly
308}
309
310# To convert Earth years into Mars years
311convert_earth2plnt_years() {
312    myear=686.9725      # Number of Earth days in Martian year
313    eyear=365.256363004 # Number of days in Earth year
314    r_plnt2earth_yr=$(echo "$myear/$eyear" | bc -l)
315    r_plnt2earth_yr=$(printf "%.4f" $r_plnt2earth_yr) # Rounding to the 4th decimal to respect the precision of Martian year
316    if [ -v n_planetary_years ]; then
317        ntot_yr_sim=$n_planetary_years
318        echo "Number of years to be simulated: $ntot_yr_sim Martian years."
319    elif [ -v n_earth_years ]; then
320        ntot_yr_sim=$(echo "$n_earth_years/$r_plnt2earth_yr" | bc -l)
321        echo "Number of years to be simulated: $n_earth_years Earth years = $ntot_yr_sim Martian years."
322    fi
323}
324
325# To initialize the workflow script
326ini_workflow() {
327    echo "This is a chained simulation for PEM and PCM runs in $dir on $machine by $user."
328    convert_earth2plnt_years
329    n_yr_sim=0.
330    i_pem_run=1
331    i_pcm_run=1
332    if [ -f "startfi.nc" ]; then
333        cp startfi.nc starts/
334    fi
335    if [ -f "start.nc" ]; then
336        cp start.nc starts/
337    elif [ -f "start1D.txt" ]; then
338        cp start1D.txt starts/
339    fi
340    if [ -f "startevo.nc" ]; then
341        cp startevo.nc starts/
342    fi
343
344    # Create a file to manage years of the chained simulation and store some info from the PEM runs
345    echo $n_yr_sim $ntot_yr_sim $r_plnt2earth_yr $i_pcm_run $i_pem_run $n_pcm_runs $n_pcm_runs_ini > pem_workflow.sts
346}
347
348# To submit the PCM runs
349# arg1: execution mode
350# arg2: number of PCM runs to submit
351# arg3: local number of the PCM run from which to start (optional)
352submit_pcm_phase() {
353    find . -type f -name "pcm_run*.job" ! -name "pcm_run.job" -delete
354    ii=1
355    if [ ! -z $3 ]; then
356        ii=$3
357    fi
358    if [ $(echo "$n_yr_sim < $ntot_yr_sim" | bc -l) -eq 1 ]; then
359        echo "Run \"PCM $i_pcm_run\" ($ii/$2)"
360        if [ $1 -eq 0 ]; then # Mode: processing scripts
361            sed -i "s/^k=-\?[0-9]\+$/k=$(echo "$ii - $2 + 2" | bc)/" pcm_run.job
362            ./pcm_run.job
363            if [ $? -ne 0 ]; then
364                abort_workflow
365            fi
366        else # Mode: submitting jobs
367            cp pcm_run.job pcm_run${i_pcm_run}.job
368            sed -i -E "/^$job_name/s/(.*[^0-9])([0-9]+)(_[^0-9]*)?$/\1${i_pcm_run}\3/" pcm_run${i_pcm_run}.job
369            sed -i "s/^k=-\?[0-9]\+$/k=$(echo "$ii - $2 + 2" | bc)/" pcm_run${i_pcm_run}.job
370            if [[ "$job_scheduler" == "SLURM" ]]; then
371                jobID=$(sbatch --parsable pcm_run${i_pcm_run}.job)
372            elif [[ "$job_scheduler" == "PBS" ]]; then
373                jobID=$(qsub pcm_run${i_pcm_run}.job | cut -d. -f1)
374            fi
375            # Create a file to cancel the dependent jobs of the cycle
376            echo "#!/bin/bash" > kill_pem_workflow.sh
377            chmod +x kill_pem_workflow.sh
378            echo $kill_job $jobID >> kill_pem_workflow.sh
379        fi
380        ((i_pcm_run++))
381        ((ii++))
382    else
383        end_workflow
384    fi
385    for ((i = $ii; i <= $2; i++)); do
386        if [ $(echo "$n_yr_sim < $ntot_yr_sim" | bc -l) -eq 1 ]; then
387            echo "Run \"PCM $i_pcm_run\" ($i/$2)"
388            if [ $1 -eq 0 ]; then # Mode: processing scripts
389                sed -i "s/^k=-\?[0-9]\+$/k=$(echo "$i - $2 + 2" | bc)/" pcm_run.job
390                ./pcm_run.job
391                if [ $? -ne 0 ]; then
392                    abort_workflow
393                fi
394            else # Mode: submitting jobs
395                cp pcm_run.job pcm_run${i_pcm_run}.job
396                sed -i -E "/^$job_name/s/(.*[^0-9])([0-9]+)(_[^0-9]*)?$/\1${i_pcm_run}\3/" pcm_run${i_pcm_run}.job
397                sed -i "s/^k=-\?[0-9]\+$/k=$(echo "$i - $2 + 2" | bc)/" pcm_run${i_pcm_run}.job
398                if [[ "$job_scheduler" == "SLURM" ]]; then
399                    jobID=$(sbatch --parsable --dependency=afterok:${jobID} pcm_run${i_pcm_run}.job)
400                elif [[ "$job_scheduler" == "PBS" ]]; then
401                    jobID=$(qsub -W depend=afterok:${jobID} pcm_run${i_pcm_run}.job | cut -d. -f1)
402                fi
403                echo $kill_job $jobID >> kill_pem_workflow.sh
404            fi
405            ((i_pcm_run++))
406        else
407            end_workflow
408        fi
409    done
410}
411
412# To submit the PEM run
413# arg1: execution mode
414submit_pem_phase() {
415    if [ $(echo "$n_yr_sim < $ntot_yr_sim" | bc -l) -eq 1 ]; then
416        echo "Run \"PEM $i_pem_run\""
417        if [ $1 -eq 0 ]; then # Mode: processing scripts
418            ./pem_run.job
419            if [ $? -ne 0 ]; then
420                abort_workflow
421            fi
422        else # Mode: submitting jobs
423            sed -i -E "/^$job_name/s/(.*[^0-9])([0-9]+)(_[^0-9]*)?$/\1${i_pem_run}\3/" pem_run.job
424            if [[ "$job_scheduler" == "SLURM" ]]; then
425                jobID=$(sbatch --parsable pem_run.job)
426            elif [[ "$job_scheduler" == "PBS" ]]; then
427                jobID=$(qsub pem_run.job | cut -d. -f1)
428            fi
429            # Create a file to cancel the dependent jobs of the cycle
430            echo "#!/bin/bash" > kill_pem_workflow.sh
431            chmod +x kill_pem_workflow.sh
432            echo $kill_job $jobID >> kill_pem_workflow.sh
433        fi
434    else
435        end_workflow
436    fi
437}
438
439# To make one cycle of PCM and PEM runs
440# arg1: execution mode
441# arg2: number of PCM runs to submit
442# arg3: local number of the PCM run from which to start (optional)
443submit_cycle() {
444    # PCM runs
445    submit_pcm_phase $1 $2 $3
446
447    # PEM run
448    if [ $(echo "$n_yr_sim < $ntot_yr_sim" | bc -l) -eq 1 ]; then
449        echo "Run \"PEM $i_pem_run\""
450        if [ $1 -eq 0 ]; then # Mode: processing scripts
451            ./pem_run.job
452            if [ $? -ne 0 ]; then
453                abort_workflow
454            fi
455        else # Mode: submitting jobs
456            sed -i -E "/^$job_name/s/(.*[^0-9])([0-9]+)(_[^0-9]*)?$/\1${i_pem_run}\3/" pem_run.job
457            if [[ "$job_scheduler" == "SLURM" ]]; then
458                jobID=$(sbatch --parsable --dependency=afterok:${jobID} pem_run.job)
459            elif [[ "$job_scheduler" == "PBS" ]]; then
460                jobID=$(qsub -W depend=afterok:${jobID} pem_run.job | cut -d. -f1)
461            fi
462            echo $kill_job $jobID >> kill_pem_workflow.sh
463        fi
464    else
465        end_workflow
466    fi
467}
468
469# To clean-up files after resuming
470# arg1: file name prefix to clean
471# arg2: file name extension to clean
472# arg3: file number from which to clean
473cleanup() {
474    prefix=$1
475    extension=$2
476    if [ -z "$extension" ]; then
477        for file in ${prefix}*; do
478            num=${file#$prefix}
479            if [[ $num =~ ^[0-9]+$ ]] && [ $num -gt $3 ]; then
480                rm $file
481            fi
482        done
483    else
484        for file in ${prefix}*${extension}; do
485            num=${file#$prefix}
486            num=${num%$extension}
487            if [[ $num =~ ^[0-9]+$ ]] && [ $num -gt $3 ]; then
488                rm $file
489            fi
490        done
491    fi
492}
493
494# To resume workflow from PCM run
495# arg1: execution mode
496resume_from_pcm_run() {
497    i_pcm_run=$(($i_resume + 1))
498    cleanup diags/diagfi .nc $i_resume
499    cleanup diags/diagsoil .nc $i_resume
500    cleanup diags/Xoutdaily4pem .nc $i_resume
501    cleanup diags/Xoutyearly4pem .nc $i_resume
502    cleanup logs/run_pcm .log $i_resume
503    cleanup starts/restart1D .txt $i_resume
504    cleanup starts/restart .nc $i_resume
505    cleanup starts/restartfi .nc $i_resume
506    cp starts/restartfi${i_resume}.nc startfi.nc
507    if [ -f "starts/restart${i_resume}.nc" ]; then
508        cp starts/restart${i_resume}.nc start.nc
509    elif [ -f "starts/restart1D${i_resume}.txt" ]; then
510        cp starts/restart1D${i_resume}.txt start1D.txt
511    fi
512    if [ $i_resume -le $n_pcm_runs_ini ]; then
513        # PCM resumption during the initialization cycle
514        i_pem_run=1
515        n_yr_sim=0
516        sed -i "1s/.*/$n_yr_sim $ntot_yr_sim $r_plnt2earth_yr $i_pcm_run $i_pem_run $n_pcm_runs $n_pcm_runs_ini/" pem_workflow.sts
517        cleanup diags/diagevo .nc $(($i_pem_run - 1))
518        cleanup diags/diagevo_soil .nc $(($i_pem_run - 1))
519        cleanup logs/run_pem .log $(($i_pem_run - 1))
520        cleanup starts/restart1D_postpem .txt $(($i_pem_run - 1))
521        cleanup starts/restart_postpem .nc $(($i_pem_run - 1))
522        cleanup starts/restartfi_postpem .nc $(($i_pem_run - 1))
523        cleanup starts/restartevo .nc $(($i_pem_run - 1))
524        rm -f startevo.nc
525        if [ -f "starts/startevo.nc" ]; then
526            cp starts/startevo.nc .
527        fi
528        if [ $i_resume -eq $(($n_pcm_runs_ini - 1)) ]; then
529            cp diags/Xoutdaily4pem${i_resume}.nc Xoutdaily4pem_Y1.nc
530            cp diags/Xoutyearly4pem${i_resume}.nc Xoutyearly4pem_Y1.nc
531            submit_cycle $1 $n_pcm_runs_ini $i_pcm_run
532        elif [ $i_resume -eq $n_pcm_runs_ini ]; then
533            cp diags/Xoutdaily4pem$(($i_resume - 1)).nc Xoutdaily4pem_Y1.nc
534            cp diags/Xoutyearly4pem$(($i_resume - 1)).nc Xoutyearly4pem_Y1.nc
535            cp diags/Xoutdaily4pem${i_resume}.nc Xoutdaily4pem_Y2.nc
536            cp diags/Xoutyearly4pem${i_resume}.nc Xoutyearly4pem_Y2.nc
537            submit_pem_phase $1 # The next job is a PEM run
538        else
539            submit_cycle $1 $n_pcm_runs_ini $i_pcm_run
540        fi
541    else
542        # PCM resumption during a cycle
543        i_pem_run=$(echo "($i_pcm_run - $n_pcm_runs_ini)/$n_pcm_runs + 1" | bc)
544        il=$(echo "($i_resume - $n_pcm_runs_ini + 1)%$n_pcm_runs + 1" | bc)
545        n_yr_sim=$(awk "NR==$i_pem_run {printf \"%s\n\", \$3}" "pem_workflow.sts")
546        sed -i "1s/.*/$n_yr_sim $ntot_yr_sim $r_plnt2earth_yr $i_pcm_run $i_pem_run $n_pcm_runs $n_pcm_runs_ini/" pem_workflow.sts
547        cleanup diags/diagevo .nc $(($i_pem_run - 1))
548        cleanup diags/diagevo_soil .nc $(($i_pem_run - 1))
549        cleanup logs/run_pem .log $(($i_pem_run - 1))
550        cleanup starts/restart1D_postpem .txt $(($i_pem_run - 1))
551        cleanup starts/restart_postpem .nc $(($i_pem_run - 1))
552        cleanup starts/restartfi_postpem .nc $(($i_pem_run - 1))
553        cleanup starts/restartevo .nc $(($i_pem_run - 1))
554        cp starts/restartevo$(($i_pem_run - 1)).nc startevo.nc
555        if [ $il -eq $(($n_pcm_runs - 1)) ]; then # Second to last PCM run
556            cp diags/Xoutdaily4pem${i_resume}.nc Xoutdaily4pem_Y1.nc
557            cp diags/Xoutyearly4pem${i_resume}.nc Xoutyearly4pem_Y1.nc
558            submit_cycle $1 $n_pcm_runs $(($il + 1))
559        elif [ $il -eq $n_pcm_runs ]; then # Last PCM run so the next job is a PEM run
560            cp diags/Xoutdaily4pem$(($i_resume - 1)).nc Xoutdaily4pem_Y1.nc
561            cp diags/Xoutyearly4pem$(($i_resume - 1)).nc Xoutyearly4pem_Y1.nc
562            cp diags/Xoutdaily4pem${i_resume}.nc Xoutdaily4pem_Y2.nc
563            cp diags/Xoutyearly4pem${i_resume}.nc Xoutyearly4pem_Y2.nc
564            submit_pem_phase $1
565        else
566            submit_cycle $1 $n_pcm_runs $(($il + 1))
567        fi
568    fi
569}
570
571# To resume workflow from PEM run
572# arg1: execution mode
573resume_from_pem_run() {
574    i_pem_run=$(echo "$i_resume + 1" | bc)
575    i_pcm_run=$(echo "$n_pcm_runs_ini + $n_pcm_runs*($i_resume - 1) + 1" | bc)
576    n_yr_sim=$(awk "NR==$i_pem_run {printf \"%s\n\", \$3}" "pem_workflow.sts")
577    sed -i "1s/.*/$n_yr_sim $ntot_yr_sim $r_plnt2earth_yr $i_pcm_run $i_pem_run $n_pcm_runs $n_pcm_runs_ini/" pem_workflow.sts
578    cleanup diags/diagfi .nc $(($i_pcm_run - 1))
579    cleanup diags/diagsoil .nc $(($i_pcm_run - 1))
580    cleanup logs/run_pcm .log $(($i_pcm_run - 1))
581    cleanup starts/restart1D .txt $(($i_pcm_run - 1))
582    cleanup starts/restart .nc $(($i_pcm_run - 1))
583    cleanup starts/restartfi .nc $(($i_pcm_run - 1))
584    cleanup diags/Xoutdaily4pem .nc $(($i_pcm_run - 1))
585    cleanup diags/Xoutyearly4pem .nc $(($i_pcm_run - 1))
586    cleanup diags/diagevo .nc $i_resume
587    cleanup diags/diagevo_soil .nc $i_resume
588    cleanup logs/run_pem .log $i_resume
589    cleanup starts/restart1D_postpem .txt $i_resume
590    cleanup starts/restart_postpem .nc $i_resume
591    cleanup starts/restartfi_postpem .nc $i_resume
592    cleanup starts/restartevo .nc $i_resume
593    cp starts/restartevo${i_resume}.nc startevo.nc
594    cp starts/restartfi_postpem${i_resume}.nc startfi.nc
595    if [ -f "starts/restart_postpem${i_resume}.nc" ]; then
596        cp starts/restart_postpem${i_resume}.nc start.nc
597    elif [ -f "starts/restart1D_postpem${i_resume}.txt" ]; then
598        cp starts/restart1D_postpem${i_resume}.txt start1D.txt
599    fi
600    submit_cycle $1 $n_pcm_runs
601}
Note: See TracBrowser for help on using the repository browser.