1 | #!/bin/bash |
---|
2 | ######################################################################## |
---|
3 | ######## Library of bash functions for the PEM launching script ######## |
---|
4 | ######################################################################## |
---|
5 | |
---|
6 | # To end the launching script |
---|
7 | endlaunch() { |
---|
8 | # Restore the previous value of LC_NUMERIC |
---|
9 | LC_NUMERIC=$OLD_LC_NUMERIC |
---|
10 | |
---|
11 | date |
---|
12 | echo "Successful end of the launching script for the PEM simulation." |
---|
13 | exit 0 |
---|
14 | } |
---|
15 | |
---|
16 | # To end the launching script with error |
---|
17 | errlaunch() { |
---|
18 | # Restore the previous value of LC_NUMERIC |
---|
19 | LC_NUMERIC=$OLD_LC_NUMERIC |
---|
20 | |
---|
21 | date |
---|
22 | echo "End with error of the launching script for the PEM." |
---|
23 | exit 1 |
---|
24 | } |
---|
25 | |
---|
26 | # To check what is the job scheduler |
---|
27 | function job_scheduler() { |
---|
28 | if command -v squeue &> /dev/null; then |
---|
29 | echo "SLURM is installed on $machine." |
---|
30 | name_job="#SBATCH --job-name=" |
---|
31 | kill_job="scancel" |
---|
32 | submit_job="sbatch --parsable" |
---|
33 | submit_dependjob="sbatch --parsable --dependency" |
---|
34 | elif command -v qstat &> /dev/null; then |
---|
35 | echo "PBS/TORQUE is installed on $machine." |
---|
36 | name_job="#PBS -N " |
---|
37 | kill_job="qdel" |
---|
38 | submit_job="qsub" |
---|
39 | submit_dependjob="qsub -W depend" |
---|
40 | else |
---|
41 | echo "Error: neither SLURM nor TORQUE/PBS is installed on $machine!" |
---|
42 | echo "You need to adapt the script to your job scheduler." |
---|
43 | errlaunch |
---|
44 | fi |
---|
45 | } |
---|
46 | |
---|
47 | # To check if everything necessary for the launching script is ok |
---|
48 | checklaunch() { |
---|
49 | # Save the current value of LC_NUMERIC and set it to a locale that uses a dot as the decimal separator |
---|
50 | OLD_LC_NUMERIC=$LC_NUMERIC |
---|
51 | LC_NUMERIC=en_US.UTF-8 |
---|
52 | |
---|
53 | if [ -v n_mars_years ] && [ ! -z "$n_mars_years" ]; then |
---|
54 | if [ $(echo "$n_mars_years < 0." | bc -l) -eq 1 ]; then |
---|
55 | echo "Error: the value of 'n_mars_years' must be >0!" |
---|
56 | errlaunch |
---|
57 | fi |
---|
58 | elif [ -v n_earth_years ] && [ ! -z "$n_earth_years" ]; then |
---|
59 | if [ $(echo "$n_earth_years < 0." | bc -l) -eq 1 ]; then |
---|
60 | echo "Error: the value of 'n_earth_years' must be >0!" |
---|
61 | errlaunch |
---|
62 | fi |
---|
63 | else |
---|
64 | echo "Error: no number of years to be simulated has been set!" |
---|
65 | errlaunch |
---|
66 | fi |
---|
67 | if [ $nPCM_ini -lt 2 ] || [ -z "$nPCM_ini" ]; then |
---|
68 | echo "Error: the value of 'nPCM_ini' must be >1!" |
---|
69 | errlaunch |
---|
70 | fi |
---|
71 | if [ $nPCM -lt 2 ] || [ -z "$nPCM" ]; then |
---|
72 | echo "Error: the value of 'nPCM' must be >1!" |
---|
73 | errlaunch |
---|
74 | fi |
---|
75 | if [ ! -f "PCMrun.job" ]; then |
---|
76 | echo "Error: file \"PCMrun.job\" does not exist in $dir!" |
---|
77 | errlaunch |
---|
78 | fi |
---|
79 | if [ ! -f "PEMrun.job" ]; then |
---|
80 | echo "Error: file \"PEMrun.job\" does not exist in $dir!" |
---|
81 | errlaunch |
---|
82 | fi |
---|
83 | if [ ! -f "run_PCM.def" ]; then |
---|
84 | echo "Error: file \"run_PCM.def\" does not exist in $dir!" |
---|
85 | errlaunch |
---|
86 | fi |
---|
87 | if [ ! -f "run_PEM.def" ]; then |
---|
88 | echo "Error: file \"run_PEM.def\" does not exist in $dir!" |
---|
89 | errlaunch |
---|
90 | fi |
---|
91 | if [ ! -f "context_lmdz_physics.xml" ]; then |
---|
92 | echo "Error: file \"context_lmdz_physics.xml\" does not exist in $dir!" |
---|
93 | errlaunch |
---|
94 | fi |
---|
95 | if [ ! -f "field_def_physics_mars.xml" ]; then |
---|
96 | echo "Error: file \"field_def_physics_mars.xml\" does not exist in $dir!" |
---|
97 | errlaunch |
---|
98 | fi |
---|
99 | if [ ! -f "file_def_physics_mars.xml" ]; then |
---|
100 | echo "Error: file \"file_def_physics_mars.xml\" does not exist in $dir!" |
---|
101 | errlaunch |
---|
102 | fi |
---|
103 | if [ ! -f "iodef.xml" ]; then |
---|
104 | echo "Error: file \"iodef.xml\" does not exist in $dir!" |
---|
105 | errlaunch |
---|
106 | fi |
---|
107 | if [ ! -d "out_PCM" ]; then |
---|
108 | mkdir out_PCM |
---|
109 | fi |
---|
110 | if [ ! -d "out_PEM" ]; then |
---|
111 | mkdir out_PEM |
---|
112 | fi |
---|
113 | if [ ! -d "starts" ]; then |
---|
114 | mkdir starts |
---|
115 | fi |
---|
116 | if [ ! -d "diags" ]; then |
---|
117 | mkdir diags |
---|
118 | fi |
---|
119 | if [ $mode -ne 0 ]; then |
---|
120 | job_scheduler |
---|
121 | fi |
---|
122 | } |
---|
123 | |
---|
124 | # To convert Earth years into Mars years |
---|
125 | convertyears() { |
---|
126 | myear=686.9725 # Number of Earth days in Martian year |
---|
127 | eyear=365.256363004 # Number of days in Earth year |
---|
128 | convert_years=$(echo "$myear/$eyear" | bc -l) |
---|
129 | convert_years=$(printf "%.4f" $convert_years) # Rounding to the 4th decimal to respect the precision of Martian year |
---|
130 | if [ -v n_mars_years ]; then |
---|
131 | n_myear=$n_mars_years |
---|
132 | echo "Number of years to be simulated: $n_myear Martian years." |
---|
133 | elif [ -v n_earth_years ]; then |
---|
134 | n_myear=$(echo "$n_earth_years/$convert_years" | bc -l) |
---|
135 | echo "Number of years to be simulated: $n_earth_years Earth years = $n_myear Martian years." |
---|
136 | fi |
---|
137 | } |
---|
138 | |
---|
139 | # To initialize the launching script |
---|
140 | initlaunch() { |
---|
141 | echo "This is a chained simulation for PEM and PCM runs in $dir on $machine by $user." |
---|
142 | convertyears |
---|
143 | i_myear=0. |
---|
144 | iPEM=1 |
---|
145 | iPCM=1 |
---|
146 | cp startfi.nc starts/ |
---|
147 | if [ -f "start.nc" ]; then |
---|
148 | cp start.nc starts/ |
---|
149 | elif [ -f "star1D.nc" ]; then |
---|
150 | cp star1D.txt starts/ |
---|
151 | fi |
---|
152 | if [ -f "startpem.nc" ]; then |
---|
153 | cp startpem.nc starts/ |
---|
154 | fi |
---|
155 | |
---|
156 | # Create a file to manage years of the chained simulation and store some info from the PEM runs |
---|
157 | echo $i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini > info_PEM.txt |
---|
158 | } |
---|
159 | |
---|
160 | # To submit the PCM runs |
---|
161 | # arg1: launching mode |
---|
162 | # arg2: counting method |
---|
163 | # arg3: number of PCM runs to launch |
---|
164 | # arg4: local number of the PCM run from which to start (optional) |
---|
165 | submitPCM() { |
---|
166 | find . -type f -name "PCMrun*.job" ! -name "PCMrun.job" -delete |
---|
167 | ii=1 |
---|
168 | if [ ! -z $4 ]; then |
---|
169 | ii=$4 |
---|
170 | fi |
---|
171 | if [ $(echo "$i_myear < $n_myear" | bc -l) -eq 1 ]; then |
---|
172 | echo "Run PCM $iPCM: call $ii/$3..." |
---|
173 | if [ $1 -eq 0 ]; then # Mode: processing scripts |
---|
174 | sed -i "s/^k=[0-9]\+$/k=$(echo "3 - $nPCM_ini" | bc)/" PCMrun.job |
---|
175 | ./PCMrun.job |
---|
176 | if [ $? -ne 0 ]; then |
---|
177 | errlaunch |
---|
178 | fi |
---|
179 | else # Mode: submitting jobs |
---|
180 | cp PCMrun.job PCMrun${iPCM}.job |
---|
181 | sed -i -E "s/($name_job[^0-9]*[0-9]*[^0-9]*)[0-9]+$/\1${iPCM}/" PCMrun${iPCM}.job |
---|
182 | sed -i "s/^k=[0-9]\+$/k=$(echo "3 - $nPCM_ini" | bc)/" PCMrun${iPCM}.job |
---|
183 | jobID=$(eval "$submit_job PCMrun${iPCM}.job") |
---|
184 | # Create a file to cancel the dependent jobs of the cycle |
---|
185 | echo "#!/bin/bash" > kill_launchPEM.sh |
---|
186 | chmod +x kill_launchPEM.sh |
---|
187 | echo $kill_job $jobID >> kill_launchPEM.sh |
---|
188 | fi |
---|
189 | ((iPCM++)) |
---|
190 | if [ $2 -ne 0 ]; then # Counting: PCM runs taken into account |
---|
191 | i_myear=$(echo "$i_myear + 1." | bc -l) |
---|
192 | fi |
---|
193 | ((ii++)) |
---|
194 | else |
---|
195 | endlaunch |
---|
196 | fi |
---|
197 | for ((i = $ii; i <= $3; i++)); do |
---|
198 | if [ $(echo "$i_myear < $n_myear" | bc -l) -eq 1 ]; then |
---|
199 | echo "Run PCM $iPCM: call $i/$3..." |
---|
200 | if [ $1 -eq 0 ]; then # Mode: processing scripts |
---|
201 | sed -i "s/^k=[0-9]\+$/k=$(echo "$i + 2 - $nPCM_ini" | bc)/" PCMrun.job |
---|
202 | ./PCMrun.job |
---|
203 | if [ $? -ne 0 ]; then |
---|
204 | errlaunch |
---|
205 | fi |
---|
206 | else # Mode: submitting jobs |
---|
207 | cp PCMrun.job PCMrun${iPCM}.job |
---|
208 | sed -i -E "s/($name_job[^0-9]*[0-9]*[^0-9]*)[0-9]+$/\1${iPCM}/" PCMrun${iPCM}.job |
---|
209 | sed -i "s/^k=[0-9]\+$/k=$(echo "$i + 2 - $nPCM_ini" | bc)/" PCMrun${iPCM}.job |
---|
210 | jobID=$(eval "$submit_dependjob=afterok:${jobID} PCMrun${iPCM}.job") |
---|
211 | echo $kill_job $jobID >> kill_launchPEM.sh |
---|
212 | fi |
---|
213 | ((iPCM++)) |
---|
214 | if [ $2 -ne 0 ]; then # Counting: PCM runs taken into account |
---|
215 | i_myear=$(echo "$i_myear + 1." | bc -l) |
---|
216 | fi |
---|
217 | else |
---|
218 | endlaunch |
---|
219 | fi |
---|
220 | done |
---|
221 | } |
---|
222 | |
---|
223 | # To submit the PEM run |
---|
224 | # arg1: launching mode |
---|
225 | submitPEM() { |
---|
226 | if [ $(echo "$i_myear < $n_myear" | bc -l) -eq 1 ]; then |
---|
227 | echo "Run PEM $iPEM" |
---|
228 | if [ $1 -eq 0 ]; then # Mode: processing scripts |
---|
229 | ./PEMrun.job |
---|
230 | if [ $? -ne 0 ]; then |
---|
231 | errlaunch |
---|
232 | fi |
---|
233 | else # Mode: submitting jobs |
---|
234 | sed -i -E "s/($name_job[^0-9]*[0-9]*[^0-9]*)[0-9]+$/\1${iPEM}/" PEMrun.job |
---|
235 | jobID=$(eval "$submit_job PEMrun.job") |
---|
236 | # Create a file to cancel the dependent jobs of the cycle |
---|
237 | echo "#!/bin/bash" > kill_launchPEM.sh |
---|
238 | chmod +x kill_launchPEM.sh |
---|
239 | echo $kill_job $jobID >> kill_launchPEM.sh |
---|
240 | fi |
---|
241 | else |
---|
242 | endlaunch |
---|
243 | fi |
---|
244 | } |
---|
245 | |
---|
246 | # To make one cycle of PCM and PEM runs |
---|
247 | # arg1: launching mode |
---|
248 | # arg2: counting method |
---|
249 | # arg3: number of PCM runs to launch |
---|
250 | # arg4: local number of the PCM run from which to start (optional) |
---|
251 | cyclelaunch() { |
---|
252 | # PCM runs |
---|
253 | submitPCM $1 $2 $3 $4 |
---|
254 | |
---|
255 | # PEM run |
---|
256 | if [ $(echo "$i_myear < $n_myear" | bc -l) -eq 1 ]; then |
---|
257 | echo "Run PEM $iPEM" |
---|
258 | if [ $1 -eq 0 ]; then # Mode: processing scripts |
---|
259 | ./PEMrun.job |
---|
260 | if [ $? -ne 0 ]; then |
---|
261 | errlaunch |
---|
262 | fi |
---|
263 | else # Mode: submitting jobs |
---|
264 | sed -i -E "s/($name_job[^0-9]*[0-9]*[^0-9]*)[0-9]+$/\1${iPEM}/" PEMrun.job |
---|
265 | jobID=$(eval "$submit_dependjob=afterok:${jobID} PEMrun.job") |
---|
266 | echo $kill_job $jobID >> kill_launchPEM.sh |
---|
267 | fi |
---|
268 | else |
---|
269 | endlaunch |
---|
270 | fi |
---|
271 | } |
---|
272 | |
---|
273 | # To clean files after the starting run of the relaunch |
---|
274 | # arg1: file name prefix to clean |
---|
275 | # arg2: file name extension to clean |
---|
276 | # arg3: file number from which to clean |
---|
277 | cleanfiles() { |
---|
278 | prefix=$1 |
---|
279 | extension=$2 |
---|
280 | if [ -z "$extension" ]; then |
---|
281 | for file in ${prefix}*; do |
---|
282 | num=${file#$prefix} |
---|
283 | if [[ $num =~ ^[0-9]+$ ]] && [ $num -gt $3 ]; then |
---|
284 | rm $file |
---|
285 | fi |
---|
286 | done |
---|
287 | else |
---|
288 | for file in ${prefix}*${extension}; do |
---|
289 | num=${file#$prefix} |
---|
290 | num=${num%$extension} |
---|
291 | if [[ $num =~ ^[0-9]+$ ]] && [ $num -gt $3 ]; then |
---|
292 | rm $file |
---|
293 | fi |
---|
294 | done |
---|
295 | fi |
---|
296 | } |
---|
297 | |
---|
298 | # To relaunch from PCM run |
---|
299 | # arg1: launching mode |
---|
300 | # arg2: counting method |
---|
301 | relaunchPCM() { |
---|
302 | iPCM=$(($irelaunch + 1)) |
---|
303 | cleanfiles diags/diagfi .nc $irelaunch |
---|
304 | cleanfiles diags/diagsoil .nc $irelaunch |
---|
305 | cleanfiles diags/data2reshape .nc $irelaunch |
---|
306 | cleanfiles "out_PCM/run" "" $irelaunch |
---|
307 | cleanfiles starts/restart1D .txt $irelaunch |
---|
308 | cleanfiles starts/restart .nc $irelaunch |
---|
309 | cleanfiles starts/restartfi .nc $irelaunch |
---|
310 | cp starts/restartfi${irelaunch}.nc startfi.nc |
---|
311 | if [ -f "starts/restart${irelaunch}.nc" ]; then |
---|
312 | cp starts/restart${irelaunch}.nc start.nc |
---|
313 | elif [ -f "starts/restart1D${irelaunch}.txt" ]; then |
---|
314 | cp starts/restart1D${irelaunch}.txt start1D.txt |
---|
315 | fi |
---|
316 | if [ $irelaunch -le $nPCM_ini ]; then |
---|
317 | # PCM relaunch during the initialization cycle |
---|
318 | iPEM=1 |
---|
319 | if [ $2 -ne 0 ]; then # Counting: PCM runs taken into account |
---|
320 | i_myear=$irelaunch |
---|
321 | else # Counting: only PEM runs count |
---|
322 | i_myear=0 |
---|
323 | fi |
---|
324 | sed -i "1s/.*/$i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini/" info_PEM.txt |
---|
325 | cleanfiles diags/diagpem .nc $(($iPEM - 1)) |
---|
326 | cleanfiles diags/diagsoilpem .nc $(($iPEM - 1)) |
---|
327 | cleanfiles "out_PEM/run" "" $(($iPEM - 1)) |
---|
328 | cleanfiles starts/restart1D_postPEM .txt $(($iPEM - 1)) |
---|
329 | cleanfiles starts/restart_postPEM .nc $(($iPEM - 1)) |
---|
330 | cleanfiles starts/restartfi_postPEM .nc $(($iPEM - 1)) |
---|
331 | cleanfiles starts/restartpem .nc $(($iPEM - 1)) |
---|
332 | rm -f startpem.nc |
---|
333 | if [ $irelaunch -eq $(($nPCM_ini - 1)) ]; then |
---|
334 | cp diags/data2reshape${irelaunch}.nc data2reshape_Y1.nc |
---|
335 | cyclelaunch $1 $2 $nPCM_ini $iPCM |
---|
336 | elif [ $irelaunch -eq $nPCM_ini ]; then |
---|
337 | cp diags/data2reshape$(($irelaunch - 1)).nc data2reshape_Y1.nc |
---|
338 | cp diags/data2reshape${irelaunch}.nc data2reshape_Y2.nc |
---|
339 | submitPEM $1 # The next job is a PEM run |
---|
340 | else |
---|
341 | cyclelaunch $1 $2 $nPCM_ini $iPCM |
---|
342 | fi |
---|
343 | else |
---|
344 | # PCM relaunch during a cycle |
---|
345 | iPEM=$(echo "($iPCM - $nPCM_ini)/$nPCM + 1" | bc) |
---|
346 | il=$(echo "($irelaunch - $nPCM_ini + 1)%$nPCM + 1" | bc) |
---|
347 | if [ $2 -ne 0 ]; then # Counting: PCM runs taken into account |
---|
348 | i_myear=$(($(awk "NR==$iPEM {print \$3}" "info_PEM.txt") + $il)) |
---|
349 | else # Counting: only PEM runs count |
---|
350 | i_myear=$(awk "NR==$iPEM {print \$3}" "info_PEM.txt") |
---|
351 | fi |
---|
352 | sed -i "1s/.*/$i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini/" info_PEM.txt |
---|
353 | cleanfiles diags/diagpem .nc $(($iPEM - 1)) |
---|
354 | cleanfiles diags/diagsoilpem .nc $(($iPEM - 1)) |
---|
355 | cleanfiles "out_PEM/run" "" $(($iPEM - 1)) |
---|
356 | cleanfiles starts/restart1D_postPEM .txt $(($iPEM - 1)) |
---|
357 | cleanfiles starts/restart_postPEM .nc $(($iPEM - 1)) |
---|
358 | cleanfiles starts/restartfi_postPEM .nc $(($iPEM - 1)) |
---|
359 | cleanfiles starts/restartpem .nc $(($iPEM - 1)) |
---|
360 | cp starts/restartpem$(($iPEM - 1)).nc startpem.nc |
---|
361 | if [ $il -eq $(($nPCM - 1)) ]; then # Second to last PCM run |
---|
362 | cp diags/data2reshape${irelaunch}.nc data2reshape_Y1.nc |
---|
363 | cyclelaunch $1 $2 $nPCM $il |
---|
364 | elif [ $il -eq $nPCM ]; then # Last PCM run so the next job is a PEM run |
---|
365 | cp diags/data2reshape$(($irelaunch - 1)).nc data2reshape_Y1.nc |
---|
366 | cp diags/data2reshape${irelaunch}.nc data2reshape_Y2.nc |
---|
367 | submitPEM $1 |
---|
368 | else |
---|
369 | cyclelaunch $1 $2 $nPCM $il |
---|
370 | fi |
---|
371 | fi |
---|
372 | } |
---|
373 | |
---|
374 | # To relaunch from PEM run |
---|
375 | # arg1: launching mode |
---|
376 | # arg2: counting method |
---|
377 | relaunchPEM() { |
---|
378 | iPEM=$(echo "$irelaunch + 1" | bc) |
---|
379 | iPCM=$(echo "$nPCM_ini + $nPCM*($irelaunch - 1) + 1" | bc) |
---|
380 | i_myear=$(awk "NR==$(($iPEM + 1)) {print \$3}" "info_PEM.txt") |
---|
381 | sed -i "1s/.*/$i_myear $n_myear $convert_years $iPCM $iPEM $nPCM $nPCM_ini/" info_PEM.txt |
---|
382 | cleanfiles diags/diagfi .nc $(($iPCM - 1)) |
---|
383 | cleanfiles diags/diagsoil .nc $(($iPCM - 1)) |
---|
384 | cleanfiles "out_PCM/run" "" $(($iPCM - 1)) |
---|
385 | cleanfiles starts/restart1D .txt $(($iPCM - 1)) |
---|
386 | cleanfiles starts/restart .nc $(($iPCM - 1)) |
---|
387 | cleanfiles starts/restartfi .nc $(($iPCM - 1)) |
---|
388 | cleanfiles diags/data2reshape .nc $(($iPCM - 1)) |
---|
389 | cleanfiles diags/diagpem .nc $irelaunch |
---|
390 | cleanfiles diags/diagsoilpem .nc $irelaunch |
---|
391 | cleanfiles "out_PEM/run" "" $irelaunch |
---|
392 | cleanfiles starts/restart1D_postPEM .txt $irelaunch |
---|
393 | cleanfiles starts/restart_postPEM .nc $irelaunch |
---|
394 | cleanfiles starts/restartfi_postPEM .nc $irelaunch |
---|
395 | cleanfiles starts/restartpem .nc $irelaunch |
---|
396 | cp starts/restartpem${irelaunch}.nc startpem.nc |
---|
397 | cp starts/restartfi_postPEM${irelaunch}.nc startfi.nc |
---|
398 | if [ -f "starts/restart_postPEM${irelaunch}.nc" ]; then |
---|
399 | cp starts/restart_postPEM${irelaunch}.nc start.nc |
---|
400 | elif [ -f "starts/restart1D_postPEM${irelaunch}.txt" ]; then |
---|
401 | cp starts/restart1D_postPEM${irelaunch}.txt start1D.txt |
---|
402 | fi |
---|
403 | cyclelaunch $1 $2 $nPCM |
---|
404 | } |
---|