Changeset 5000

Timestamp:

Jul 1, 2024, 10:09:23 AM (5 months ago)

Author:

abarral

Message:

update lmdz_env.sh to separate bash run and srun

Location:

BOL/LMDZ_Setup_amaury

Files:

• lmdz_env.sh (modified) (4 diffs)
• script_SIMU (modified) (1 diff)
• setup.sh (modified) (3 diffs)

BOL/LMDZ_Setup_amaury/lmdz_env.sh

@@ -44 +44 @@
       NB_MPI_MAX=2000
       NB_OMP_MAX=20
-      RUNCMD="srun --label -n"
+      MPICMD="srun -n"
+      RUNBASHCMD="srun -A $account@cpu --label -n 1 -c"
       SUBMITCMD="sbatch -A $account@cpu"
       ;;
@@ -54 +55 @@
       module load cdo/2.3.0
 
-      root_dir=~/"LMDZ_SETUP_ROOT/LMDZ_Setup";
+      root_dir="/data/abarral/LMDZ_SETUP_ROOT/LMDZ_Setup";
       ARCH="X64_MESOIPSL-GNU"
       SIMRUNBASEDIR="$SCRATCH/"
       LMDZD="$root_dir/LMDZD"
       LMDZ_INIT="$root_dir/LMDZ_Init"
-      NB_MPI_MAX=20
+      NB_MPI_MAX=5
       NB_OMP_MAX=1
-      RUNCMD="mpirun -n"  # on spirit, we can't run MPI using srun from within sbatch
+      MPICMD="mpirun -np"  # on spirit, we can't run MPI using srun from within sbatch
+      RUNBASHCMD="bash"
       SUBMITCMD="sbatch"
       ;;
@@ -94 +96 @@
       NB_MPI_MAX=1  # TODO 2000 in JZ
       NB_OMP_MAX=1  # TODO 20 in JZ
-      RUNCMD="srun --label --account=$account --constraint=GENOA -n"
+      MPICMD="srun -n"
+      RUNBASHCMD="srun --label --account=$account --constraint=GENOA --ntasks-per-node=1 -n 1 --time=00:10:00 -c"
       SUBMITCMD="sbatch --constraint=GENOA --account=$account"
       ;;
@@ -105 +108 @@
       NB_MPI_MAX=2  # Max number of MPI cores (only for running simulations)
       NB_OMP_MAX=1  # Max number of OMP threads (only for running simulations)
-      RUNCMD="mpirun -np" # command to run a job, as $runcmd <nprocs> <script>
+      MPICMD="mpirun -np" # command to run an mpi executable, as $MPICMD <nprocs> <script>
+      RUNBASHCMD="bash" # command to run a bash job, as $runbashcmd (nthreads) <script> [nthreads only supplied if =/="bash"]
       SUBMITCMD="."  # command to sumbit a job, as $submitcmd <script>
       ;;

BOL/LMDZ_Setup_amaury/script_SIMU

@@ -322 +322 @@
 echo '##################################################################'
 
-time $RUNCMD $ntasks ./gcm.e > listing
+time $MPICMD $ntasks ./gcm.e > listing
 if [ ! -f restartphy.nc ] ; then
 echo PROBLEME PAS DE FICHIER RESTARTPHY

BOL/LMDZ_Setup_amaury/setup.sh

@@ -222 +222 @@
   chmod +x install_lmdz.sh
   local make_j=8
-  echo "./install_lmdz.sh -noclean $optim -v $version $svnopt -d $resol -rad $rad -bench 0 -parallel mpi_omp $ins_cosp $ins_xios $ins_aero $ins_inlandsis -name $LMDZname -veget $veget -netcdf $netcdf -arch $ARCH -make_j $make_j" >> install_lmdz_options.$$.sh
+  # We launch using $MPICMD, except if it's using mpirun (no srun equivalent for bash script) => if supported, the compilation runs in a cluster job
+  jobcmd="\"$RUNBASHCMD $make_j\""
+  if [[ ${hostname:0:5} = "jean-" ]]; then jobcmd="\"$RUNBASHCMD $make_j --partition=compil\""; fi  # On JeanZay: compile on the <compil> partition
+  if [[ $(echo "$RUNBASHCMD" | cut -c -4) = "bash" ]]; then
+     jobcmd="bash"
+  fi
+  echo "./install_lmdz.sh -noclean $optim -v $version $svnopt -d $resol -rad $rad -bench 0 -parallel mpi_omp $ins_cosp $ins_xios $ins_aero $ins_inlandsis -name $LMDZname -veget $veget -netcdf $netcdf -arch $ARCH -make_j $make_j -jobcmd $jobcmd" >> install_lmdz_options.$$.sh
   chmod +x install_lmdz_options.$$.sh
   echo "Running install_lmdz_options.$$.sh"
   set -o pipefail
-  # We launch using $RUNCMD, except if it's using mpirun (no srun equivalent for bash script) => if supported, the compilation runs in a cluster job
-  local install_cmd="$RUNCMD $make_j"
-  if [[ $(echo "$RUNCMD" | cut -c -6) = "mpirun" ]]; then
-     install_cmd=""
-  fi
-   gcm=$MODEL/$($install_cmd ./install_lmdz_options.$$.sh | tee /dev/tty | tail -n 1 | sed -n "s:.* executable is \(.*\.e\).*:\1:p")
-#    gcm=$MODEL/$(./install_lmdz_options.$$.sh | tee /dev/tty | tail -n 1 | sed -n "s:.* executable is \(.*\.e\).*:\1:p")
+    gcm=$MODEL/$(./install_lmdz_options.$$.sh | tee /dev/tty | tail -n 1 | sed -n "s:.* executable is \(.*\.e\).*:\1:p")
   set +o pipefail
   mv install_lmdz.sh install_lmdz.$$.sh
@@ -433 +433 @@
   -e "s/nthreads=.*./nthreads=$omp/" \
   -e "s/MAINDIR=.*.$/MAINDIR=$(basename "$local")/" \
-  -e "s:STORED=.*.*:STORED=$(dirname $local):" \
+  -e "s:STORED=.*.*:STORED=$(dirname "$local"):" \
   -e "s:SCRATCHD=.*.*:SCRATCHD=$SIMRUNBASEDIR:" \
   -e "s/stopsim=.*.$/stopsim=$stopsim/" \
@@ -640 +640 @@
   sed -e 's/anneeref=.*.$/anneeref='\$yr'/' DEF/run.def >| run.def
   echo Starting initialisation
-  OMP_NUM_THREADS=1 $RUNCMD 1 $ce0l  # ce0l requires MPI=OMP=1
+  OMP_NUM_THREADS=1 $MPICMD 1 $ce0l  # ce0l requires MPI=OMP=1
   if [ $climato = 0 ]; then mv limit.nc limit.\$yr.nc; fi
 done