Changeset 4980 for BOL

Timestamp:

Jun 13, 2024, 2:39:53 PM (5 months ago)

Author:

abarral

Message:

(WIP) update spirit arch
update script_SIMU env
fix setup.sh, add netcdf option
fix aerosols detection

Location:

BOL/LMDZ_Setup_amaury

Files:

• lmdz_env.sh (modified) (3 diffs)
• main.sh (modified) (1 diff)
• script_SIMU (modified) (5 diffs)
• setup.sh (modified) (11 diffs)

BOL/LMDZ_Setup_amaury/lmdz_env.sh

@@ -51 +51 @@
     module load gcc/11.2.0
     module load openmpi/4.0.7
+    module load cdo/2.3.0
 
     root_dir=~/"LMDZ_SETUP_ROOT/LMDZ_Setup";
@@ -59 +60 @@
     NB_MPI_MAX=20
     NB_OMP_MAX=1
-    RUNCMD="srun --label -n"
+    RUNCMD="mpirun -n"  # on spirit, we can't run MPI using srun from within sbatch
     SUBMITCMD="sbatch"
     ;;
@@ -79 +80 @@
     SUBMITCMD="sbatch --account=$account"
     ;;
-  *) echo "WARNING: RUNNING THIS SCRIPT ON A LOCAL COMPUTER IS DISCOURAGED (lacklusted cpus and memory)"
+  *) echo "WARNING: RUNNING THIS SCRIPT ON A LOCAL COMPUTER IS DISCOURAGED (lackluster cpus and memory)"
     root_dir="/home/abarral/PycharmProjects/installLMDZ/LMDZ_Setup"; # Where LMDZ_Setup is extracted. Can't use $(pwd) since this script gets copied and ran from several locations
     ARCH="local-gfortran-parallel"

BOL/LMDZ_Setup_amaury/main.sh

@@ -48 +48 @@
 # Grid number of points IMxJMxLM
 resol="144x142x79"
+resol="32x32x39"  # TODO TEMP
 
 # Using XIOS for IOs: "-xios" for enabled, else ""

BOL/LMDZ_Setup_amaury/script_SIMU

@@ -23 +23 @@
 #@SP#SBATCH --time=00:30:00
 #@SP#SBATCH --output=outNOM_SIMU%j
-#@SP#SBATCH --error=outNOM_SIMU%j#@SP#Spirit
-#@SP#SBATCH --job-name=NOM_SIMU
-#@SP#SBATCH --ntasks=8
-#@SP#SBATCH --cpus-per-task=8
-#@SP#SBATCH --hint=nomultithread
-#@SP#SBATCH --time=00:30:00
-#@SP#SBATCH --output=outNOM_SIMU%j
 #@SP#SBATCH --error=outNOM_SIMU%j
 #@ADS#Adastra
@@ -40 +33 @@
 #@ADS#SBATCH --error=outNOM_SIMU%j
 
-set -ex
-set +u  # Amaury 06/24: for local use (no $SLURM_CPUS_PER_TASK) - should be removed if we ever rewrite this whole legacy script properly...
+set -eux
 
 # Number of MPI processes :
@@ -48 +40 @@
 nthreads=8
 export OMP_NUM_THREADS=$nthreads
-
-# For Jean-Zay (replacing the next 3 lines, commented out) 
-#export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
-
+#@JZ#export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK  # For Jean-Zay (replacing the next 3 lines, commented out)
 # private memory for each thread
 export OMP_STACKSIZE=800M
-# Binding
-export OMP_PLACES=cores
+
+
+#@JZ#export OMP_PLACES=cores  # "binding" present in old script_SIMU, but terribly reduces performance on Spirit...
+ulimit -s unlimited
+#@SP ulimit -Ss 8192
 
 simul=NOM_SIMU
-ulimit -s unlimited
-#### reste ada  : export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/smplocal/pub/NetCDF/4.1.3/lib:/smplocal/pub/HDF5/1.8.9/seq/lib
-
 
 reseau_local=0
@@ -403 +392 @@
       if [ $num -le 250 ] ; then
          sed -e 's/^simul=.*.$/simul='$simul_new'/' -e 's/^\# @ job_nam.*.=.*.$/\# @ job_name = '$simul_new'/' tmp_$simul >| tmp_$simul_new
-         sbatch tmp_$simul_new
+         $SUBMITCMD tmp_$simul_new
       fi
    fi
@@ -417 +406 @@
 
 cd $SCRIPTDIR
-sbatch tmp_$simul
+$SUBMITCMD tmp_$simul

BOL/LMDZ_Setup_amaury/setup.sh

@@ -13 +13 @@
   #   add "-debug" to the compilation options in the compile.sh script created in the present folder, and run it (./compile.sh gcm)
   #   ALTERNATIVELY : you can remove $optim in the definition of "LMDZname" below
-  optim=""
-  #optim="-debug"
+  #optim=""
+  optim="-debug"  # TODO temp
 
   #NOTE : "testmode=y" is for running in test mode :
@@ -41 +41 @@
   # - number: orchidee version number : only rev 7983 on branch _2_2, and 7994 on trunk, are available
   veget="CMIP6"
+  veget="none"  # TODO temp
   #AS : If you have installed the model with a given "veget" option, and you want to change it :
   #   --> RECOMMENDED : re-install the model from scratch in a new TEST_PROD folder
@@ -64 +65 @@
   #          For aerosols=n, the corresponding flags will automatically be set to "n".
   #   With XIOS : adjust DEF/XMLfiles*/file*xml
+
+  netcdf=0  # TODO set back to 0
 }
 
@@ -209 +212 @@
   # (Temporary) Constraints for aerosols=spla :
   # --> resolution 128x88x79 and rad=rrtm
-  if [[ $aerosols = 1 && $resol != "128x88x79" ]]; then
+  if [[ $aerosols = "spla" && $resol != "128x88x79" ]]; then
     echo 'STOP: For now, <aerosols=spla> requires <resol=128x88x79>, and uses the zoomed grid from gcm.def_zNAfrica_BiJe, for which forcing & initial files are available'
     echo "Right now resol=<$resol>"
@@ -235 +238 @@
   cp "$local/install_lmdz.sh" .
   chmod +x install_lmdz.sh
-  echo "./install_lmdz.sh -noclean $optim -v $version $svnopt -d $resol -rad $rad -bench 0 -parallel mpi_omp $ins_cosp $ins_xios $ins_aero $ins_inlandsis -name $LMDZname -veget $veget -netcdf 0 -arch $ARCH" >> install_lmdz_options.$$.sh
+  echo "./install_lmdz.sh -noclean $optim -v $version $svnopt -d $resol -rad $rad -bench 0 -parallel mpi_omp $ins_cosp $ins_xios $ins_aero $ins_inlandsis -name $LMDZname -veget $veget -netcdf $netcdf -arch $ARCH" >> install_lmdz_options.$$.sh
   chmod +x install_lmdz_options.$$.sh
   echo "Running install_lmdz_options.$$.sh"
@@ -286 +289 @@
   # TEMPORAIREMENT pour spla on force l'utilisation de gcm.def_zNAfrica_BiJe (avec resolution 128x88x79)
   #----------------------------------------------------------------------
-  if [[ $aerosols = spla ]]; then cp DEF/gcm.def_zNAfrica_BiJe DEF/gcm.def; fi
+  if [[ $aerosols = "spla" ]]; then cp DEF/gcm.def_zNAfrica_BiJe DEF/gcm.def; fi
 
   # Inscription du choix ok_guide dans DEF/guide.def
@@ -302 +305 @@
   #   cf options veget, aerosols, cosp, xios
   #---------------------------------------------------------------------
-  if [[ $veget = none ]]; then  VEGET="n"; else VEGET="y"; fi
+  if [[ $veget = "none" ]]; then  VEGET="n"; else VEGET="y"; fi
   sed -i'' -e 's/VEGET=.*.$/VEGET='$VEGET'/' DEF/config.def
 
@@ -603 +606 @@
 #@JZ#SBATCH --job-name=Init         # nom du job
 #@JZ#SBATCH --ntasks=1              # Nombre de processus MPI
-#@JZ#SBATCH --cpus-per-task=16      # nombre de threads OpenMP
+#@JZ#SBATCH --cpus-per-task=1      # nombre de threads OpenMP
 #@JZ# /!\ Attention, la ligne suivante est trompeuse mais dans le vocabulaire
 #@JZ# de Slurm "multithread" fait bien référence à l'hyperthreading.
@@ -615 +618 @@
 #@SP#SBATCH --job-name=Init
 #@SP#SBATCH --ntasks=1
-#@SP#SBATCH --cpus-per-task=16
+#@SP#SBATCH --cpus-per-task=1
 #@SP#SBATCH --hint=nomultithread
 #@SP#SBATCH --time=00:10:00
@@ -623 +626 @@
 #@ADS#SBATCH --job-name=Init
 #@ADS#SBATCH --ntasks=1
-#@ADS#SBATCH --cpus-per-task=16
+#@ADS#SBATCH --cpus-per-task=1
 #@ADS#SBATCH --hint=nomultithread
 #@ADS#SBATCH --time=00:10:00
@@ -636 +639 @@
 . lmdz_env.sh
 ulimit -s unlimited
-export OMP_STACKSIZE=800M
-export OMP_NUM_THREADS=1
 cd $SIMRUNDIR/$INIT
 echo "Executable : $ce0l"