source: BOL/LMDZ_Setup/lmdz_env.sh @ 5480

Last change on this file since 5480 was 5462, checked in by fhourdin, 3 weeks ago

Automatisation gfortran<10
File size: 10.4 KB
Line 
1#!/bin/bash
2########################################################
3# This file loads the required modules and sets the paths for simulations
4# To modify the paths:
5#    1) In the <set_env> function, find the section corresponding to your machine
6#    2) Modify the variables, which are documented in the *) section of <set_env>
7# See the end of <set_env> for the explanation of each
8########################################################
9
10# <root_dir> will be set by sed by setup.sh here
11root_dir=/data/hourdin/TESTS/SPLA
12pub_store=1
13PARALLEL=1
14
15function get_hostname {
16  if grep -q "Adastra" /etc/motd 2>/dev/null ; then
17    hostname="adastra"
18  elif which idrquota &> /dev/null; then
19    hostname="jean-zay"
20  else
21    hostname=$(hostname)
22  fi
23}
24
25function set_env {  # Platform-specific
26  case ${hostname:0:5} in
27#-------------------------------------------------------------------------
28    jean-) # Jean-zay, Idris-CNRS super computer
29#-------------------------------------------------------------------------
30      module purge
31      compilo=19.0.4 # available 2013.0, 2017.2
32      module load intel-compilers/$compilo
33      #module load intel-mpi/$compilo
34      module load intel-mkl/$compilo
35      module load hdf5/1.10.5-mpi
36      module load netcdf/4.7.2-mpi
37      module load netcdf-fortran/4.5.2-mpi
38      module load subversion/1.9.7
39      #Pour module gcc, voir : https://trac.lmd.jussieu.fr/LMDZ/wiki/PortageJeanZay
40      #module load gcc/6.5.0
41      module load nco
42      module load cdo
43      # Imputation de la consommation sur le groupe (projet) actif par defaut,
44      #   idrproj indique le groupe (projet) actif par defaut
45      #   idrproj -d newproj   redefinit "newproj" en tant que  projet actif,
46      #        alors $STORE, $WORK etc vont designer les espaces de "newproj")
47      account="lmd"  # $(idrproj | grep active | awk '{ print $1}') doesn't work on compute nodes
48      ARCH="X64_JEANZAY_PBIOIPSL"
49      SIMRUNBASEDIR="$SCRATCH/$(basename $root_dir)"
50      LMDZD="$WORK/LMDZD"
51      LMDZ_INIT="$WORK/LMDZ/pub"
52      NB_MPI_MAX=2000
53      NB_OMP_MAX=20
54      NB_CORE_PER_NODE_MAX=0
55      MPICMD="srun -n"
56      RUNBASHCMD="srun -A $account@cpu --label -n 1 -c"
57      #SUBMITCMD="sbatch -A $account@cpu"
58      submitcmd() {
59         sbatch -A $account@cpu $1
60      }
61      ;;
62#-------------------------------------------------------------------------
63    spiri) # Spirit : IPSL cluster
64#-------------------------------------------------------------------------
65      module purge
66      module load subversion/1.13.0
67      module load gcc/11.2.0
68      module load openmpi/4.0.7
69      module load cdo/2.3.0
70
71      ARCH="X64_MESOIPSL-GNU"
72      SIMRUNBASEDIR="$SCRATCH/$(basename $root_dir)"
73      LMDZD="$root_dir/LMDZD"
74      LMDZ_INIT="$HOME/LMDZ/pub"
75      NB_MPI_MAX=5
76      NB_OMP_MAX=1
77      NB_CORE_PER_NODE_MAX=0
78      N_HYPERTHREADING=1
79      MPICMD="mpirun -np"  # on spirit, we can't run MPI using srun from within sbatch
80      RUNBASHCMD="bash"
81      #SUBMITCMD="sbatch"
82      submitcmd() {
83         sbatch $1
84      }
85      ;;
86#-------------------------------------------------------------------------
87    adast) # Adastra, Cines computing center
88#-------------------------------------------------------------------------
89      module purge
90      module load PrgEnv-gnu  # we need to load the env because lmdz links some shared libraries
91      module load gcc/13.2.0  # required, see https://dci.dci-gitlab.cines.fr/webextranet/user_support/index.html#prgenv-and-compilers
92      export CRAY_CPU_TARGET=x86-64  # to suppress warnings during Cmake netcdf95 build
93      export FI_CXI_RX_MATCH_MODE=hybrid  # 09/24 otherwise we get random SIGABRT e.g. "libfabric:2490616:1725895288::cxi:core:cxip_ux_onload_cb():2657<warn> c1456: RXC (0x5130:21) PtlTE 84:[Fatal] LE resources not recovered during flow control. FI_CXI_RX_MATCH_MODE=[hybrid|software] is required"
94
95      function cdo {  # cdo is available as a spack cmd which requires a specific, incompatible env
96        unset cdo
97        module purge
98        module load develop GCC-CPU-4.0.0 cdo/2.4.2-omp-mpi
99        cdo "$@"
100        set_env
101      }
102
103      function ferret {
104        unset ferret
105        module purge
106        module load GCC-CPU-3.1.0
107        module load ferret
108        ferret "$@"
109        set_env
110      }
111
112      account=$(/usr/sbin/my_project.py -l 2>&1 | head -1 | cut -d " " -f 3- | cut -c 5-)
113      ARCH="X64_ADASTRA-GNU"
114      SIMRUNBASEDIR="$SCRATCHDIR/$(basename $root_dir)"
115      LMDZD="$WORKDIR/LMDZD"
116      LMDZ_INIT="$WORKDIR/LMDZ/pub"
117      NB_MPI_MAX=2000
118      NB_OMP_MAX=200
119      NB_CORE_PER_NODE_MAX=192
120      N_HYPERTHREADING=1  # Adastra has SMT=2 enabled, but we found no actual performance improvement for the latlon model. Maybe useful for Dynamico ?
121      MPICMD="srun -n"
122#      RUNBASHCMD="srun --label --account=$account --constraint=GENOA --ntasks-per-node=1 -n 1 --time=00:15:00 -c"
123      RUNBASHCMD="bash"  # On Adastra the docs says we can use login nodes for compilation
124      #SUBMITCMD="env $(env | grep -E "SLURM_|SBATCH_|SRUN_" | cut -d= -f1 | awk '{print "-u " $0}' | tr '\n' ' ' ) sbatch --constraint=GENOA --account=$account"  # we need to remove the existing SLURM variables otherwise they may be unexpectedly inherited by the submitted script
125      submitcmd() {
126        env $(env | grep -E "SLURM_|SBATCH_|SRUN_" | cut -d= -f1 | awk '{print "-u " $0}' | tr '\n' ' ' ) sbatch --constraint=GENOA --account=$account $1
127      }  # we need to remove the existing SLURM variables otherwise they may be unexpectedly inherited by the submitted script
128      ;;
129#-------------------------------------------------------------------------
130    *)  # Local machine.
131#-------------------------------------------------------------------------
132      SIMRUNBASEDIR="$root_dir/SCRATCH/"  # Where the simulations will be executed ($SIMRUNBASEDIR/LMDZ_Setup/...)
133      LMDZD="$root_dir/../LMDZD"  # Where the sources will be downloaded and compiled
134      if [[ $pub_store ]] ; then LMDZ_INIT="$HOME/LMDZ/pub" ; else LMDZ_INIT="$root_dir/LMDZ/pub" ; fi
135      # Old fortran versions
136      if [ $( gfortran --version | head -1  | awk ' { print $NF } ' | cut -d. -f1 ) -le 9 ] ; then 
137          gfortranv=gfortran9 ; archdir="-arch_dir arch"
138          if [ ! -d $LMDZD/arch ] ; then mkdir -p $LMDZD ; cd $LMDZD ; 
139            wget http://lmdz.lmd.jussieu.fr/pub/src_archives/misc/arch/LMDZ/arch-local-gfortran9.tar
140            tar xvf arch-local-gfortran9.tar
141            cd -
142          fi
143      else
144          gfortranv=gfortran ; archdir=
145      fi
146      if [[ $PARALLEL = 0 ]] ; then
147         ARCH="local-${gfortranv} $archdir"  # The arch file to use
148         NB_MPI_MAX=1  # Max number of MPI cores (only for running simulations)
149         NB_OMP_MAX=1  # Max number of OMP threads (only for running simulations)
150         MPICMD="" # command to run an mpi executable, as $MPICMD <nprocs> <script>
151      else
152         ARCH="local-${gfortranv}-parallel $archdir"  # The arch file to use
153         NB_MPI_MAX=2  # Max number of MPI cores (only for running simulations)
154         NB_OMP_MAX=2  # Max number of OMP threads (only for running simulations)
155         MPICMD="mpirun -np" # command to run an mpi executable, as $MPICMD <nprocs> <script>
156      fi
157
158      NB_CORE_PER_NODE_MAX=0  # Max number of cores per node (real cores, not hyperthreading - only for running simulations, cluster-specific)
159      N_HYPERTHREADING=1  # How many hyperthreading threads per physical core
160      RUNBASHCMD="bash" # command to run a bash job, as $runbashcmd (nthreads) <script> [nthreads only supplied if =/="bash"]
161      #SUBMITCMD="."  # command to sumbit a job, as $submitcmd <script>
162      submitcmd() {
163         nohup bash $1 > out.$$ 2>err.$$ &
164      }
165      ;;
166  esac
167}
168
169#-----------------------------------------------------------------------------------------------------
170function wget_pub() { # geting file from http:lmdz... and saving on $LMDZ_INIT
171#-----------------------------------------------------------------------------------------------------
172  local dir=$1
173  local file=$2
174  local target_dir=$LMDZ_INIT/$dir
175  if [ ! -f $target_dir/$file ] ; then
176     mkdir -p $target_dir
177     cd $target_dir
178     wget --no-check-certificate -nv "http://lmdz.lmd.jussieu.fr/pub/$dir/$file"
179     cd -
180  fi
181}
182
183#-----------------------------------------------------------------------------------------------------
184function cp_from_pub() { # geting file from http:lmdz... and saving on $LMDZ_INIT
185#-----------------------------------------------------------------------------------------------------
186  local dir=$1
187  local file=$2
188  cp -f $LMDZ_INIT/$dir/$file .
189}
190
191#-----------------------------------------------------------------------------------------------------
192function ln_from_pub() { # geting file from http:lmdz... and saving on $LMDZ_INIT
193#-----------------------------------------------------------------------------------------------------
194  local dir=$1
195  local file=$2
196  ln -sf $LMDZ_INIT/$dir/$file .
197}
198
199#-----------------------------------------------------------------------------------------------------
200function get_input_files() {
201#-----------------------------------------------------------------------------------------------------
202  local method=$1
203  local target=$2
204  case $target in
205
206      Orchidee) local files="PFTmap_IPCC_2000.nc cartepente2d_15min.nc routing.nc routing_simple.nc lai2D.nc \
207      alb_bg_modisopt_2D_ESA_v2.nc reftemp.nc \
208      soils_param.nc woodharvest_2000.nc PFTmap_15PFT.v1_2000.nc soil_bulk_and_ph.nc  \
209      ndep_nhx.nc ndep_noy.nc nfert_cropland.nc nfert_pasture.nc nmanure_cropland.nc nmanure_pasture.nc bnf.nc" ;;
210
211      AerChem) local files="aerosols1850_from_inca.nc aerosols9999_from_inca.nc" ;;
212
213      SPLA_WA/emissions) local files="donnees_lisa.nc SOILSPEC.data              \
214         cly.dat $( for i in $(seq -w 1 12 ) ; do echo dust$i.nc ; done ) wth.dat   \
215         carbon_emissions.nc sulphur_emissions_antro.nc sulphur_emissions_nat.nc \
216         sulphur_emissions_volc.nc" ;;
217
218      *) echo target $target non available in get_input_files ; exit 1 ;;
219  esac
220
221  if [[ $method != wget_pub && $method != ln_from_pub ]] ; then
222     echo method $method not available in get_input_files ; exit 1
223  fi
224  for file in $files ; do $method 3DInputData/$target $file ; done
225
226}
227
228get_hostname
229echo "Setting up lmdz_env on $hostname"
230set_env
231
232if [[ ! (-d $root_dir && -f $root_dir/.lmdz_setup_root_dir && -f $root_dir/lmdz_env.sh) ]]; then
233  echo "STOP: root_dir $root_dir not found, either you are running on an unsupported cluster, or the initialisation failed midway"; exit 1
234fi
Note: See TracBrowser for help on using the repository browser.