| 1 | #!/bin/bash |
|---|
| 2 | # Sets up tuning |
|---|
| 3 | # If sourced, will only load its functions, otherwise will run the setup |
|---|
| 4 | # Modif Najda 8/8/25 : add package seaborn to python environment |
|---|
| 5 | |
|---|
| 6 | set -exvu |
|---|
| 7 | unset http_proxy |
|---|
| 8 | unset https_proxy |
|---|
| 9 | |
|---|
| 10 | function check_known_issues () { |
|---|
| 11 | if [ -f ~/.Renviron ] ; then |
|---|
| 12 | echo ERROR in $0 |
|---|
| 13 | echo A file ~/.Renviron is present on your system |
|---|
| 14 | echo It is a standard file for managing R environments |
|---|
| 15 | echo It is however incompatible with htexplo installation |
|---|
| 16 | echo Remove it after checking it does not interfer with other uses of R |
|---|
| 17 | exit 1 |
|---|
| 18 | fi |
|---|
| 19 | } |
|---|
| 20 | |
|---|
| 21 | |
|---|
| 22 | function clean_setup() { |
|---|
| 23 | # A terme mettre le nom des machine dans les noms des renv_* pour pouvoir |
|---|
| 24 | # conserver plusieurs machines / versions en parallele ? |
|---|
| 25 | cd "$installdir" |
|---|
| 26 | rm -rf env.sh ExeterUQ_MOGP LES mogp_emulator renv_* venv_* |
|---|
| 27 | cd - > /dev/null |
|---|
| 28 | rm -rf WORK |
|---|
| 29 | } |
|---|
| 30 | |
|---|
| 31 | function load_options() { |
|---|
| 32 | # Read cmd line options and give default values |
|---|
| 33 | |
|---|
| 34 | # Special tricks to deactivate the git clone and so when the network |
|---|
| 35 | # is not available. Can be used only after the setup.sh has been run succesfully |
|---|
| 36 | # once |
|---|
| 37 | network="on" # on/off |
|---|
| 38 | |
|---|
| 39 | htune_dir=$(pwd) |
|---|
| 40 | |
|---|
| 41 | # command-line arguments |
|---|
| 42 | model="" |
|---|
| 43 | experience="" |
|---|
| 44 | installdir="../install_shared" |
|---|
| 45 | expdir="WORK" |
|---|
| 46 | python="python" |
|---|
| 47 | clean=0 |
|---|
| 48 | # Old (ExeterUQ) or new (ExeterUQ_MOGP) or newer (ExeterUQ_dgpsi) of Exter UQ programs |
|---|
| 49 | _exeter=_MOGP |
|---|
| 50 | |
|---|
| 51 | while (($# > 0)); do |
|---|
| 52 | case $1 in |
|---|
| 53 | -h|-help|--help) cat <<eof |
|---|
| 54 | $0 [ -model model_name ] [ -exp exp_name ] [-python python_cmd] [ -installdir install_dir ] |
|---|
| 55 | |
|---|
| 56 | -model model name (located in ./models) (default <$model>) [available: $(ls models | cat | xargs)] |
|---|
| 57 | -exp Experience name (default <$experience>) |
|---|
| 58 | -python Python command to use (default <$python>) |
|---|
| 59 | -installdir Where to install the tuning environment (default <$installdir>) |
|---|
| 60 | -expdir Where to run the experiments (default <$expdir>) |
|---|
| 61 | -exeter MOGP|dgpsi (default MOGP) |
|---|
| 62 | eof |
|---|
| 63 | exit 0 ;; |
|---|
| 64 | "-model") model=$2; shift; shift;; |
|---|
| 65 | "-exp") experience=$2; shift; shift;; |
|---|
| 66 | "-python") python=$2; shift; shift;; |
|---|
| 67 | "-installdir") installdir=$2; shift; shift;; |
|---|
| 68 | "-expdir") expdir=$2; shift; shift;; |
|---|
| 69 | "-exeter") _exeter=_$2; shift; shift;; |
|---|
| 70 | "clean") clean=1; shift;; |
|---|
| 71 | *) bash $0 -h; exit 1 |
|---|
| 72 | esac |
|---|
| 73 | done |
|---|
| 74 | |
|---|
| 75 | if ! $python -V ; then |
|---|
| 76 | echo "Switching $python->python3" |
|---|
| 77 | python="python3" |
|---|
| 78 | fi |
|---|
| 79 | |
|---|
| 80 | ExeterUQ="ExeterUQ"$_exeter |
|---|
| 81 | |
|---|
| 82 | mkdir -p "$installdir" |
|---|
| 83 | mkdir -p "$expdir" |
|---|
| 84 | installdir="$(realpath "$installdir")" |
|---|
| 85 | expdir="$(realpath "$expdir")" |
|---|
| 86 | |
|---|
| 87 | if [[ ! -d "models/$model" ]]; then |
|---|
| 88 | echo "STOP: $(pwd)/models/$model does not exists" |
|---|
| 89 | $0 -h; exit 1 |
|---|
| 90 | fi |
|---|
| 91 | |
|---|
| 92 | if [[ $experience = "" ]]; then experience="$model"; fi |
|---|
| 93 | |
|---|
| 94 | } |
|---|
| 95 | |
|---|
| 96 | function save_base_environment(){ |
|---|
| 97 | echo "set +u" > base_env.profile |
|---|
| 98 | env |grep PATH|sed "s,^,export ,g" >> base_env.profile |
|---|
| 99 | } |
|---|
| 100 | |
|---|
| 101 | function load_platform_environment() { |
|---|
| 102 | # platform-specific setup |
|---|
| 103 | local msg |
|---|
| 104 | |
|---|
| 105 | hostname=$(hostname) ; echo "hostname: $hostname" |
|---|
| 106 | case ${hostname:0:6} in |
|---|
| 107 | "jean-z") cat > env.sh <<eod |
|---|
| 108 | set +u # <module load> can trigger -u |
|---|
| 109 | module purge |
|---|
| 110 | compilo=19.0.4 # available 2013.0, 2017.2 |
|---|
| 111 | module load intel-compilers/\$compilo |
|---|
| 112 | module load intel-mpi/\$compilo |
|---|
| 113 | module load hdf5/1.10.5/intel-\$compilo-mpi |
|---|
| 114 | module load netcdf/4.7.0/intel-\$compilo-mpi |
|---|
| 115 | module load netcdf-fortran/4.4.5/intel-\$compilo-mpi |
|---|
| 116 | module load subversion/1.9.7/gcc-4.8.5 |
|---|
| 117 | module load intel-mkl/\$compilo |
|---|
| 118 | module load nco |
|---|
| 119 | module load cdo |
|---|
| 120 | module load ferret |
|---|
| 121 | module load r |
|---|
| 122 | module load python/2.7.16 |
|---|
| 123 | login=$(whoami) ; groupe=$(echo \$login | cut -c2-4) |
|---|
| 124 | # Inputation de la consommation sur le groupe \$groupe |
|---|
| 125 | # Peut se changer a la main : |
|---|
| 126 | # groupe=gzi |
|---|
| 127 | submit="sbatch -A \${groupe}@cpu " |
|---|
| 128 | run="srun --label -n " |
|---|
| 129 | SCRATCHD=\$SCRATCH |
|---|
| 130 | STORED=\$STORE |
|---|
| 131 | LMDZD=\$WORK |
|---|
| 132 | eod |
|---|
| 133 | ;; |
|---|
| 134 | |
|---|
| 135 | "spirit"|"ricard"|"ghio") cat > env.sh <<eod |
|---|
| 136 | set +u # <module load> can trigger -u |
|---|
| 137 | module purge |
|---|
| 138 | module load R/4.4.1 # also auto-loads python 3.11 |
|---|
| 139 | export PATH="/net/nfs/tools/u20/0.22/comp/linux-ubuntu20.04-x86_64/gcc/11.5.0/bin/:\$PATH" # gcc compatible w/ the R version |
|---|
| 140 | export MPLBACKEND='Agg' # non-interactive plots |
|---|
| 141 | export TAR=/usr/bin/tar # for R package install |
|---|
| 142 | eod |
|---|
| 143 | ;; |
|---|
| 144 | |
|---|
| 145 | "pxtrop") if [[ "$hostname" = "pxtropics11" ]] ; then |
|---|
| 146 | msg="export LD_LIBRARY_PATH=\$HOME/.local/lib/:\$LD_LIBRARY_PATH" |
|---|
| 147 | else |
|---|
| 148 | msg="# No special environment needed on this computer" |
|---|
| 149 | fi |
|---|
| 150 | echo "$msg" > env.sh |
|---|
| 151 | ;; |
|---|
| 152 | "beleno") cat > env.sh <<eod |
|---|
| 153 | # Using intel compiler for belenos |
|---|
| 154 | # Just miss pdfjam |
|---|
| 155 | set +u # <module load> can trigger -u |
|---|
| 156 | # module purge tries to unalias condam but the alias does not exist so it raises an error |
|---|
| 157 | # so I create a false condam alias so that it can be unaliased... |
|---|
| 158 | alias condam="ls" |
|---|
| 159 | module () |
|---|
| 160 | { |
|---|
| 161 | eval \`/usr/bin/modulecmd bash \$*\` |
|---|
| 162 | } |
|---|
| 163 | module purge > /dev/null 2>&1 |
|---|
| 164 | module load intel > /dev/null 2>&1 |
|---|
| 165 | module load nco > /dev/null 2>&1 |
|---|
| 166 | module load eccodes > /dev/null 2>&1 |
|---|
| 167 | module load cdo > /dev/null 2>&1 |
|---|
| 168 | module load python/3.7.6 > /dev/null 2>&1 |
|---|
| 169 | module load R/3.6.1 > /dev/null 2>&1 |
|---|
| 170 | export PATH="/opt/softs/anaconda3/bin:\$PATH" |
|---|
| 171 | export LD_LIBRARY_PATH=/opt/softs/anaconda3/lib/:\$LD_LIBRARY_PATH |
|---|
| 172 | export TAR=/usr/bin/tar # for R package install |
|---|
| 173 | eod |
|---|
| 174 | ;; |
|---|
| 175 | "oldbel") cat > env.sh <<eod |
|---|
| 176 | # env.sh used for belenos with gcc - to keep track for a while |
|---|
| 177 | # I also kept the export path to najda local dir on belenos !! |
|---|
| 178 | s config uses software R and pdfjam installed by villefranquen |
|---|
| 179 | # relevant paths are added to PATH and LD_LIBRARY_PATH |
|---|
| 180 | set +u # <module load> can trigger -u |
|---|
| 181 | # module purge tries to unalias condam but the alias does not exist so it raises an error |
|---|
| 182 | # so I create a false condam alias so that it can be unaliased... |
|---|
| 183 | alias condam="ls" |
|---|
| 184 | module purge |
|---|
| 185 | module load gcc/14.1.0 |
|---|
| 186 | module load openmpi/gnu/4.1.2.1 |
|---|
| 187 | module load python/3.10.12 |
|---|
| 188 | export PATH=/opt/softs/libraries/GCC_5.3.0/cdo-1.7.0/bin/:\$PATH |
|---|
| 189 | export LD_LIBRARY_PATH=/opt/softs/gcc/14.1.0/lib64/:/opt/softs/anaconda3/envs/Python310/lib:\$LD_LIBRARY_PATH |
|---|
| 190 | export PATH=/home/cnrm_other/ge/mrmc/villefranquen/.local/bin:\$PATH |
|---|
| 191 | export LD_LIBRARY_PATH=/home/cnrm_other/ge/mrmc/villefranquen/.local/lib:\$LD_LIBRARY_PATH |
|---|
| 192 | export MPLBACKEND='Agg' # non-interactive plots |
|---|
| 193 | export TAR=/usr/bin/tar # for R package install |
|---|
| 194 | eod |
|---|
| 195 | ;; |
|---|
| 196 | |
|---|
| 197 | *) cat > env.sh <<eod |
|---|
| 198 | # No special environment needed on this computer |
|---|
| 199 | eod |
|---|
| 200 | esac |
|---|
| 201 | |
|---|
| 202 | |
|---|
| 203 | # write the sed_i command |
|---|
| 204 | cat > sed_i<<eod |
|---|
| 205 | # define our own sed_i POSIX command to replace sed -i calls |
|---|
| 206 | set -e |
|---|
| 207 | finam="\${@: -1}" |
|---|
| 208 | tmpfi=\$finam.tmpsed |
|---|
| 209 | command="\${@:1:\$#-1}" |
|---|
| 210 | make_exe=0 |
|---|
| 211 | test -x \$finam && make_exe=1 |
|---|
| 212 | sed \$command \$finam > \$tmpfi |
|---|
| 213 | mv \$tmpfi \$finam |
|---|
| 214 | if [ \$make_exe -eq 1 ] ; then |
|---|
| 215 | chmod +x \$finam |
|---|
| 216 | fi |
|---|
| 217 | eod |
|---|
| 218 | |
|---|
| 219 | # store it in $installdir/bin and add this repo to the PATH |
|---|
| 220 | chmod +x sed_i |
|---|
| 221 | mkdir -p $installdir/bin |
|---|
| 222 | mv sed_i $installdir/bin |
|---|
| 223 | echo "set -u ; export PATH=$installdir/bin:\$PATH ; set +u" >> env.sh |
|---|
| 224 | |
|---|
| 225 | #chmod +x env.sh |
|---|
| 226 | source ./env.sh |
|---|
| 227 | } |
|---|
| 228 | |
|---|
| 229 | function install_python_local { |
|---|
| 230 | version=$1 |
|---|
| 231 | # Remove / from the string if running ./setup.sh |
|---|
| 232 | log=$( echo log_${0}.out$$ | sed -e 's:/::g' ) |
|---|
| 233 | here=$PWD |
|---|
| 234 | if [[ "$version" == "*"."*"."*" ]] ; then |
|---|
| 235 | long_version=$version |
|---|
| 236 | else |
|---|
| 237 | case $version in # get last bugfix of required major.minor version |
|---|
| 238 | 3.8*) long_version=3.8.20 ;; |
|---|
| 239 | 3.9*) long_version=3.9.23 ;; |
|---|
| 240 | 3.10*) long_version=3.10.18 ;; |
|---|
| 241 | *) echo error: will not install python version $version... ; exit 1 ;; |
|---|
| 242 | esac |
|---|
| 243 | fi |
|---|
| 244 | pydir=$installdir/python-$long_version |
|---|
| 245 | mkdir -p $installdir/python-$long_version |
|---|
| 246 | cd $pydir |
|---|
| 247 | echo LALALA >> OUT |
|---|
| 248 | echo $0 >> OUT |
|---|
| 249 | echo log $log >> OUT |
|---|
| 250 | wget https://www.python.org/ftp/python/$long_version/Python-$long_version.tgz > $log 2>&1 |
|---|
| 251 | tar xzf Python-$long_version.tgz |
|---|
| 252 | cd Python-$long_version |
|---|
| 253 | ./configure --prefix=$pydir --enable-shared >> $log 2>&1 |
|---|
| 254 | make >> $log 2>&1 |
|---|
| 255 | make install >> $log 2>&1 |
|---|
| 256 | cd $pydir |
|---|
| 257 | rm Python-$long_version.tgz |
|---|
| 258 | cd $here |
|---|
| 259 | echo $pydir/bin/python3 |
|---|
| 260 | } |
|---|
| 261 | |
|---|
| 262 | function load_programs() { |
|---|
| 263 | # Testing availability of various programs |
|---|
| 264 | |
|---|
| 265 | #----------------------------------------------------------------------- |
|---|
| 266 | # Testing availability of various programs |
|---|
| 267 | #----------------------------------------------------------------------- |
|---|
| 268 | if [[ ${hostname:0:6} = "beleno" ]] ; then listprog="R Rscript cdo ncks $python" ; else listprog="R Rscript cdo ncks $python" pdfjam ;fi |
|---|
| 269 | #for exe in R Rscript cdo ncks $python pdfjam ; do |
|---|
| 270 | for exe in $listprog ; do |
|---|
| 271 | if [[ $(which "$exe") = "" ]] ; then echo "STOP: Can't find program <$exe>" ; exit 1 ; fi |
|---|
| 272 | done |
|---|
| 273 | |
|---|
| 274 | # specific version of python is needed for dgpsi |
|---|
| 275 | if [[ $ExeterUQ == "ExeterUQ_dgpsi" ]] ; then |
|---|
| 276 | python_version=$($python -V | cut -c8- | cut -d. -f1-2) |
|---|
| 277 | case $python_version in |
|---|
| 278 | 3.9*|3.10*|3.11*|3.12*) echo ok, python version is compatible with dgpsi ;; |
|---|
| 279 | *) |
|---|
| 280 | echo python version is not compatible with dgpsi |
|---|
| 281 | if [ -f $installdir/python*/bin/python3 ] ; then |
|---|
| 282 | echo "found python in $installdir ; switch to $installdir/python*/bin/python3" |
|---|
| 283 | python=$installdir/python*/bin/python3 |
|---|
| 284 | else |
|---|
| 285 | echo "do you want to install python3.9? (y/n)" |
|---|
| 286 | read -r ans |
|---|
| 287 | if [[ "$ans" == "y" || "$ans" == "yes" || "$ans" == "Y" || "$ans" == "Yes" ]] ; then |
|---|
| 288 | python=$(install_python_local 3.9) |
|---|
| 289 | else |
|---|
| 290 | exit 1 |
|---|
| 291 | fi |
|---|
| 292 | fi |
|---|
| 293 | # set variables to use the correct version of python |
|---|
| 294 | python=$(realpath $python) |
|---|
| 295 | set +u |
|---|
| 296 | export LD_LIBRARY_PATH=$(realpath $installdir/python*/lib/):$LD_LIBRARY_PATH |
|---|
| 297 | ;; |
|---|
| 298 | esac |
|---|
| 299 | fi |
|---|
| 300 | } |
|---|
| 301 | |
|---|
| 302 | function load_python_and_R() { |
|---|
| 303 | # Ensures python and R environments are set up correctly |
|---|
| 304 | local python_version |
|---|
| 305 | python_version=$($python -V | cut -c8- | cut -d. -f1-2) |
|---|
| 306 | local venv="venv_$python_version" |
|---|
| 307 | |
|---|
| 308 | local R_version |
|---|
| 309 | R_version=$(R --version | head -n 1 | cut -c11- | cut -d" " -f1-1) |
|---|
| 310 | local renv="renv_$R_version" |
|---|
| 311 | |
|---|
| 312 | function load_python() { |
|---|
| 313 | #----------------------------------------------------------------------- |
|---|
| 314 | # Ensure that the same version of mogp or dgpsi tools will be used |
|---|
| 315 | # with this version of the HighTune project |
|---|
| 316 | #----------------------------------------------------------------------- |
|---|
| 317 | |
|---|
| 318 | # MOGP |
|---|
| 319 | #----------------------------------------------------------------------- |
|---|
| 320 | # This commit corresponds to the last one on devel |
|---|
| 321 | # Author: Eric Daub <45598892+edaub@users.noreply.github.com> |
|---|
| 322 | # Date: Wed Jun 3 18:18:33 2020 +0100 |
|---|
| 323 | # mogp_commit=8dd1b6f76b93c8b96540f240b0dc0d6739892d00 |
|---|
| 324 | # 19 fevrier 2021 : last commit on master |
|---|
| 325 | #----------------------------------------------------------------------- |
|---|
| 326 | local mogp_commit="0.5.0" |
|---|
| 327 | |
|---|
| 328 | # DGPSI |
|---|
| 329 | #----------------------------------------------------------------------- |
|---|
| 330 | # commit 5a072ea corresponds to dgpsi v2.5.0 on https://github.com/mingdeyu/DGP |
|---|
| 331 | # commit 46ffb81 corresponds to dgpsi v2.6.0 on https://github.com/mingdeyu/DGP |
|---|
| 332 | #----------------------------------------------------------------------- |
|---|
| 333 | local dgpsi_version="2.6.0" |
|---|
| 334 | |
|---|
| 335 | # Setup virtualenv |
|---|
| 336 | echo $python |
|---|
| 337 | python=$(which "$python") # can't do before, as we need to have loaded the platform modules |
|---|
| 338 | if [[ ! -d $venv ]]; then |
|---|
| 339 | $python -m venv "$venv" |
|---|
| 340 | fi |
|---|
| 341 | source "$venv/bin/activate" |
|---|
| 342 | echo "Loaded venv, python is $(python -V)" |
|---|
| 343 | |
|---|
| 344 | if [[ $network = "on" ]]; then |
|---|
| 345 | # Install python mogp |
|---|
| 346 | if [[ $ExeterUQ = "ExeterUQ_MOGP" ]]; then |
|---|
| 347 | # Install additional required packages |
|---|
| 348 | if [[ ${hostname:0:6} = "beleno" ]] ; then listprog="install patsy matplotlib netcdf4 pandas seaborn" ; else listprog="install patsy matplotlib netcdf4 pandas=2.3.1 seaborn" ;fi |
|---|
| 349 | pip install patsy matplotlib netcdf4 pandas seaborn |
|---|
| 350 | # Try pypi, otherwise git commit |
|---|
| 351 | if ! pip install "mogp-emulator==$mogp_commit"; then |
|---|
| 352 | pip install "git+https://github.com/alan-turing-institute/mogp_emulator@$mogp_commit" |
|---|
| 353 | fi |
|---|
| 354 | ln -sf "$venv/lib/python$python_version/site-packages/mogp_emulator" . # required by HighTune |
|---|
| 355 | |
|---|
| 356 | # Install python dgpsi |
|---|
| 357 | elif [[ $ExeterUQ == "ExeterUQ_dgpsi" ]] ; then |
|---|
| 358 | list_req_dgpsi_250="'numpy>=1.18.2' 'numba>=0.51.2' 'matplotlib>=3.2.1' 'tqdm>=4.50.2' 'scipy>=1.4.1' 'scikit-learn>=0.22.0' 'jupyter>=1.0.0' 'dill>=0.3.2 <=0.3.5.1' 'pathos==0.2.9' 'multiprocess==0.70.13' 'psutil>=5.8.0' 'cython>=0.29.30' 'pybind11>=2.10.0' 'pythran>=0.11.0' 'scikit-build>=0.15.0' 'tabulate>=0.8.7'" |
|---|
| 359 | list_req_dgpsi_260='"numpy>=1.18.2" "numba>=0.51.2" "matplotlib>=3.2.1" "tqdm>=4.50.2" "scipy>=1.4.1" "scikit-learn>=0.22.0" "jupyter>=1.0.0" "dill>=0.3.2" "pathos>=0.2.9" "multiprocess>=0.70.13" "psutil>=5.8.0" "tabulate>=0.8.7"' |
|---|
| 360 | for req in netcdf4 pandas seaborn |
|---|
| 361 | do pip install $req |
|---|
| 362 | done |
|---|
| 363 | pip install 'dgpsi=='$dgpsi_version |
|---|
| 364 | fi |
|---|
| 365 | fi |
|---|
| 366 | } |
|---|
| 367 | |
|---|
| 368 | function load_R() { |
|---|
| 369 | # Setup R env variables |
|---|
| 370 | |
|---|
| 371 | # Install R packages locally |
|---|
| 372 | mkdir -p "$renv" |
|---|
| 373 | |
|---|
| 374 | if [[ $network = "on" ]]; then |
|---|
| 375 | if [[ $ExeterUQ = "ExeterUQ_MOGP" ]]; then |
|---|
| 376 | test -u $R_version && R_version=$(R --version | head -n 1 | cut -c11- | cut -d" " -f1-1) |
|---|
| 377 | Rlibs_file="$htune_dir/src/Rlibs_${R_version}.txt" |
|---|
| 378 | if [ -f $Rlibs_file ] ; then # new way |
|---|
| 379 | if ! eval "$htune_dir/src/CheckInstallPackages_.sh" $Rlibs_file ; then |
|---|
| 380 | echo "STOP: Problem encountered when installing R packages, using specifications from file $Rlibs_file"; exit 1 |
|---|
| 381 | fi |
|---|
| 382 | else # old way |
|---|
| 383 | # (Amaury) Note: R's package management is terrible. In particular, there is no way to auto-resolve the depencies of a R package for "old" versions of R. |
|---|
| 384 | # This is particularly a pain when we want to support machines with different versions of R installed. |
|---|
| 385 | # The list and order of packages below is an attempt to circumvent that, but I hope we can find a cleaner way in the future.... |
|---|
| 386 | if ! eval "$htune_dir/src/CheckInstallPackages.sh" Rcpp lattice reticulate Matrix invgamma GenSA far dotCall64 spam fields lhs maps mco ncdf4 shape tensor withr loo MASS pracma mvtnorm flexclust tmvtnorm tictoc ars HI coda MfUSampler ; then |
|---|
| 387 | echo "STOP: Problem encountered when installing R packages"; exit 1 |
|---|
| 388 | fi |
|---|
| 389 | fi |
|---|
| 390 | elif [[ $ExeterUQ = "ExeterUQ_dgpsi" ]]; then |
|---|
| 391 | # le commit 1ce49d6 correspond à la version 2.5.0 sur https://github.com/mingdeyu/dgpsi-R |
|---|
| 392 | # le commit 2ae2e0a correspond à la version 2.6.0+bugfix sur https://github.com/mingdeyu/dgpsi-R |
|---|
| 393 | dgpsiR_commit=2ae2e0a |
|---|
| 394 | tinydancer_commit=d268375 |
|---|
| 395 | if ! eval "$htune_dir/src/CheckInstallPackages.sh" devtools shape mingdeyu/dgpsi-R:$dgpsiR_commit BayesExeter/tinydancer:$tinydancer_commit ; then |
|---|
| 396 | echo "STOP: Problem encountered when installing R packages for DGPSI version"; exit 1 |
|---|
| 397 | fi |
|---|
| 398 | else |
|---|
| 399 | # (Amaury) Note: not tested |
|---|
| 400 | # |
|---|
| 401 | # When using the original version of ExeterUQ, should install rstan. |
|---|
| 402 | # Touchy .. |
|---|
| 403 | if ! eval "$htune_dir/src/CheckInstallPackages.sh" Rcpp reticulate invgamma GenSA far fields lhs maps mco ncdf4 shape tensor withr loo MASS rstan; then |
|---|
| 404 | echo "STOP: Problem encountered when installing R packages"; exit 1 |
|---|
| 405 | fi |
|---|
| 406 | fi |
|---|
| 407 | fi |
|---|
| 408 | } |
|---|
| 409 | |
|---|
| 410 | load_python |
|---|
| 411 | #echo "set -u ; export R_LIBS=\"$(realpath "$renv"):\$R_LIBS\"; export R_LIBS_USER=\"$(realpath "$renv")\"; source \"$(pwd)/$venv/bin/activate\" ; set +u" >> env.sh |
|---|
| 412 | echo "test -u \$renv && export R_LIBS=\"$(realpath "$renv"):\$R_LIBS\"; export R_LIBS_USER=\"$(realpath "$renv")\"; source \"$(pwd)/$venv/bin/activate\"" >> env.sh |
|---|
| 413 | source env.sh |
|---|
| 414 | |
|---|
| 415 | load_R |
|---|
| 416 | } |
|---|
| 417 | |
|---|
| 418 | function load_exeterUQ() { |
|---|
| 419 | #----------------------------------------------------------------------- |
|---|
| 420 | # Insure the same version of ExeterUQ tools will be used |
|---|
| 421 | # with this version of the HighTune project |
|---|
| 422 | #----------------------------------------------------------------------- |
|---|
| 423 | # This commit corresponds to the last one on master |
|---|
| 424 | # Merge: 089036e e284f7e |
|---|
| 425 | # Author: vicvolodina93 <52710405+vicvolodina93@users.noreply.github.com> |
|---|
| 426 | # Date: Thu Jun 4 07:59:41 2020 +0100 |
|---|
| 427 | # exeter_commit=9e45cde952d75c10515646f412226f6072606b81 |
|---|
| 428 | # exeter_commit=ef50736999f1879f19759ca84655a326e7a6b74d |
|---|
| 429 | # 19/02/2021 : exeter_commit=1cdb5c5fedb266e196627336b23ba02cc1375c0a |
|---|
| 430 | # 23/02/2021 : exeter_commit=c67cacb47363389ca767fed5ba30d4a82fb13a2d |
|---|
| 431 | # 25/02/2021 : exeter_commit=7dd118fc0d3da1c3441be2712a6b09a1257e166a |
|---|
| 432 | # 26/02/2021 : exeter_commit=283afc55e10e40e147726827e75cbd3a7dde06e0 |
|---|
| 433 | # 01/03/2021 |
|---|
| 434 | #----------------------------------------------------------------------- |
|---|
| 435 | |
|---|
| 436 | exeter_commit=06b008a50b1598d44c9e94a679482ec0896d6cd0 |
|---|
| 437 | isnew=0 |
|---|
| 438 | if [[ ! -d $ExeterUQ ]] ; then |
|---|
| 439 | git clone "https://github.com/BayesExeter/$ExeterUQ" |
|---|
| 440 | isnew=1 |
|---|
| 441 | fi |
|---|
| 442 | cd "$ExeterUQ" |
|---|
| 443 | if [[ $ExeterUQ = ExeterUQ_MOGP ]] ; then |
|---|
| 444 | set +e |
|---|
| 445 | command -v git > /dev/null 2>&1 |
|---|
| 446 | if [ $? -eq 0 ] ; then |
|---|
| 447 | git checkout $exeter_commit |
|---|
| 448 | else |
|---|
| 449 | echo "warning: git command not available, could not checkout commit $exeter_commit in $ExeterUQ" |
|---|
| 450 | fi |
|---|
| 451 | set -e |
|---|
| 452 | fi |
|---|
| 453 | |
|---|
| 454 | ### Bricolage momentané |
|---|
| 455 | # patch pour que ça marche en 2D (2 paramètres) (Naj) |
|---|
| 456 | if [[ $isnew -eq 1 ]] ; then |
|---|
| 457 | patch HistoryMatching/HistoryMatching.R "$htune_dir/src/patchs/patch_HistoryMatching" |
|---|
| 458 | fi |
|---|
| 459 | cd .. |
|---|
| 460 | } |
|---|
| 461 | |
|---|
| 462 | function install_model() { |
|---|
| 463 | echo '======================================================================' |
|---|
| 464 | echo ' 3. CONFIGURING THE SPECIFIC EXPERIMENT ' |
|---|
| 465 | echo '======================================================================' |
|---|
| 466 | |
|---|
| 467 | if [[ $model = "" ]] ; then # no model => we setup the very basic example, and run wave 1 |
|---|
| 468 | mkdir -p "$expdir/EXEMPLE/" |
|---|
| 469 | if [[ $ExeterUQ == "ExeterUQ_MOGP" ]] ; then exemple=exemple.sh_mogp ; else exemple=exemple.sh_dgpsi ; fi |
|---|
| 470 | cp "$htune_dir/src/$exemple" "$expdir/EXEMPLE/exemple.sh" |
|---|
| 471 | cd "$expdir/EXEMPLE" |
|---|
| 472 | chmod +x exemple.sh |
|---|
| 473 | bash exemple.sh |
|---|
| 474 | echo "CHECK RESULTS IN $expdir/EXEMPLE/" |
|---|
| 475 | exit 0 |
|---|
| 476 | fi |
|---|
| 477 | |
|---|
| 478 | # Copy model-specific files locally |
|---|
| 479 | rm -rf tmp |
|---|
| 480 | mkdir -p tmp/models # put in tmp/ because setup_$model.sh installs in ../. |
|---|
| 481 | cp -rf "$htune_dir/models/$model" tmp/models |
|---|
| 482 | if [[ $model = "LMDZ" ]]; then |
|---|
| 483 | # set ecrad path for LMDZ |
|---|
| 484 | set -xv |
|---|
| 485 | sed_i -e "s:ecrad_dir=.*.$:ecrad_dir=$(realpath "$htune_dir/models/ECRAD"):" "tmp/models/LMDZ/serie_LMDZ.sh" |
|---|
| 486 | set +xv |
|---|
| 487 | fi |
|---|
| 488 | |
|---|
| 489 | echo '-----------------------------------------------------------------------' |
|---|
| 490 | echo ' Possibility to have a setup_* specific of a given model' |
|---|
| 491 | echo '-----------------------------------------------------------------------' |
|---|
| 492 | if [[ -f "tmp/models/$model/setup_$model.sh" ]]; then |
|---|
| 493 | cd tmp; "$htune_dir/models/$model/setup_$model.sh"; cd - |
|---|
| 494 | fi |
|---|
| 495 | mkdir -p "$expdir/$experience" |
|---|
| 496 | cp -rf "tmp/models/$model/"* "$expdir/$experience/" |
|---|
| 497 | rm -rf tmp |
|---|
| 498 | } |
|---|
| 499 | |
|---|
| 500 | function download_experiment_data() { |
|---|
| 501 | echo '-----------------------------------------------------------------------' |
|---|
| 502 | echo ' Downloading the data relevant to the current experiment' |
|---|
| 503 | echo '-----------------------------------------------------------------------' |
|---|
| 504 | |
|---|
| 505 | if [[ $model = "ECRAD" ]] ; then |
|---|
| 506 | |
|---|
| 507 | if [[ ! -d LES1D_ecRad ]] ; then |
|---|
| 508 | echo "Downloading LES/MC results for radiation" |
|---|
| 509 | #wget http://simu:visu2018@www.umr-cnrm.fr/visu-high-tune/data_tuningtool/LES1D_ecRad.tar |
|---|
| 510 | # Naj: temporary change |
|---|
| 511 | wget https://web.lmd.jussieu.fr/~nvillefranque/pub/data/LES1D_ecRad.tar |
|---|
| 512 | tar xvf LES1D_ecRad.tar |
|---|
| 513 | rm -rf LES1D_ecRad.tar |
|---|
| 514 | fi |
|---|
| 515 | |
|---|
| 516 | if [[ ! -d RAD ]] ; then |
|---|
| 517 | echo "Downloading LES/MC results for radiation" |
|---|
| 518 | #wget http://simu:visu2018@www.umr-cnrm.fr/visu-high-tune/data_tuningtool/RAD.tar |
|---|
| 519 | # Naj: temporary change |
|---|
| 520 | wget https://web.lmd.jussieu.fr/~nvillefranque/pub/data/RAD.tar |
|---|
| 521 | tar xvf RAD.tar |
|---|
| 522 | rm -rf RAD.tar |
|---|
| 523 | fi |
|---|
| 524 | |
|---|
| 525 | else # of [[ $model = "ECRAD" ]] |
|---|
| 526 | |
|---|
| 527 | if [[ ! -d LES ]] ; then |
|---|
| 528 | echo "Downloading LES results from CNRM and LMD" |
|---|
| 529 | #wget http://simu:visu2018@www.umr-cnrm.fr/visu-high-tune/data_tuningtool/les.tar |
|---|
| 530 | wget https://web.lmd.jussieu.fr/~nvillefranque/pub/data/les.tar |
|---|
| 531 | if [[ ! -f les.tar ]] ; then echo "Can not download les.tar" ; exit 1 ; fi |
|---|
| 532 | tar xvf les.tar |
|---|
| 533 | rm -f les.tar |
|---|
| 534 | #----------------------------------------------------------------------- |
|---|
| 535 | # Bricolage avec le cas RCE oceanique. A reprendre |
|---|
| 536 | # cd LES/RCE_OCE/ ; mv REF DAILY ; mkdir -p REF |
|---|
| 537 | # for i in 0 1 ; do |
|---|
| 538 | # cdo daymean DAILY/LES$i.nc REF/LES$i.nc |
|---|
| 539 | # ncks -v zf DAILY/LES$i.nc -A REF/LES$i.nc |
|---|
| 540 | # ncrename -d z,levf REF/LES$i.nc -O |
|---|
| 541 | # # On prend le calendrier du cas LMDZ1D en attendant mieux. |
|---|
| 542 | # # Avec le nouveau format, devrait permettre d'etre propre |
|---|
| 543 | # # Tous les cas idealises au 1er janvier 2000 ? |
|---|
| 544 | # ncatted -a "units,time,o,c,days since 1997-11-01 00:00:00" REF/LES$i.nc -O |
|---|
| 545 | # done |
|---|
| 546 | # cd - |
|---|
| 547 | |
|---|
| 548 | # on supprime le repertoire gabls4 provenant de la base de donnee CNRM-LMD et on le |
|---|
| 549 | # remplace par celui du leslmd.tar (avec le bon calendrier) |
|---|
| 550 | rm -rf LES/GABLS4 |
|---|
| 551 | wget --no-check-certificate http://www.lmd.jussieu.fr/~lmdz/HighTune/leslmd.tar |
|---|
| 552 | if [ ! -f leslmd.tar ] ; then echo Can not download leslmd.tar ; exit 1 ; fi |
|---|
| 553 | tar xvf leslmd.tar |
|---|
| 554 | rm -f leslmd.tar |
|---|
| 555 | |
|---|
| 556 | # Convective files pre processed by Louis d'Alençon en Lamine Thiam |
|---|
| 557 | cd LES |
|---|
| 558 | \rm -rf RCE_OCE AMMA CINDY |
|---|
| 559 | wget http://www.lmd.jussieu.fr/~hourdin/git_d/data/AdditionalLES/les.tar |
|---|
| 560 | tar xvf les.tar |
|---|
| 561 | \rm -f les.tar |
|---|
| 562 | cd - |
|---|
| 563 | |
|---|
| 564 | #----------------------------------------------------------------------- |
|---|
| 565 | |
|---|
| 566 | fi # of [[ ! -d LES ]] |
|---|
| 567 | fi # of [[ $model = "ECRAD" ]] |
|---|
| 568 | } |
|---|
| 569 | |
|---|
| 570 | function create_working_dir() { |
|---|
| 571 | echo '-----------------------------------------------------------------------' |
|---|
| 572 | echo "Creating and installing the working directory $expdir/$experience" |
|---|
| 573 | echo '-----------------------------------------------------------------------' |
|---|
| 574 | |
|---|
| 575 | local DIR0 |
|---|
| 576 | DIR0=$(pwd) |
|---|
| 577 | |
|---|
| 578 | mkdir -p "$expdir/$experience/log" |
|---|
| 579 | cp -f "$htune_dir/src/"*.py "$htune_dir/src/"*.sh "$htune_dir/src/"*.R "$expdir/$experience/" |
|---|
| 580 | |
|---|
| 581 | cd "$expdir/$experience" |
|---|
| 582 | if [[ "$model" = "ECRAD" ]] ; then |
|---|
| 583 | ln -sf "$DIR0/RAD" . |
|---|
| 584 | ln -sf "$DIR0/LES1D_ecRad" . |
|---|
| 585 | else |
|---|
| 586 | ln -sf "$DIR0/LES" . |
|---|
| 587 | fi |
|---|
| 588 | # ln -s $DIR0/$ExeterUQ/BuildEmulator . |
|---|
| 589 | |
|---|
| 590 | ### Bricolage momentané |
|---|
| 591 | # patch while waiting daniel to do the comit |
|---|
| 592 | # on ExetreUQ package |
|---|
| 593 | cp -r "$DIR0/$ExeterUQ/BuildEmulator" . |
|---|
| 594 | mv BuildEmulator_tmp.R BuildEmulator/BuildEmulator.R |
|---|
| 595 | |
|---|
| 596 | if [[ ! -d HistoryMatching ]] ; then |
|---|
| 597 | ln -s "$DIR0/$ExeterUQ/HistoryMatching" . |
|---|
| 598 | fi |
|---|
| 599 | if [[ ! -d mogp_emulator ]] ; then |
|---|
| 600 | ln -s "$DIR0/mogp_emulator" . |
|---|
| 601 | fi |
|---|
| 602 | |
|---|
| 603 | cp "$DIR0/env.sh" . |
|---|
| 604 | } |
|---|
| 605 | |
|---|
| 606 | # If sourced: returns, else run setup |
|---|
| 607 | if [[ ! "${BASH_SOURCE[0]}" = "$0" ]]; then return 0; fi |
|---|
| 608 | |
|---|
| 609 | check_known_issues |
|---|
| 610 | load_options "$@" |
|---|
| 611 | if [[ $clean = 1 ]] ; then clean_setup ; exit 0 ; fi |
|---|
| 612 | cd "$installdir" |
|---|
| 613 | save_base_environment |
|---|
| 614 | load_platform_environment |
|---|
| 615 | load_programs |
|---|
| 616 | load_python_and_R |
|---|
| 617 | if [[ $ExeterUQ == "ExeterUQ_MOGP" ]] ; then load_exeterUQ ; fi |
|---|
| 618 | install_model |
|---|
| 619 | download_experiment_data |
|---|
| 620 | create_working_dir |
|---|
| 621 | |
|---|
| 622 | echo "End of $0" |
|---|
