| 1 | #!/bin/bash |
|---|
| 2 | # Sets up tuning |
|---|
| 3 | # If sourced, will only load its functions, otherwise will run the setup |
|---|
| 4 | # Modif Najda 8/8/25 : add package seaborn to python environment |
|---|
| 5 | |
|---|
| 6 | set -exvu |
|---|
| 7 | unset http_proxy |
|---|
| 8 | unset https_proxy |
|---|
| 9 | |
|---|
| 10 | function check_known_issues () { |
|---|
| 11 | if [ -f ~/.Renviron ] ; then |
|---|
| 12 | echo ERROR in $0 |
|---|
| 13 | echo A file ~/.Renviron is present on your system |
|---|
| 14 | echo It is a standard file for managing R environments |
|---|
| 15 | echo It is however incompatible with htexplo installation |
|---|
| 16 | echo Remove it after checking it does not interfer with other uses of R |
|---|
| 17 | exit 1 |
|---|
| 18 | fi |
|---|
| 19 | } |
|---|
| 20 | |
|---|
| 21 | |
|---|
| 22 | function clean_setup() { |
|---|
| 23 | # A terme mettre le nom des machine dans les noms des renv_* pour pouvoir |
|---|
| 24 | # conserver plusieurs machines / versions en parallele ? |
|---|
| 25 | cd "$installdir" |
|---|
| 26 | rm -rf env.sh ExeterUQ_MOGP LES mogp_emulator renv_* venv_* |
|---|
| 27 | cd - > /dev/null |
|---|
| 28 | rm -rf WORK |
|---|
| 29 | } |
|---|
| 30 | |
|---|
| 31 | function load_options() { |
|---|
| 32 | # Read cmd line options and give default values |
|---|
| 33 | |
|---|
| 34 | # Special tricks to deactivate the git clone and so when the network |
|---|
| 35 | # is not available. Can be used only after the setup.sh has been run succesfully |
|---|
| 36 | # once |
|---|
| 37 | network="on" # on/off |
|---|
| 38 | |
|---|
| 39 | htune_dir=$(pwd) |
|---|
| 40 | |
|---|
| 41 | # command-line arguments |
|---|
| 42 | model="" |
|---|
| 43 | experience="" |
|---|
| 44 | installdir="../install_shared" |
|---|
| 45 | expdir="WORK" |
|---|
| 46 | python="python" |
|---|
| 47 | clean=0 |
|---|
| 48 | # Old (ExeterUQ) or new (ExeterUQ_MOGP) or newer (ExeterUQ_dgpsi) of Exter UQ programs |
|---|
| 49 | _exeter=_MOGP |
|---|
| 50 | |
|---|
| 51 | while (($# > 0)); do |
|---|
| 52 | case $1 in |
|---|
| 53 | -h|-help|--help) cat <<eof |
|---|
| 54 | $0 [ -model model_name ] [ -exp exp_name ] [-python python_cmd] [ -installdir install_dir ] |
|---|
| 55 | |
|---|
| 56 | -model model name (located in ./models) (default <$model>) [available: $(ls models | cat | xargs)] |
|---|
| 57 | -exp Experience name (default <$experience>) |
|---|
| 58 | -python Python command to use (default <$python>) |
|---|
| 59 | -installdir Where to install the tuning environment (default <$installdir>) |
|---|
| 60 | -expdir Where to run the experiments (default <$expdir>) |
|---|
| 61 | -exeter MOGP|dgpsi (default MOGP) |
|---|
| 62 | eof |
|---|
| 63 | exit 0 ;; |
|---|
| 64 | "-model") model=$2; shift; shift;; |
|---|
| 65 | "-exp") experience=$2; shift; shift;; |
|---|
| 66 | "-python") python=$2; shift; shift;; |
|---|
| 67 | "-installdir") installdir=$2; shift; shift;; |
|---|
| 68 | "-expdir") expdir=$2; shift; shift;; |
|---|
| 69 | "-exeter") _exeter=_$2; shift; shift;; |
|---|
| 70 | "clean") clean=1; shift;; |
|---|
| 71 | *) bash $0 -h; exit 1 |
|---|
| 72 | esac |
|---|
| 73 | done |
|---|
| 74 | |
|---|
| 75 | if ! $python -V ; then |
|---|
| 76 | echo "Switching $python->python3" |
|---|
| 77 | python="python3" |
|---|
| 78 | fi |
|---|
| 79 | |
|---|
| 80 | ExeterUQ="ExeterUQ"$_exeter |
|---|
| 81 | |
|---|
| 82 | mkdir -p "$installdir" |
|---|
| 83 | mkdir -p "$expdir" |
|---|
| 84 | installdir="$(realpath "$installdir")" |
|---|
| 85 | expdir="$(realpath "$expdir")" |
|---|
| 86 | |
|---|
| 87 | if [[ ! -d "models/$model" ]]; then |
|---|
| 88 | echo "STOP: $(pwd)/models/$model does not exists" |
|---|
| 89 | $0 -h; exit 1 |
|---|
| 90 | fi |
|---|
| 91 | |
|---|
| 92 | if [[ $experience = "" ]]; then experience="$model"; fi |
|---|
| 93 | |
|---|
| 94 | } |
|---|
| 95 | |
|---|
| 96 | function save_base_environment(){ |
|---|
| 97 | echo "set +u" > base_env.profile |
|---|
| 98 | env |grep PATH|sed "s,^,export ,g" >> base_env.profile |
|---|
| 99 | } |
|---|
| 100 | |
|---|
| 101 | function load_platform_environment() { |
|---|
| 102 | # platform-specific setup |
|---|
| 103 | local msg |
|---|
| 104 | |
|---|
| 105 | hostname=$(hostname) ; echo "hostname: $hostname" |
|---|
| 106 | case ${hostname:0:6} in |
|---|
| 107 | "jean-z") cat > env.sh <<eod |
|---|
| 108 | set +u # <module load> can trigger -u |
|---|
| 109 | module purge |
|---|
| 110 | compilo=19.0.4 # available 2013.0, 2017.2 |
|---|
| 111 | module load intel-compilers/\$compilo |
|---|
| 112 | module load intel-mpi/\$compilo |
|---|
| 113 | module load hdf5/1.10.5/intel-\$compilo-mpi |
|---|
| 114 | module load netcdf/4.7.0/intel-\$compilo-mpi |
|---|
| 115 | module load netcdf-fortran/4.4.5/intel-\$compilo-mpi |
|---|
| 116 | module load subversion/1.9.7/gcc-4.8.5 |
|---|
| 117 | module load intel-mkl/\$compilo |
|---|
| 118 | module load nco |
|---|
| 119 | module load cdo |
|---|
| 120 | module load ferret |
|---|
| 121 | module load r |
|---|
| 122 | module load python/2.7.16 |
|---|
| 123 | login=$(whoami) ; groupe=$(echo \$login | cut -c2-4) |
|---|
| 124 | # Inputation de la consommation sur le groupe \$groupe |
|---|
| 125 | # Peut se changer a la main : |
|---|
| 126 | # groupe=gzi |
|---|
| 127 | submit="sbatch -A \${groupe}@cpu " |
|---|
| 128 | run="srun --label -n " |
|---|
| 129 | SCRATCHD=\$SCRATCH |
|---|
| 130 | STORED=\$STORE |
|---|
| 131 | LMDZD=\$WORK |
|---|
| 132 | eod |
|---|
| 133 | ;; |
|---|
| 134 | |
|---|
| 135 | "spirit"|"ricard"|"ghio") cat > env.sh <<eod |
|---|
| 136 | set +u # <module load> can trigger -u |
|---|
| 137 | module purge |
|---|
| 138 | module load R/4.4.1 # also auto-loads python 3.11 |
|---|
| 139 | export PATH="/net/nfs/tools/u20/0.22/comp/linux-ubuntu20.04-x86_64/gcc/11.5.0/bin/:\$PATH" # gcc compatible w/ the R version |
|---|
| 140 | export MPLBACKEND='Agg' # non-interactive plots |
|---|
| 141 | export TAR=/usr/bin/tar # for R package install |
|---|
| 142 | eod |
|---|
| 143 | ;; |
|---|
| 144 | |
|---|
| 145 | "pxtrop") if [[ "$hostname" = "pxtropics11" ]] ; then |
|---|
| 146 | msg="export LD_LIBRARY_PATH=\$HOME/.local/lib/:\$LD_LIBRARY_PATH" |
|---|
| 147 | else |
|---|
| 148 | msg="# No special environment needed on this computer" |
|---|
| 149 | fi |
|---|
| 150 | echo "$msg" > env.sh |
|---|
| 151 | ;; |
|---|
| 152 | "beleno") cat > env.sh <<eod |
|---|
| 153 | # this config uses software R and pdfjam installed by villefranquen |
|---|
| 154 | # relevant paths are added to PATH and LD_LIBRARY_PATH |
|---|
| 155 | set +u # <module load> can trigger -u |
|---|
| 156 | # module purge tries to unalias condam but the alias does not exist so it raises an error |
|---|
| 157 | # so I create a false condam alias so that it can be unaliased... |
|---|
| 158 | alias condam="ls" |
|---|
| 159 | module purge |
|---|
| 160 | module load gcc/14.1.0 |
|---|
| 161 | module load openmpi/gnu/4.1.2.1 |
|---|
| 162 | module load python/3.10.12 |
|---|
| 163 | export PATH=/opt/softs/libraries/GCC_5.3.0/cdo-1.7.0/bin/:\$PATH |
|---|
| 164 | export LD_LIBRARY_PATH=/opt/softs/gcc/14.1.0/lib64/:/opt/softs/anaconda3/envs/Python310/lib:\$LD_LIBRARY_PATH |
|---|
| 165 | export MPLBACKEND='Agg' # non-interactive plots |
|---|
| 166 | export TAR=/usr/bin/tar # for R package install |
|---|
| 167 | eod |
|---|
| 168 | ;; |
|---|
| 169 | *) cat > env.sh <<eod |
|---|
| 170 | # No special environment needed on this computer |
|---|
| 171 | eod |
|---|
| 172 | esac |
|---|
| 173 | |
|---|
| 174 | |
|---|
| 175 | # write the sed_i command |
|---|
| 176 | cat > sed_i<<eod |
|---|
| 177 | # define our own sed_i POSIX command to replace sed -i calls |
|---|
| 178 | set -e |
|---|
| 179 | finam="\${@: -1}" |
|---|
| 180 | tmpfi=\$finam.tmpsed |
|---|
| 181 | command="\${@:1:\$#-1}" |
|---|
| 182 | make_exe=0 |
|---|
| 183 | test -x \$finam && make_exe=1 |
|---|
| 184 | sed \$command \$finam > \$tmpfi |
|---|
| 185 | mv \$tmpfi \$finam |
|---|
| 186 | if [ \$make_exe -eq 1 ] ; then |
|---|
| 187 | chmod +x \$finam |
|---|
| 188 | fi |
|---|
| 189 | eod |
|---|
| 190 | |
|---|
| 191 | # store it in $installdir/bin and add this repo to the PATH |
|---|
| 192 | chmod +x sed_i |
|---|
| 193 | mkdir -p $installdir/bin |
|---|
| 194 | mv sed_i $installdir/bin |
|---|
| 195 | echo "set -u ; export PATH=$installdir/bin:\$PATH ; set +u" >> env.sh |
|---|
| 196 | |
|---|
| 197 | #chmod +x env.sh |
|---|
| 198 | source ./env.sh |
|---|
| 199 | } |
|---|
| 200 | |
|---|
| 201 | function install_python_local { |
|---|
| 202 | version=$1 |
|---|
| 203 | # Remove / from the string if running ./setup.sh |
|---|
| 204 | log=$( echo log_${0}.out$$ | sed -e 's:/::g' ) |
|---|
| 205 | here=$PWD |
|---|
| 206 | if [[ "$version" == "*"."*"."*" ]] ; then |
|---|
| 207 | long_version=$version |
|---|
| 208 | else |
|---|
| 209 | case $version in # get last bugfix of required major.minor version |
|---|
| 210 | 3.8*) long_version=3.8.20 ;; |
|---|
| 211 | 3.9*) long_version=3.9.23 ;; |
|---|
| 212 | 3.10*) long_version=3.10.18 ;; |
|---|
| 213 | *) echo error: will not install python version $version... ; exit 1 ;; |
|---|
| 214 | esac |
|---|
| 215 | fi |
|---|
| 216 | pydir=$installdir/python-$long_version |
|---|
| 217 | mkdir -p $installdir/python-$long_version |
|---|
| 218 | cd $pydir |
|---|
| 219 | echo LALALA >> OUT |
|---|
| 220 | echo $0 >> OUT |
|---|
| 221 | echo log $log >> OUT |
|---|
| 222 | wget https://www.python.org/ftp/python/$long_version/Python-$long_version.tgz > $log 2>&1 |
|---|
| 223 | tar xzf Python-$long_version.tgz |
|---|
| 224 | cd Python-$long_version |
|---|
| 225 | ./configure --prefix=$pydir --enable-shared >> $log 2>&1 |
|---|
| 226 | make >> $log 2>&1 |
|---|
| 227 | make install >> $log 2>&1 |
|---|
| 228 | cd $pydir |
|---|
| 229 | rm Python-$long_version.tgz |
|---|
| 230 | cd $here |
|---|
| 231 | echo $pydir/bin/python3 |
|---|
| 232 | } |
|---|
| 233 | |
|---|
| 234 | function load_programs() { |
|---|
| 235 | # Testing availability of various programs |
|---|
| 236 | |
|---|
| 237 | #----------------------------------------------------------------------- |
|---|
| 238 | # Testing availability of various programs |
|---|
| 239 | #----------------------------------------------------------------------- |
|---|
| 240 | for exe in R Rscript cdo ncks $python pdfjam ; do |
|---|
| 241 | if [[ $(which "$exe") = "" ]] ; then echo "STOP: Can't find program <$exe>" ; exit 1 ; fi |
|---|
| 242 | done |
|---|
| 243 | |
|---|
| 244 | # specific version of python is needed for dgpsi |
|---|
| 245 | if [[ $ExeterUQ == "ExeterUQ_dgpsi" ]] ; then |
|---|
| 246 | python_version=$($python -V | cut -c8- | cut -d. -f1-2) |
|---|
| 247 | case $python_version in |
|---|
| 248 | 3.9*|3.10*|3.11*|3.12*) echo ok, python version is compatible with dgpsi ;; |
|---|
| 249 | *) |
|---|
| 250 | echo python version is not compatible with dgpsi |
|---|
| 251 | if [ -f $installdir/python*/bin/python3 ] ; then |
|---|
| 252 | echo "found python in $installdir ; switch to $installdir/python*/bin/python3" |
|---|
| 253 | python=$installdir/python*/bin/python3 |
|---|
| 254 | else |
|---|
| 255 | echo "do you want to install python3.9? (y/n)" |
|---|
| 256 | read -r ans |
|---|
| 257 | if [[ "$ans" == "y" || "$ans" == "yes" || "$ans" == "Y" || "$ans" == "Yes" ]] ; then |
|---|
| 258 | python=$(install_python_local 3.9) |
|---|
| 259 | else |
|---|
| 260 | exit 1 |
|---|
| 261 | fi |
|---|
| 262 | fi |
|---|
| 263 | # set variables to use the correct version of python |
|---|
| 264 | python=$(realpath $python) |
|---|
| 265 | set +u |
|---|
| 266 | export LD_LIBRARY_PATH=$(realpath $installdir/python*/lib/):$LD_LIBRARY_PATH |
|---|
| 267 | ;; |
|---|
| 268 | esac |
|---|
| 269 | fi |
|---|
| 270 | } |
|---|
| 271 | |
|---|
| 272 | function load_python_and_R() { |
|---|
| 273 | # Ensures python and R environments are set up correctly |
|---|
| 274 | local python_version |
|---|
| 275 | python_version=$($python -V | cut -c8- | cut -d. -f1-2) |
|---|
| 276 | local venv="venv_$python_version" |
|---|
| 277 | |
|---|
| 278 | local R_version |
|---|
| 279 | R_version=$(R --version | head -n 1 | cut -c11- | cut -d" " -f1-1) |
|---|
| 280 | local renv="renv_$R_version" |
|---|
| 281 | |
|---|
| 282 | function load_python() { |
|---|
| 283 | #----------------------------------------------------------------------- |
|---|
| 284 | # Ensure that the same version of mogp or dgpsi tools will be used |
|---|
| 285 | # with this version of the HighTune project |
|---|
| 286 | #----------------------------------------------------------------------- |
|---|
| 287 | |
|---|
| 288 | # MOGP |
|---|
| 289 | #----------------------------------------------------------------------- |
|---|
| 290 | # This commit corresponds to the last one on devel |
|---|
| 291 | # Author: Eric Daub <45598892+edaub@users.noreply.github.com> |
|---|
| 292 | # Date: Wed Jun 3 18:18:33 2020 +0100 |
|---|
| 293 | # mogp_commit=8dd1b6f76b93c8b96540f240b0dc0d6739892d00 |
|---|
| 294 | # 19 fevrier 2021 : last commit on master |
|---|
| 295 | #----------------------------------------------------------------------- |
|---|
| 296 | local mogp_commit="0.5.0" |
|---|
| 297 | |
|---|
| 298 | # DGPSI |
|---|
| 299 | #----------------------------------------------------------------------- |
|---|
| 300 | # commit 5a072ea corresponds to dgpsi v2.5.0 on https://github.com/mingdeyu/DGP |
|---|
| 301 | # commit 46ffb81 corresponds to dgpsi v2.6.0 on https://github.com/mingdeyu/DGP |
|---|
| 302 | #----------------------------------------------------------------------- |
|---|
| 303 | local dgpsi_version="2.6.0" |
|---|
| 304 | |
|---|
| 305 | # Setup virtualenv |
|---|
| 306 | echo $python |
|---|
| 307 | python=$(which "$python") # can't do before, as we need to have loaded the platform modules |
|---|
| 308 | if [[ ! -d $venv ]]; then |
|---|
| 309 | $python -m venv "$venv" |
|---|
| 310 | fi |
|---|
| 311 | source "$venv/bin/activate" |
|---|
| 312 | echo "Loaded venv, python is $(python -V)" |
|---|
| 313 | |
|---|
| 314 | if [[ $network = "on" ]]; then |
|---|
| 315 | # Install python mogp |
|---|
| 316 | if [[ $ExeterUQ = "ExeterUQ_MOGP" ]]; then |
|---|
| 317 | # Install additional required packages |
|---|
| 318 | pip install patsy matplotlib netcdf4 pandas seaborn |
|---|
| 319 | # Try pypi, otherwise git commit |
|---|
| 320 | if ! pip install "mogp-emulator==$mogp_commit"; then |
|---|
| 321 | pip install "git+https://github.com/alan-turing-institute/mogp_emulator@$mogp_commit" |
|---|
| 322 | fi |
|---|
| 323 | ln -sf "$venv/lib/python$python_version/site-packages/mogp_emulator" . # required by HighTune |
|---|
| 324 | |
|---|
| 325 | # Install python dgpsi |
|---|
| 326 | elif [[ $ExeterUQ == "ExeterUQ_dgpsi" ]] ; then |
|---|
| 327 | list_req_dgpsi_250="'numpy>=1.18.2' 'numba>=0.51.2' 'matplotlib>=3.2.1' 'tqdm>=4.50.2' 'scipy>=1.4.1' 'scikit-learn>=0.22.0' 'jupyter>=1.0.0' 'dill>=0.3.2 <=0.3.5.1' 'pathos==0.2.9' 'multiprocess==0.70.13' 'psutil>=5.8.0' 'cython>=0.29.30' 'pybind11>=2.10.0' 'pythran>=0.11.0' 'scikit-build>=0.15.0' 'tabulate>=0.8.7'" |
|---|
| 328 | list_req_dgpsi_260='"numpy>=1.18.2" "numba>=0.51.2" "matplotlib>=3.2.1" "tqdm>=4.50.2" "scipy>=1.4.1" "scikit-learn>=0.22.0" "jupyter>=1.0.0" "dill>=0.3.2" "pathos>=0.2.9" "multiprocess>=0.70.13" "psutil>=5.8.0" "tabulate>=0.8.7"' |
|---|
| 329 | for req in netcdf4 pandas seaborn |
|---|
| 330 | do pip install $req |
|---|
| 331 | done |
|---|
| 332 | pip install 'dgpsi=='$dgpsi_version |
|---|
| 333 | fi |
|---|
| 334 | fi |
|---|
| 335 | } |
|---|
| 336 | |
|---|
| 337 | function load_R() { |
|---|
| 338 | # Setup R env variables |
|---|
| 339 | |
|---|
| 340 | # Install R packages locally |
|---|
| 341 | mkdir -p "$renv" |
|---|
| 342 | |
|---|
| 343 | if [[ $network = "on" ]]; then |
|---|
| 344 | if [[ $ExeterUQ = "ExeterUQ_MOGP" ]]; then |
|---|
| 345 | # (Amaury) Note: R's package management is terrible. In particular, there is no way to auto-resolve the depencies of a R package for "old" versions of R. |
|---|
| 346 | # This is particularly a pain when we want to support machines with different versions of R installed. |
|---|
| 347 | # The list and order of packages below is an attempt to circumvent that, but I hope we can find a cleaner way in the future.... |
|---|
| 348 | if ! eval "$htune_dir/src/CheckInstallPackages.sh" Rcpp lattice Matrix reticulate invgamma GenSA far dotCall64 spam fields lhs maps mco ncdf4 shape tensor withr loo MASS pracma mvtnorm flexclust tmvtnorm tictoc ars HI coda MfUSampler ; then |
|---|
| 349 | echo "STOP: Problem encountered when installing R packages"; exit 1 |
|---|
| 350 | fi |
|---|
| 351 | elif [[ $ExeterUQ = "ExeterUQ_dgpsi" ]]; then |
|---|
| 352 | # le commit 1ce49d6 correspond à la version 2.5.0 sur https://github.com/mingdeyu/dgpsi-R |
|---|
| 353 | # le commit 2ae2e0a correspond à la version 2.6.0+bugfix sur https://github.com/mingdeyu/dgpsi-R |
|---|
| 354 | dgpsiR_commit=2ae2e0a |
|---|
| 355 | tinydancer_commit=d268375 |
|---|
| 356 | if ! eval "$htune_dir/src/CheckInstallPackages.sh" devtools shape mingdeyu/dgpsi-R:$dgpsiR_commit BayesExeter/tinydancer:$tinydancer_commit ; then |
|---|
| 357 | echo "STOP: Problem encountered when installing R packages for DGPSI version"; exit 1 |
|---|
| 358 | fi |
|---|
| 359 | else |
|---|
| 360 | # (Amaury) Note: not tested |
|---|
| 361 | # |
|---|
| 362 | # When using the original version of ExeterUQ, should install rstan. |
|---|
| 363 | # Touchy .. |
|---|
| 364 | if ! eval "$htune_dir/src/CheckInstallPackages.sh" Rcpp reticulate invgamma GenSA far fields lhs maps mco ncdf4 shape tensor withr loo MASS rstan; then |
|---|
| 365 | echo "STOP: Problem encountered when installing R packages"; exit 1 |
|---|
| 366 | fi |
|---|
| 367 | fi |
|---|
| 368 | fi |
|---|
| 369 | } |
|---|
| 370 | |
|---|
| 371 | load_python |
|---|
| 372 | #echo "set -u ; export R_LIBS=\"$(realpath "$renv"):\$R_LIBS\"; export R_LIBS_USER=\"$(realpath "$renv")\"; source \"$(pwd)/$venv/bin/activate\" ; set +u" >> env.sh |
|---|
| 373 | echo "test -u \$renv && export R_LIBS=\"$(realpath "$renv"):\$R_LIBS\"; export R_LIBS_USER=\"$(realpath "$renv")\"; source \"$(pwd)/$venv/bin/activate\"" >> env.sh |
|---|
| 374 | source env.sh |
|---|
| 375 | |
|---|
| 376 | load_R |
|---|
| 377 | } |
|---|
| 378 | |
|---|
| 379 | function load_exeterUQ() { |
|---|
| 380 | #----------------------------------------------------------------------- |
|---|
| 381 | # Insure the same version of ExeterUQ tools will be used |
|---|
| 382 | # with this version of the HighTune project |
|---|
| 383 | #----------------------------------------------------------------------- |
|---|
| 384 | # This commit corresponds to the last one on master |
|---|
| 385 | # Merge: 089036e e284f7e |
|---|
| 386 | # Author: vicvolodina93 <52710405+vicvolodina93@users.noreply.github.com> |
|---|
| 387 | # Date: Thu Jun 4 07:59:41 2020 +0100 |
|---|
| 388 | # exeter_commit=9e45cde952d75c10515646f412226f6072606b81 |
|---|
| 389 | # exeter_commit=ef50736999f1879f19759ca84655a326e7a6b74d |
|---|
| 390 | # 19/02/2021 : exeter_commit=1cdb5c5fedb266e196627336b23ba02cc1375c0a |
|---|
| 391 | # 23/02/2021 : exeter_commit=c67cacb47363389ca767fed5ba30d4a82fb13a2d |
|---|
| 392 | # 25/02/2021 : exeter_commit=7dd118fc0d3da1c3441be2712a6b09a1257e166a |
|---|
| 393 | # 26/02/2021 : exeter_commit=283afc55e10e40e147726827e75cbd3a7dde06e0 |
|---|
| 394 | # 01/03/2021 |
|---|
| 395 | #----------------------------------------------------------------------- |
|---|
| 396 | |
|---|
| 397 | exeter_commit=06b008a50b1598d44c9e94a679482ec0896d6cd0 |
|---|
| 398 | isnew=0 |
|---|
| 399 | if [[ ! -d $ExeterUQ ]] ; then |
|---|
| 400 | git clone "https://github.com/BayesExeter/$ExeterUQ" |
|---|
| 401 | isnew=1 |
|---|
| 402 | fi |
|---|
| 403 | cd "$ExeterUQ" |
|---|
| 404 | if [[ $ExeterUQ = ExeterUQ_MOGP ]] ; then |
|---|
| 405 | set +e |
|---|
| 406 | command -v git > /dev/null 2>&1 |
|---|
| 407 | if [ $? -eq 0 ] ; then |
|---|
| 408 | git checkout $exeter_commit |
|---|
| 409 | else |
|---|
| 410 | echo "warning: git command not available, could not checkout commit $exeter_commit in $ExeterUQ" |
|---|
| 411 | fi |
|---|
| 412 | set -e |
|---|
| 413 | fi |
|---|
| 414 | |
|---|
| 415 | ### Bricolage momentané |
|---|
| 416 | # patch pour que ça marche en 2D (2 paramètres) (Naj) |
|---|
| 417 | if [[ $isnew -eq 1 ]] ; then |
|---|
| 418 | patch HistoryMatching/HistoryMatching.R "$htune_dir/src/patchs/patch_HistoryMatching" |
|---|
| 419 | fi |
|---|
| 420 | cd .. |
|---|
| 421 | } |
|---|
| 422 | |
|---|
| 423 | function install_model() { |
|---|
| 424 | echo '======================================================================' |
|---|
| 425 | echo ' 3. CONFIGURING THE SPECIFIC EXPERIMENT ' |
|---|
| 426 | echo '======================================================================' |
|---|
| 427 | |
|---|
| 428 | if [[ $model = "" ]] ; then # no model => we setup the very basic example, and run wave 1 |
|---|
| 429 | mkdir -p "$expdir/EXEMPLE/" |
|---|
| 430 | if [[ $ExeterUQ == "ExeterUQ_MOGP" ]] ; then exemple=exemple.sh_mogp ; else exemple=exemple.sh_dgpsi ; fi |
|---|
| 431 | cp "$htune_dir/src/$exemple" "$expdir/EXEMPLE/exemple.sh" |
|---|
| 432 | cd "$expdir/EXEMPLE" |
|---|
| 433 | chmod +x exemple.sh |
|---|
| 434 | bash exemple.sh |
|---|
| 435 | echo "CHECK RESULTS IN $expdir/EXEMPLE/" |
|---|
| 436 | exit 0 |
|---|
| 437 | fi |
|---|
| 438 | |
|---|
| 439 | # Copy model-specific files locally |
|---|
| 440 | rm -rf tmp |
|---|
| 441 | mkdir -p tmp/models # put in tmp/ because setup_$model.sh installs in ../. |
|---|
| 442 | cp -rf "$htune_dir/models/$model" tmp/models |
|---|
| 443 | if [[ $model = "LMDZ" ]]; then |
|---|
| 444 | # set ecrad path for LMDZ |
|---|
| 445 | set -xv |
|---|
| 446 | sed_i -e "s:ecrad_dir=.*.$:ecrad_dir=$(realpath "$htune_dir/models/ECRAD"):" "tmp/models/LMDZ/serie_LMDZ.sh" |
|---|
| 447 | set +xv |
|---|
| 448 | fi |
|---|
| 449 | |
|---|
| 450 | echo '-----------------------------------------------------------------------' |
|---|
| 451 | echo ' Possibility to have a setup_* specific of a given model' |
|---|
| 452 | echo '-----------------------------------------------------------------------' |
|---|
| 453 | if [[ -f "tmp/models/$model/setup_$model.sh" ]]; then |
|---|
| 454 | cd tmp; "$htune_dir/models/$model/setup_$model.sh"; cd - |
|---|
| 455 | fi |
|---|
| 456 | mkdir -p "$expdir/$experience" |
|---|
| 457 | cp -rf "tmp/models/$model/"* "$expdir/$experience/" |
|---|
| 458 | rm -rf tmp |
|---|
| 459 | } |
|---|
| 460 | |
|---|
| 461 | function download_experiment_data() { |
|---|
| 462 | echo '-----------------------------------------------------------------------' |
|---|
| 463 | echo ' Downloading the data relevant to the current experiment' |
|---|
| 464 | echo '-----------------------------------------------------------------------' |
|---|
| 465 | |
|---|
| 466 | if [[ $model = "ECRAD" ]] ; then |
|---|
| 467 | |
|---|
| 468 | if [[ ! -d LES1D_ecRad ]] ; then |
|---|
| 469 | echo "Downloading LES/MC results for radiation" |
|---|
| 470 | #wget http://simu:visu2018@www.umr-cnrm.fr/visu-high-tune/data_tuningtool/LES1D_ecRad.tar |
|---|
| 471 | # Naj: temporary change |
|---|
| 472 | wget https://web.lmd.jussieu.fr/~nvillefranque/pub/data/LES1D_ecRad.tar |
|---|
| 473 | tar xvf LES1D_ecRad.tar |
|---|
| 474 | rm -rf LES1D_ecRad.tar |
|---|
| 475 | fi |
|---|
| 476 | |
|---|
| 477 | if [[ ! -d RAD ]] ; then |
|---|
| 478 | echo "Downloading LES/MC results for radiation" |
|---|
| 479 | #wget http://simu:visu2018@www.umr-cnrm.fr/visu-high-tune/data_tuningtool/RAD.tar |
|---|
| 480 | # Naj: temporary change |
|---|
| 481 | wget https://web.lmd.jussieu.fr/~nvillefranque/pub/data/RAD.tar |
|---|
| 482 | tar xvf RAD.tar |
|---|
| 483 | rm -rf RAD.tar |
|---|
| 484 | fi |
|---|
| 485 | |
|---|
| 486 | else # of [[ $model = "ECRAD" ]] |
|---|
| 487 | |
|---|
| 488 | if [[ ! -d LES ]] ; then |
|---|
| 489 | echo "Downloading LES results from CNRM and LMD" |
|---|
| 490 | #wget http://simu:visu2018@www.umr-cnrm.fr/visu-high-tune/data_tuningtool/les.tar |
|---|
| 491 | wget https://web.lmd.jussieu.fr/~nvillefranque/pub/data/les.tar |
|---|
| 492 | if [[ ! -f les.tar ]] ; then echo "Can not download les.tar" ; exit 1 ; fi |
|---|
| 493 | tar xvf les.tar |
|---|
| 494 | rm -f les.tar |
|---|
| 495 | #----------------------------------------------------------------------- |
|---|
| 496 | # Bricolage avec le cas RCE oceanique. A reprendre |
|---|
| 497 | # cd LES/RCE_OCE/ ; mv REF DAILY ; mkdir -p REF |
|---|
| 498 | # for i in 0 1 ; do |
|---|
| 499 | # cdo daymean DAILY/LES$i.nc REF/LES$i.nc |
|---|
| 500 | # ncks -v zf DAILY/LES$i.nc -A REF/LES$i.nc |
|---|
| 501 | # ncrename -d z,levf REF/LES$i.nc -O |
|---|
| 502 | # # On prend le calendrier du cas LMDZ1D en attendant mieux. |
|---|
| 503 | # # Avec le nouveau format, devrait permettre d'etre propre |
|---|
| 504 | # # Tous les cas idealises au 1er janvier 2000 ? |
|---|
| 505 | # ncatted -a "units,time,o,c,days since 1997-11-01 00:00:00" REF/LES$i.nc -O |
|---|
| 506 | # done |
|---|
| 507 | # cd - |
|---|
| 508 | |
|---|
| 509 | # on supprime le repertoire gabls4 provenant de la base de donnee CNRM-LMD et on le |
|---|
| 510 | # remplace par celui du leslmd.tar (avec le bon calendrier) |
|---|
| 511 | rm -rf LES/GABLS4 |
|---|
| 512 | wget --no-check-certificate http://www.lmd.jussieu.fr/~lmdz/HighTune/leslmd.tar |
|---|
| 513 | if [ ! -f leslmd.tar ] ; then echo Can not download leslmd.tar ; exit 1 ; fi |
|---|
| 514 | tar xvf leslmd.tar |
|---|
| 515 | rm -f leslmd.tar |
|---|
| 516 | |
|---|
| 517 | # Convective files pre processed by Louis d'Alençon en Lamine Thiam |
|---|
| 518 | cd LES |
|---|
| 519 | \rm -rf RCE_OCE AMMA CINDY |
|---|
| 520 | wget http://www.lmd.jussieu.fr/~hourdin/git_d/data/AdditionalLES/les.tar |
|---|
| 521 | tar xvf les.tar |
|---|
| 522 | \rm -f les.tar |
|---|
| 523 | cd - |
|---|
| 524 | |
|---|
| 525 | #----------------------------------------------------------------------- |
|---|
| 526 | |
|---|
| 527 | fi # of [[ ! -d LES ]] |
|---|
| 528 | fi # of [[ $model = "ECRAD" ]] |
|---|
| 529 | } |
|---|
| 530 | |
|---|
| 531 | function create_working_dir() { |
|---|
| 532 | echo '-----------------------------------------------------------------------' |
|---|
| 533 | echo "Creating and installing the working directory $expdir/$experience" |
|---|
| 534 | echo '-----------------------------------------------------------------------' |
|---|
| 535 | |
|---|
| 536 | local DIR0 |
|---|
| 537 | DIR0=$(pwd) |
|---|
| 538 | |
|---|
| 539 | mkdir -p "$expdir/$experience/log" |
|---|
| 540 | cp -f "$htune_dir/src/"*.py "$htune_dir/src/"*.sh "$htune_dir/src/"*.R "$expdir/$experience/" |
|---|
| 541 | |
|---|
| 542 | cd "$expdir/$experience" |
|---|
| 543 | if [[ "$model" = "ECRAD" ]] ; then |
|---|
| 544 | ln -sf "$DIR0/RAD" . |
|---|
| 545 | ln -sf "$DIR0/LES1D_ecRad" . |
|---|
| 546 | else |
|---|
| 547 | ln -sf "$DIR0/LES" . |
|---|
| 548 | fi |
|---|
| 549 | # ln -s $DIR0/$ExeterUQ/BuildEmulator . |
|---|
| 550 | |
|---|
| 551 | ### Bricolage momentané |
|---|
| 552 | # patch while waiting daniel to do the comit |
|---|
| 553 | # on ExetreUQ package |
|---|
| 554 | cp -r "$DIR0/$ExeterUQ/BuildEmulator" . |
|---|
| 555 | mv BuildEmulator_tmp.R BuildEmulator/BuildEmulator.R |
|---|
| 556 | |
|---|
| 557 | if [[ ! -d HistoryMatching ]] ; then |
|---|
| 558 | ln -s "$DIR0/$ExeterUQ/HistoryMatching" . |
|---|
| 559 | fi |
|---|
| 560 | if [[ ! -d mogp_emulator ]] ; then |
|---|
| 561 | ln -s "$DIR0/mogp_emulator" . |
|---|
| 562 | fi |
|---|
| 563 | |
|---|
| 564 | cp "$DIR0/env.sh" . |
|---|
| 565 | } |
|---|
| 566 | |
|---|
| 567 | # If sourced: returns, else run setup |
|---|
| 568 | if [[ ! "${BASH_SOURCE[0]}" = "$0" ]]; then return 0; fi |
|---|
| 569 | |
|---|
| 570 | check_known_issues |
|---|
| 571 | load_options "$@" |
|---|
| 572 | if [[ $clean = 1 ]] ; then clean_setup ; exit 0 ; fi |
|---|
| 573 | cd "$installdir" |
|---|
| 574 | save_base_environment |
|---|
| 575 | load_platform_environment |
|---|
| 576 | load_programs |
|---|
| 577 | load_python_and_R |
|---|
| 578 | if [[ $ExeterUQ == "ExeterUQ_MOGP" ]] ; then load_exeterUQ ; fi |
|---|
| 579 | install_model |
|---|
| 580 | download_experiment_data |
|---|
| 581 | create_working_dir |
|---|
| 582 | |
|---|
| 583 | echo "End of $0" |
|---|
