#! /bin/bash

########################################################
#PBS -S  /bin/bash
#PBS -j  oe
########################################################
### available queues: 
### short std h12 day days3 week weeks2 infini
### 2h    6h  12h 24h 72h   168h 340h   840h
########################################################
#PBS -q  week
#PBS -N  run
########################################################
### single processor (e.g. testing)
##PBS -l  nodes=1:ppn=1
########################################################
## standard run 121x121x61
#PBS -l  nodes=1:ppn=4,mem=4gb,vmem=6gb
########################################################
### large-domain run 321x321x61 (OK but slower)
##PBS -l  nodes=1:ppn=16,mem=16gb,vmem=24gb
########################################################
### large-domain run 321x321x61
##PBS -l  nodes=1:ppn=32,mem=32gb,vmem=48gb
########################################################
### need for large memory
##PBS -l  mem=120gb
##PBS -l  vmem=120gb
########################################################
#PBS -v  step=1
## NB: "qsub -v step=XX launch" overrides value above
########################################################

#### PREAMBLE
ulimit -s unlimited

# This finds out the number of nodes we have
NP=$(wc -l $PBS_NODEFILE | awk '{print $1}')
echo "Total CPU count = $NP"

#### STEP 1
if [[ ${step} -le 1 ]] ; then
  cd $PBS_O_WORKDIR/gcm
  #
  ./get_startday_from_namelist.sh | ./launch_gcm
  #
  if [[ $NP -gt 24 ]] ; then
    echo "--- Total CPU count is above 24"
    echo "--- For the standard GCM resolution, this is too much"
    echo "--- So we run the GCM only with 24 processors"
    $WHERE_MPI/mpirun -np 24 gcm.e > log_gcm
  else
    $WHERE_MPI/mpirun gcm.e > log_gcm
  fi
  #
  cd $PBS_O_WORKDIR/prep
  echo 1 | create_readmeteo.exe
  readmeteo.exe < readmeteo.def
fi

##### STEP 2
if [[ ${step} -le 2 ]] ; then
  cd $PBS_O_WORKDIR/geogrid
  \rm geo_em.*
  geogrid.exe
  #
  cd $PBS_O_WORKDIR/metgrid
  \rm met_em.*
  metgrid.exe
fi

##### STEP 3
if [[ ${step} -le 3 ]] ; then
  cd $PBS_O_WORKDIR/
  \rm wrfi* wrfb*
  real.exe
fi

##### STEP 4
if [[ ${step} -le 4 ]] ; then
  rm -rf $PBS_O_WORKDIR/$PBS_JOBNAME
  mkdir $PBS_O_WORKDIR/$PBS_JOBNAME
  mv rsl.error.0000 $PBS_O_WORKDIR/$PBS_JOBNAME/real_rsl.error.0000
  mv rsl.out.0000 $PBS_O_WORKDIR/$PBS_JOBNAME/real_rsl.out.0000
  cp -rfL  $PBS_O_WORKDIR/*.def           $PBS_O_WORKDIR/$PBS_JOBNAME/
  cp -rfL  $PBS_O_WORKDIR/wrf.exe*        $PBS_O_WORKDIR/$PBS_JOBNAME/
  cp -rfL  $PBS_O_WORKDIR/namelist.input  $PBS_O_WORKDIR/$PBS_JOBNAME/
  cp -rfL  $PBS_O_WORKDIR/namelist.wps    $PBS_O_WORKDIR/$PBS_JOBNAME/
  cp -rfL  $PBS_O_WORKDIR/wrfi*           $PBS_O_WORKDIR/$PBS_JOBNAME/
  cp -rfL  $PBS_O_WORKDIR/wrfb*           $PBS_O_WORKDIR/$PBS_JOBNAME/
  cd $PBS_O_WORKDIR/$PBS_JOBNAME/
  ln -sf $PBS_O_WORKDIR/data_physics ./
  $WHERE_MPI/mpirun wrf.exe
  #gdb wrf.exe -ex=run
fi