source: BOL/Multi_atlas/Utils/multi.sh @ 3745

#set -vx

local=`pwd -P`
cd $local
echo REPERTOIRE LOCAL :  $local

echo '############################################################'
echo  0/ gestion des options
echo '############################################################'
pcmdi_metrics=0
tuning_metrics=1
und=0
zon=1
html=1
atlas=1
atlas_cesmep=0
get=0
cmor=1
regdyn=0
lma=0
trmm=0
axe2=0
axe4=0
esmval2=0
refld_histo=0
vertcld_calipso=0
stations_day=0
stations_mth=0
qsub=0
mjo=0
river=0

# ./job_multi.sh -1D 0 -get 0 -atlas 0 -pcmdi 0 -tuning 0 -lma 0 -trmm 0
runstxt=$local/runs.txt

while test -n "${1}"; do
   case $1 in
      -1D) und=$2 ; shift ;;
      -get) get=$2 ; shift ;;
      -cmor) cmor=$2 ; shift ;;
      -zon) zon=$2 ; shift ;;
      -atlas) atlas=$2 ; shift ;;
      -atlas_cesmep) atlas_cesmep=$2 ; shift ;;
      -pcmdi) pcmdi_metrics=$2 ; shift ;;
      -tuning) tuning_metrics=$2 ; shift ;;
      -runstxt) runstxt=$2 ; shift ;;
      -lma) lma=$2 ; shift ;;
      -trmm) trmm=$2 ; shift ;;
      -axe4) axe4=$2 ; shift ;;
      -axe2) axe2=$2 ; shift ;;
      -regdyn) regdyn=$2 ; shift ;;
      -esmval2) esmval2=$2 ; shift ;; 
      -stations_day) stations_day=$2 ; shift ;; 
      -stations_mth) stations_mth=$2 ; shift ;; 
      -refld_histo) refld_histo=$2 ; shift ;;
      -vertcld_calipso) vertcld_calipso=$2 ; shift ;;
      -mjo) mjo=$2 ; shift ;; 
      -river) river=$2 ; shift ;;
      *) comp=$1 ;;
   esac
   shift
done

rm -f out.com$$
runstxt=$local/runs.txt
echo runstxt= $runstxt

echo DODSDIR : $DODSDIR
MAIN_SE=$DODSDIR/$login/lmdz/SE
TS_DA=$DODSDIR/$login/lmdz/TS_DA
TS_MO=$DODSDIR/$login/lmdz/TS_MO
MULTIDIR=$DODSDIR/$login/lmdz/MultiSimu

if [ ! -d $MULTIDIR ] ; then mkdir -p $MULTIDIR ; fi
if [ ! -d $MAIN_SE ] ; then mkdir -p $MAIN_SE ; fi
if [ ! -d $MAIN_MO ] ; then mkdir -p $MAIN_MO ; fi
if [ ! -d $MAIN_DA ] ; then mkdir -p $MAIN_DA ; fi

COMP_D=$MULTIDIR/$comp
DEF_FILE=$COMP_D/def.txt ; if [ ! -f $DEF_FILE ] ; then echo renseigner $DEF_FILE ; exit ; fi

listsims=""  ; for s  in `awk ' {print $1} ' $DEF_FILE | uniq`          ; do listsims="$listsims $s"    ; done
echo listsims $listsims

echo ### On supprime dans def.txt les lignes vides ########
sed -i "/^[ \t]*$/d" $DEF_FILE

echo ### On teste d abord la coherence des noms des simulations dans def.txt et $runstxt ################
echo MULTI-ATLAS "$comp"
for s in $listsims ; do
srun=`grep -w "^$s "  $runstxt | awk ' { print $1 } '`
echo 's='"$s" '    srun='  "$srun"
if [ "$s" = "$srun" ] ; then
   echo Meme nom de la simulation dans def.txt et $runstxt 's=' "$s" '    srun='  "$srun"
else
   echo ATTENTION : la simulation "$s" de votre def.txt n est pas repertoriee dans $runstxt 
   echo             ou le meme nom est attribue a plusieures simulations
   echo Il faut la rajouter dans $runstxt 
   echo avec le chemin vers les SE et TS dans ciclad : /modfs/$login/IGCM_OUT
   echo             ou renomer votre simulation
exit
fi
done
#exit
listruns=""
listrunss=""
listrunse=""
if [ ! -d $MAIN_SE/ORIG ] ; then mkdir -p $MAIN_SE/ORIG ; fi
for s in $listsims ; do 
simdir=`grep -w "^$s "  $runstxt | awk ' { print $2 } '`
echo $simdir
( cd $MAIN_SE/ORIG ; ln -sf $simdir/ATM/Analyse/SE/*mth*nc . ) > /dev/null 2>&1
echo $simdir
ss=`basename $simdir `
listruns="$listruns $ss"
for year in `grep -w "^$s" $DEF_FILE | awk ' { print $2 } '` ; do
listrunss="$listrunss ${ss}_$year"
listrune="$listrune ${ss}_SE_$year"
done
done
echo listruns = $listruns
echo listrunss= $listrunss
echo listrune= $listrune 
ls -lrt $MAIN_SE/ORIG
listsimss="" ; for ss in `awk ' {print $1"_"$2 } ' $DEF_FILE`    ; do listsimss="$listsimss $ss" ; done
listsimse="" ; for se in `awk ' {print $1"_SE_"$2 } ' $DEF_FILE` ; do listsimse="$listsimse $se" ; done
echo listsims $listsims
echo listsimss $listsimss
echo listsimse $listsimse

echo SIMULATIONS: $listsims

echo '########################################################################'
echo  1/ Rapatriement des simulations manquantes
echo '########################################################################'
#### AI avril 2020 ################ Avant l acces directe aux simulations sur le dods du CCRT
# Adresses web ne sont plus a jours
# A commenter les adresses web CCRT 
# A mettre a jours les adress web IDRIS
# Rajouter d'autres machines avec rsynk, wget ou ssh ...

if [ $get = 1 ] ; then
 cd $MAIN_SE
 for sim in $listsimss ; do echo ./getun.sh ${sim} >> $local/out.com$$ ; ./getun.sh ${sim} ; done
 cd $TS_DA ; pwd
 for sim in $listsimss ; do echo ./getun.sh ${sim} >> $local/out.com$$ ; ./getun.sh ${sim} ; done
 cd $TS_MO ; pwd
 for sim in $listsimss ; do echo ./getun.sh ${sim} >> $local/out.com$$ ; ./getun.sh ${sim} ; done
fi

if [ $cmor = 1 ] ; then
 cp cmor.sh $MAIN_SE/. 
 cp season.sh $MAIN_SE/.
 cd $MAIN_SE
 chmod +x cmor.sh
 chmod +x season.sh
#########################################################################
# 1.2 "Cmorisation des fichiers mensuels : traduction au format CMIP
#########################################################################
# Deux lignes suivantes modif FH 2016/11/09
# Files are taken from /modfs/ipslfs/dods/fabric/lmdz/SE/ORIG and
# results are in /modfs/ipslfs/dods/fabric/lmdz/SE/CMOR
#########################################################################
 ./cmor.sh -runsfile $runstxt $comp
if [ $? != 0 ] ; then exit ; fi # Probleme de histmth non present. On arrete tout

#########################################################################
# 1.3 Interpolate fields on a common horizontal grid (VLR) and
#########################################################################
#     average in time "YEAR", "JJA" ...
#     Results are put on
#     /modfs/ipslfs/dods/fabric/lmdz/VLR/YEAR/[SIMU_NAME]_[period]/NC
#     If wanting to rerun season.sh for observations :
#     ./season.sh -s OBS
#########################################################################
 ./season.sh -comp $comp
 cd $local
fi

echo '########################################################################'
echo  2/ Lancement des atlas manquant
echo '########################################################################'
natlas=`wc -l $COMP_D/atlas-def | awk ' { print $1 } '`
count_atlas=0

if [ $atlas = 1 ] ; then
for sim in $listsimss ; do
   il=1
   if [ ! -f $COMP_D/atlas-def ] ; then
      echo Definir les atlas a construire dans $COMP_D/atlas-def
      echo Example :
      cat $COMP_D/../CMIP5to6/atlas-def
   fi
   while [ $il -le $natlas ] ; do
      line=`sed -n -e ${il}p $COMP_D/atlas-def`
      seas=`echo $line | awk ' { print $1 } '`
      proj=`echo $line | awk ' { print $2 } '`
      vars=`echo $line | awk ' { print $3 } '`
      atlas=${sim}+${seas}+${proj}+$vars
      echo AAAAAAAAAAAAAAAAAAAAAAAA $atlas
      grep $atlas $COMP_D/ATLAS/atlas_OK
      echo BBBBBB
      if [ "`grep $atlas $COMP_D/ATLAS/atlas_OK`" = "" ] ; then
         cd $local/atlas ; pwd ; ls -l atlas.sh
         varg=`echo $vars | sed -e 's/,/+/g'`
         echo Atlas $atlas a creer
         if [ $qsub = 0 ] ; then
            ./atlas.sh RUN=$sim,SEASON=$seas,PROJ=$proj,OUTDIR=$COMP_D/ATLAS,REF=OBS,VARS="$varg",RUNSTXT=$runstxt
         else
            $submit -v RUN=$sim,SEASON=$seas,PROJ=$proj,OUTDIR=$COMP_D/ATLAS,REF=OBS,VARS="$varg",RUNSTXT=$runstxt atlas.sh
         fi
         echo $submit -v RUN=$sim,SEASON=$seas,PROJ=$proj,OUTDIR=$COMP_D/ATLAS,REF=OBS,VARS="$varg" atlas.sh >> out.com$$
         echo $submit -v RUN=$sim,SEASON=$seas,PROJ=$proj,OUTDIR=$COMP_D/ATLAS,REF=OBS,VARS="$varg" atlas.sh
         (( count_atlas = $count_atlas + 1 ))
      else
           echo Atlas $atlas existant
      fi
      (( il = $il + 1 ))
   done
done
if [ $count_atlas != 0 ] ; then
    echo '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
    echo Les atlas manquant ont ete lances. Attendre leur
    echo creation pour relancer job_multi.sh
    echo '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
#   exit
fi
fi

# atlas_cesmep : cles pour activer les nouveau atlas (C-ESM-EP)
if [ $atlas_cesmep = 1 ] ; then
   cd ~/LMDZ/C-ESM-EP
   ./AtlasLMDZ.sh -runsfile $runstxt $comp 
fi # atlas_cesmep

echo '########################################################################'
echo  3/ Calcul des moyennes zonales nco
echo '########################################################################'


#trace moy.zonale OBS + multi-simulations
if [ $zon = 1 ] ; then
   cd $local
   mkdir -p $COMP_D/ZON
   for sim in $listsimse OBS ; do
      years=`echo $sim | sed -e 's/_SE_/ /' | awk ' { print $2 } '`
      run=`echo $sim | sed -e 's/_SE_/ /' | awk ' { print $1 } '`
      ./make_MOY.sh "$years" YEAR "$run" GLOB
      ./make_MOY.sh "$years" YEAR "$run" ZON
      ./make_MOY.sh "$years" JJA "$run"  ZON
      ./make_MOY.sh "$years" DJF "$run"  ZON
      ./make_MOY.sh "$years" JJAS "$run" AMMACROSS
   done
   cd $local
### ajout nuages LMDZ dans la compa et doit faire des liens symbol. sur
#### les OBS calipso toujurs pour le moment
   ./mk_clouds_NOcalipso.sh YEAR ZON
   ./mk_clouds_NOcalipso.sh DJF ZON
   ./mk_clouds_NOcalipso.sh JJA ZON
   ./mk_clouds_NOcalipso.sh JJAS AMMACROSS
#
   pwd
   ./multi_ZONE.sh YEAR $comp ZON -runsfile $runstxt
   ./multi_ZONE.sh JJA $comp ZON -runsfile $runstxt
   ./multi_ZONE.sh DJF $comp ZON -runsfile $runstxt
   ./multi_ZONE.sh JJAS $comp AMMACROSS -runsfile $runstxt

fi

echo '########################################################################'
echo 4/ Lancement du 1D si necessaire
echo '########################################################################'
if [ ! -f $COMP_D/1D/1D.html -a $und = 1  ] ; then
    cd $COMP_D
    ~/LMDZ/1D/serie1d.sh $comp
    cd $local
fi

echo '########################################################################'
echo 5/ Métriques 
echo '########################################################################'
if [ $tuning_metrics = 1 -a ! -d $COMP_D/METRICS ] ; then
   echo metrics.sh
   cd $local/METRICS
   ./metrics.sh $comp
fi

if [ $regdyn = 1 ]; then
   cd ~/users/musat/regdyn/clean_regdyn
   ./mk_all_regdyn.sh $comp 0
   cd $local
fi

if [ $esmval2 = 1 ]; then
   cd $local
   ./cvdp4atlas.sh $comp
   ./perfm4atlas.sh $comp
fi   

if [ $mjo = 1 ]; then
   cd ~/users/idelkadi/MJO
   ./make_diagnosMJO.sh $comp
fi

if [ $river = 1 ]; then 
   cd ~/users/idelkadi/AXE1/River
   ./make_diagnoRiver.sh $comp
fi  

if [ $lma = 1 ]; then
   cd $local
   ./make_lma.sh $comp
fi
if [ $axe2 = 1 ]; then
   cd $local
   ./make_axe2.sh -runstxt $runstxt $comp
fi
if [ $trmm = 1 ]; then
   cd ~/users/mbonaz/
   ./Axe2_Marine.sh $comp
   cd $local
fi
cd $local
if [ $pcmdi_metrics = 1 ] ; then
   ./make_pcmdi.sh $comp
fi
if [ $axe4 = 1 ]; then
  cd $local
   ./make_axe4.sh $comp
fi

if [ "$reflcld_histo" = "1" ] ; then
   cd ~/users/idelkadi/AXE3/
   ./make_scatReflCld.sh $comp
   cd $local
fi

if [ $vertcld_calipso = 1 ] ; then
   cd ~/users/idelkadi/AXE3/Calipso/CoupeVerticale/DJF
   ./make_VerticalMzCldCalipso.sh $comp
   cd ~/users/idelkadi/AXE3/Calipso/CoupeVerticale/JJA
   ./make_VerticalMzCldCalipso.sh $comp
   cd ~/users/idelkadi/AXE3/Calipso/Stratos
   ./make_StratosCalipso.sh $comp
fi


echo '########################################################################'
echo 6/ Stations
echo '########################################################################'

if [ $stations_day = 1 ] ; then
   ~/users/Binta/bibi.sh $comp
   ~/users/cheruy/bibi.sh $comp
fi

if [ $stations_mth = 1 ] ; then
   ~/users/traore/SE_buoy.sh $comp
fi

echo '########################################################################'
echo 7/ generation des pages html
echo '########################################################################'

if [ $html = 1 ] ; then
   echo ./make_html_libigcm_ai.sh --comp $comp
   ./make_html_libigcm.sh --comp $comp
   cd $local
   ./html_entete.sh $comp
   ./html_tuning.sh  $comp
   ./html_tuning_min.sh  $comp
   ./html_accueil.sh  $comp
   ./html_1D.sh  $comp
   ./html_lma.sh $comp
   ./html_pcmdi.sh $comp
   ./html_all.sh >| $MULTIDIR/all.html

   # html pour les métriques
#   cat $COMP_D/entete.html doc_pcmdi.html >| $COMP_D/PCMDI.html
#   ref=`echo $listsimss | awk ' { print $1 } '`
#   echo "<h1> Metrics with respect to $ref </h1>" >>  $COMP_D/PCMDI.html
#   cat $COMP_D/pcmdi_metrics.html >> $COMP_D/PCMDI.html
#   echo "<h1> Metrics with respect to AR4.00 </h1>" >>  $COMP_D/PCMDI.html
#   cat $COMP_D/metrics-AR4-as-ref.html >> $COMP_D/PCMDI.html
#   echo "<h1> Metrics with respect to CMIP5/AMIP multi model </h1>" >>  $COMP_D/PCMDI.html
#   cat $COMP_D/metrics-CMIP5-AMIP-as-ref.html >> $COMP_D/PCMDI.html
   for i in 1 2 3 4 5 6 7 ; do
        ./html_axe$i.sh  $comp
   done
   for proj in GLOB SH NH ; do
   for bias in BIAS MAPS ; do
       for season in YEAR DJF JJA ; do
        echo ./multi_atlas.sh $comp $bias $season $proj $listrunss  
        ./multi_atlas.sh $comp $bias $season $proj $listrunss  >> out.com$$
       done
   done
   done
#exit
#   cd $COMP_D
#   ln -sf BIASYEAR.html  $comp.html
#   ln -sf ACCUEIL.html  $comp.html
cd $local
./webpage_multi.sh $comp

fi