Changeset 1036 in lmdz_wrf

Timestamp:

Aug 21, 2016, 1:01:03 AM (9 years ago)

Author:

lfita

Message:

Up to file differences computing

File:

• trunk/tools/model_graphics.py (modified) (8 diffs)

trunk/tools/model_graphics.py

@@ -637 +637 @@
     Tunits = config['CFunitstime']
     #  opsur = operation surnames
-
     opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
     Opsurs = {}
@@ -726 +725 @@
             # temporal accumulated values
             print "  " + fname + ": kind '" + op + "' not ready !!"
-            wrongstats.appen(CFvarn + '_' + opers)
+            wrongstats.append(CFvarn + '_' + opers)
             break
         elif op == 'direct':
@@ -760 +759 @@
             # latitudinal mean values
             print "  " + fname + ": kind '" + op + "' not ready !!"
-            wrongstats.appen(CFvarn + '_' + opers)
+            wrongstats.append(CFvarn + '_' + opers)
             break
         elif op =='Lsec':
             # latitudinal section (latitudinal value must be given, [var]@[lat]) 
             print "  " + fname + ": kind '" + op + "' not ready !!"
-            wrongstats.appen(CFvarn + '_' + opers)
+            wrongstats.append(CFvarn + '_' + opers)
             break
         elif op =='lmean':
             # longitudinal mean values
             print "  " + fname + ": kind '" + op + "' not ready !!"
-            wrongstats.appen(CFvarn + '_' + opers)
+            wrongstats.append(CFvarn + '_' + opers)
             break
         elif op =='lsec':
             # longitudinal section (longitudinal value must be given, [var]@[lon])
             print "  " + fname + ": kind '" + op + "' not ready !!"
-            wrongstats.appen(CFvarn + '_' + opers)
+            wrongstats.append(CFvarn + '_' + opers)
             break
         elif op == 'pinterp':
@@ -1733 +1732 @@
             headerf2 = vals2[0]
 
-            diffops[vnop] = [headerf1, headerf2]
+            diffops[vnop] = [mods[0], mods[1], exps[0], exps[1], headerf1, headerf2]
 
             Ndiffop = Ndiffop + 1        
@@ -1764 +1763 @@
             headerf2 = vals2[0]
 
-            diffvars[vnop] = [headerf1, headerf2]
+            diffvars[vnop] = [mods[0], mods[1], exps[0], exps[1], headerf1, headerf2]
 
             Ndiffvar = Ndiffvar + 1        
@@ -1808 +1807 @@
 
     return diffs, Ndiffs
+
+def compute_op_diffs(config, diffallop, odir, diffscr, debug):
+    """ Function to compute operation differences
+      config= experiment configuration
+      diffallop= dictonary with the differences to compute
+      odir= output folder
+      diffscr= wether it should be done from the scratch
+    """
+    fname = 'compute_op_diffs'
+
+    # output folder
+    ofold = config['ofold']
+
+    # Home of the python scripts
+    pyH = config['pyHOME']
+
+    #  opsur = operation surnames
+    opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
+    Opsurs = {}
+    for opk in opsur:
+        opn = opk.split('_')[1]
+        vals = opsur[opk]
+        Opsurs[opn] = vals.split(':')
+
+    # Getting in working dir
+    os.chdir(odir)
+
+    # CF dimensions
+    CFdims = ['lon', 'lat', 'pres', 'time']
+
+    # Trakcing file
+    trkobjf = open(odir + '/all_diffop.inf', 'a')
+
+    for vnop in diffallop.keys():
+        vn = vnop.split('_')[0]
+        op = vnop.split('_')[1]
+        opS = op.replace('+','_')
+
+        # `p' header add related to 'pinterp'
+        if opS.find('pinterp') != -1: SgP = 'p'
+        else: SgP = ''
+
+        # Variable name within the file
+        vninF = varnoper(vn, op, Opsurs)
+
+        vals = diffallop[vnop]
+        mod1 = vals[0]
+        mod2 = vals[1]
+        exp1 = vals[2]
+        exp2 = vals[3]
+        headerf1 = vals[4]
+        headerf2 = vals[5]
+        modexpdiff = mod2 + '/' + exp2 + ' - ' + mod1 + '/' + exp1
+        modexpdiffS = mod2 + '-' + exp2 + '_' + mod1 + '-' + exp1
+
+        ifile1 = ofold + '/' + mod1 + '/' + exp1 + '/' + vn + '_' + headerf1 + SgP + \
+          '_' + opS + '.nc'
+        ifile2 = ofold + '/' + mod2 + '/' + exp2 + '/' + vn + '_' + headerf2 + SgP + \
+          '_' + opS + '.nc'
+        difffilen = odir + '/' + vn + '_diffop_' + modexpdiffS + '_' + opS + '.nc'
+
+        if diffscr:
+            sout = sub.call('rm ' + difffilen + ' >& /dev/null', shell=True)
+
+        if not os.path.isfile(difffilen):
+            if debug:
+                print "  Computing operation difference '" + op + "' of '" + vn +    \
+                  "' for: '" + modexpdiff
+                print "    var in file: '" + vninF + "'"
+
+            dfiles = 'add|' + ifile2 + '|' + vninF + ',sub|' + ifile1 + '|' + vninF
+            values = '|'.join(CFdims) + '@' + dfiles
+            pyins = 'python ' + pyH + "/nc_var.py -o compute_opersvarsfiles -S '" +  \
+              values + "' -v " + vninF
+
+            try:
+                with gen.Capturing() as output:
+                    ncvar.compute_opersvarsfiles(values,vninF)
+            except:
+                print errmsg
+                print 'ncvar.compute_opersvarsfiles(' + values + ',' + vninF + ')'
+                for sout in output: print sout
+                quit(-1)
+
+            sout = sub.call('mv opersvarsfiles_'+vninF+'.nc ' + difffilen, shell=True)
+
+            # keeping all differences
+            trkobjf.write('\n')
+            trkobjf.write("#" + vn + ' ' + op + '\n')
+            trkobjf.write( pyins + '\n')
+
+    trkobjf.close()
+
+    return
+
+def compute_diff_statistics(Ecnf, filen, ofold, varn, Opers, scr, db):
+    """ Function to compute statistics from var diff files
+      Ecnf= dictionary with the configuration of the experiment
+      filen= variable difference file to use
+      ofold= directory to write the output files
+      varn= name of the variable
+      Opers= kind of operation: (as possible multiple consecutive combination of operations separated by '+'
+        [calc1]+[calc2] will compute first [calc1] and then [calc2] 
+        acc: temporal accumulated values
+        diff: differences between models
+        direct: no statistics
+        last: last temporal value
+        Lmean: latitudinal mean values
+        Lsec: latitudinal section (latitudinal value must be given, [var]@[lat]) 
+        lmean: longitudinal mean values
+        lsec: longitudinal section (longitudinal value must be given, [var]@[lat])
+        pinterp: pressure interpolation (to the given $plevels)
+        tmean: temporal mean values
+        turb: Taylor's turbulence decomposition value
+        xmean: x-axis mean values
+        ymean: y-axis mean values
+        zsum: vertical aggregated values
+      scr= should it be done from the scratch?
+    """
+    fname='compute_diff_statistics'
+
+    # Full header of file
+    Lfilen = len(filen)
+    Hfile = filen[0:Lfilen-3]
+
+    # CF dimension-variables name
+    OpersS = '+'.join(Opers)
+    if OpersS.find('pinterp') != -1:
+        varnCFs = ['lon', 'lat', 'pres', 'time']
+    else:
+        varnCFs = ['lon', 'lat', 'time']
+
+    # Variable-dimension CF dictionary
+    CFdimvardict = {'lon': 'lon', 'lat': 'lat', 'pres': 'pres', 'time': 'time'}
+
+    # Python scripts HOME
+    pyH = Ecnf['pyHOME']
+
+    #  opsur = operation surnames
+    opsur = gen.get_specdictionary_HMT(Ecnf, H='opsur_',M='',T='')
+    Opsurs = {}
+    for opk in opsur:
+        opn = opk.split('_')[1]
+        vals = opsur[opk]
+        Opsurs[opn] = vals.split(':')
+
+    # Getting in working dir
+    os.chdir(ofold)
+
+    # File to keep track of all the statistics of var differences
+    otrackf = open(ofold + '/all_vardiffstatistics.inf', 'a')
+
+    if db: 
+        print "    computing statistics of diff variable: '", varn ,"' operation '"+ \
+      OpersS + "' using file '" + filen + " ...'"
+
+    for op in Opers:
+        # File name from previous operations, and name of the variable within the 
+        #   file (some operations change the name of the variable)
+        if op == Opers[0]:
+            Fopers = op
+            prevfile = filen
+            vninF = varn
+        else:
+            Fopers = Fopers + '_' + op
+            prevfile = fileon
+        fileon = Hfile + '_' + Fopers + '.nc'
+
+        SvarnCFs = ',' + ','.join(varnCFs) 
+        CFvarnp = vninF + SvarnCFs
+
+        if scr:
+            sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
+
+        # Just produce file in case output file does not exist
+        if not os.path.isfile(fileon):
+            if db: 
+                print '  ' + fname + ": creation of file '" + fileon + "' ..."
+            dofile = True
+        else:
+            dofile = False
+        
+        # Variables to be kept in the final file
+        varkeep = []
+
+        if op == 'acc':
+            # temporal accumulated values
+            print "  " + fname + ": kind '" + op + "' not ready !!"
+            wrongstats.append(varn + '_' + opers)
+            break
+        elif op == 'direct':
+            # no statistics
+            if dofile:
+                sout = sub.call('mv ' + prevfile + ' ' + fileon, shell=True)
+        elif op == 'last':
+            # last temporal value
+            vals='time,-9,0,0'
+            if dofile:
+                try:
+                    with gen.Capturing() as output:
+                        ncvar.DataSetSection(vals,prevfile)
+                except:
+                    print 'DataSetSection('+vals+',', prevfile, ')'
+                    for sout in output: print sout
+                    quit(-1)
+
+                if db:
+                    for sout in output: print sout
+
+                hprevfile = prevfile[0:len(prevfile)-3]
+
+                sout = sub.call('mv ' + hprevfile + '_time_B-9-E0-I0.nc ' + fileon, \
+                  shell=True)
+
+                # Keeping the operations
+                pyins=pyH + "/nc_var.py -o DataSetSection -S '" + vals + "' -f " +  \
+                  prevfile
+                otrackf.write("\n")
+                otrackf.write("# " + varn + " " + Fopers + "\n")
+                otrackf.write(pyins + "\n")
+
+        elif op =='Lmean':
+            # latitudinal mean values
+            print "  " + fname + ": kind '" + op + "' not ready !!"
+            wrongstats.append(varn + '_' + opers)
+            break
+        elif op =='Lsec':
+            # latitudinal section (latitudinal value must be given, [var]@[lat]) 
+            print "  " + fname + ": kind '" + op + "' not ready !!"
+            wrongstats.append(varn + '_' + opers)
+            break
+        elif op =='lmean':
+            # longitudinal mean values
+            print "  " + fname + ": kind '" + op + "' not ready !!"
+            wrongstats.append(varn + '_' + opers)
+            break
+        elif op =='lsec':
+            # longitudinal section (longitudinal value must be given, [var]@[lon])
+            print "  " + fname + ": kind '" + op + "' not ready !!"
+            wrongstats.append(varn + '_' + opers)
+            break
+        elif op == 'tmean':
+            # temporal mean values
+            vals='time|-1,time,mean,' + ':'.join(varnCFs)
+            dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
+            if dofile:
+                try:
+                    with gen.Capturing() as output:
+                        ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
+                except:
+                    print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
+                    for sout in output: print sout
+                    quit(-1)
+
+                if db:
+                    for sout in output: print sout
+
+                sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
+
+                # Keeping the operations
+                pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals +        \
+                  "' -f " + prevfile + " -v " + CFvarnp
+                otrackf.write("\n")
+                otrackf.write("# " + varn + " " + Fopers + "\n")
+                otrackf.write(pyins + "\n")
+
+            # removing dimension variable-dimension 'time'
+            varnCFs.remove('time')
+
+            varkeep.append('timestats')
+
+#        elif op == 'turb':
+#            # turbulence values
+#            vals='turbulence|'${CFvarn}
+#            dims='time@time,'${dnz}'@'${vdnz}',lat@lat,lon@lon,pres@pres'
+#            pyout=`python ${pyHOME}/diagnostics.py -d ${dims} -v ${vals} -f ${cfiles}`
+
+#            # Keeping the operations
+#            pyins="python "${pyHOME}"/diagnostics.py -d "${dims}" -v '"${vals}
+#            pyins=${pyins}"' -f ${cfiles}"
+#            echo " " >> ${odir}/all_statsvars.inf
+#            echo "# ${CFvarn}" "${vark}" >> ${odir}/all_statsvars.inf
+#            echo ${pyins} >> ${odir}/all_statsvars.inf
+
+#            varkeep=':'${CFvarn}'turb'
+
+        elif op == 'xmean':
+            # x-axis mean values
+            vals='lon|-1,lon,mean,' + ':'.join(varnCFs)
+            dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
+            if dofile:
+                try:
+                    with gen.Capturing() as output:
+                        ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
+                except:
+                    print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
+                    for sout in output: print sout
+                    quit(-1)
+
+                if db:
+                    for sout in output: print sout
+
+                sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
+
+                # Keeping the operations
+                pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals +        \
+                  "' -f " + prevfile + " -v " + CFvarnp
+                otrackf.write("\n")
+                otrackf.write("# " + varn + " " + Fopers + "\n")
+                otrackf.write(pyins + "\n")
+
+            varkeep.append('lonstats')
+
+        elif op == 'ymean':
+            # y-axis mean values
+            vals='lat|-1,lat,mean,' + ':'.join(varnCFs)
+            dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
+            if dofile:
+                try:
+                    with gen.Capturing() as output:
+                        ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
+                except:
+                    print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
+                    for sout in output: print sout
+                    quit(-1)
+
+                if db:
+                    for sout in output: print sout
+
+                sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
+
+                # Keeping the operations
+                pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals +        \
+                  "' -f " + prevfile + " -v " + CFvarnp
+                otrackf.write("\n")
+                otrackf.write("# " + varn + " " + Fopers + "\n")
+                otrackf.write(pyins + "\n")
+
+            varkeep.append('latstats')
+
+        elif op == 'zsum':
+            # vertical aggregated values
+            print "  " + fname + ": kind '" + op + "' not ready !!"
+            wrongstats.append(varn + '_' + opers)
+            break
+        else:
+            print errmsg
+            print '  ' + fname + ": operation '" + op + "' not ready !!"
+            quit(-1)
+
+        # End of kind of operation
+
+        # Variable name in file (vninF) changed due to operation 
+        #   but only if previous operation does not the same 'statistic'
+        chvn = gen.dictionary_key_list(Opsurs, op)
+        if chvn is not None:
+            oldvninF = vninF
+            vninF = vninF + chvn
+            CFvarnp = CFvarnp.replace(oldvninF,vninF)
+                
+        if dofile:
+            if len(varkeep) > 0:
+                varkeepS = ',' + ','.join(varkeep)
+            else:
+                varkeepS = ''
+
+            # Adding such CF dimension variables which might not be in the 
+            #   post-operation file
+            try:
+                with gen.Capturing() as output:  
+                    varinfile = ncvar.ivars(fileon)
+            except:
+                print 'ivars('+fileon+')'
+                for sout in output: print sout
+                quit(-1)
+
+            for CFvn in varnCFs:
+                if not gen.searchInlist(varinfile, CFvn):
+                    if db:
+                        print '  ' + fname + ": recupering CF variable '" + CFvn + "'"
+                    try:
+                        with gen.Capturing() as output:
+                            ncvar.fvaradd(prevfile+','+CFvn, fileon)
+                    except:
+                        print 'fvaradd(', prevfile, ','+CFvn+','+fileon+')'
+                        for sout in output: print sout
+                        quit(-1)
+
+                    if db:
+                        for sout in output: print sout
+
+            totalvarkeeps = CFvarnp + ',' + ','.join(varnCFs) + varkeepS
+            try:
+                with gen.Capturing() as output:
+                    oclean = ncvar.cleaning_varsfile(totalvarkeeps,fileon)
+            except:
+                print 'cleaning_varsfile('+totalvarkeeps+','+fileon+')'
+                for sout in output: print sout
+                quit(-1)
+
+            if db:
+                for sout in output: print sout
+
+    # End of operations
+    otrackf.close()
+
+    return
+
+def compute_var_diffs(config, diffallvar, odir, diffscr, debug):
+    """ Function to compute variable differences
+      config= experiment configuration
+      diffallvar= dictonary with the differences to compute
+      odir= output folder
+      diffscr= wether it should be done from the scratch
+    """
+    fname = 'compute_var_diffs'
+
+    # output folder
+    ofold = config['ofold']
+
+    # Home of the python scripts
+    pyH = config['pyHOME']
+
+    #  opsur = operation surnames
+    opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
+    Opsurs = {}
+    for opk in opsur:
+        opn = opk.split('_')[1]
+        vals = opsur[opk]
+        Opsurs[opn] = vals.split(':')
+
+    # Getting in working dir
+    os.chdir(odir)
+
+    # CF dimensions
+    CFdims = ['lon', 'lat', 'pres', 'time']
+
+    # Trakcing file
+    trkobjf = open(odir + '/all_diffvar.inf', 'a')
+
+    for vnop in diffallvar.keys():
+        vn = vnop.split('_')[0]
+        op = vnop.split('_')[1]
+        opSfinal = op.replace('+','_')
+
+        lop = op.split('+')
+        # `p' header add related to 'pinterp'
+        if op.find('pinterp') != -1:
+            print warnmsg
+            print '  ' + fname + ': there is the vertical interpolation as operation'
+            print '    variable differences will be computed from that files where '
+            print '    vertical interpolation was just done'
+            ipop = lop.index('pinterp')
+            opS = '_' + '_'.join(lop[0:ipop+1])
+            doops = lop[ipop+1:]
+            SgP = 'p'
+            vninF = varnoper(vn, '+'.join(lop[0:ipop+1]), Opsurs)
+            print '    operations to perform:', doops
+        else: 
+            SgP = ''
+            opS = ''
+            doops = lop
+            vninF = vn
+
+        Sdoops = '+'.join(doops)
+
+        # Variable name within the file
+
+        vals = diffallvar[vnop]
+        mod1 = vals[0]
+        mod2 = vals[1]
+        exp1 = vals[2]
+        exp2 = vals[3]
+        headerf1 = vals[4]
+        headerf2 = vals[5]
+        modexpdiff = mod2 + '/' + exp2 + ' - ' + mod1 + '/' + exp1
+        modexpdiffS = mod2 + '-' + exp2 + '_' + mod1 + '-' + exp1
+
+        ifile1 = ofold + '/' + mod1 + '/' + exp1 + '/' + vn + '_' + headerf1 + SgP + \
+          opS + '.nc'
+        ifile2 = ofold + '/' + mod2 + '/' + exp2 + '/' + vn + '_' + headerf2 + SgP + \
+          opS + '.nc'
+        difffilen = odir + '/' + vn + '_diffvar_' + modexpdiffS + opS + '.nc'
+        difffinalfilen = odir + '/' + vn + '_diffvar_' + modexpdiffS + opS + '_' +   \
+          Sdoops + '.nc'
+
+        if diffscr:
+            sout = sub.call('rm ' + difffilen + ' >& /dev/null', shell=True)
+            sout = sub.call('rm ' + difffinalfilen + ' >& /dev/null', shell=True)
+
+        if not os.path.isfile(difffinalfilen):
+            if debug:
+                print "  Computing variable difference of '" + vn + "' for: '" +     \
+                  modexpdiff + "' and performing operation '" + Sdoops + "'"
+                print "    var in file: '" + vninF + "'"
+
+            dfiles = 'add|' + ifile2 + '|' + vninF + ',sub|' + ifile1 + '|' + vninF
+            values = '|'.join(CFdims) + '@' + dfiles
+            pyins = 'python ' + pyH + "/nc_var.py -o compute_opersvarsfiles -S '" +  \
+              values + "' -v " + vninF
+
+            try:
+                with gen.Capturing() as output:
+                    ncvar.compute_opersvarsfiles(values,vninF)
+            except:
+                print errmsg
+                print 'ncvar.compute_opersvarsfiles(' + values + ',' + vninF + ')'
+                for sout in output: print sout
+                quit(-1)
+
+            sout = sub.call('mv opersvarsfiles_'+vninF+'.nc ' + difffilen, shell=True)
+
+            # keeping all differences
+            trkobjf.write('\n')
+            trkobjf.write("#" + vn + ' ' + op + '\n')
+            trkobjf.write( pyins + '\n')
+
+            # Computing statisitcs
+            compute_diff_statistics(config, difffilen, odir, vninF, doops, diffscr,  \
+              debug)
+
+    trkobjf.close()
+
+    return
 
 # Files with information about the configuration of the script
@@ -2005 +2528 @@
                 if diffscratch:
                     sub.call('rm -rf' + owdir +' >& /dev/null', shell=True)
-                    sub.call('rm ' + diffvarcompf +' >& /dev/null', shell=True)
+                    sub.call('rm ' + difff +' >& /dev/null', shell=True)
                     objf = open(owdir+'/all_' + fdiff,'w')
                     objf.write("## Computation and drawing of differences " +        \
+                      "between '" + exp2 + "'-'" + exp1 + "'\n")
+                    objf.close()
+                    difff = owdir + '/all_vardiffstatistics.inf'
+                    sub.call('rm ' +  +' >& /dev/null', shell=True)
+                    objf = open(owdir+'/all_' + fdiff,'w')
+                    objf.write("## Computation of all differences statistics " +     \
                       "between '" + exp2 + "'-'" + exp1 + "'\n")
                     objf.close()
@@ -2036 +2565 @@
             gen.printing_dictionary(alldiffvar)
 
+# Computing differences
+##
+        print "    Computing operation differences ..."
+        compute_op_diffs(cnf, alldiffop, owdir, diffscratch, dbg)
+        print "    Computing variable differences ..."
+        compute_var_diffs(cnf, alldiffvar, owdir, diffscratch, dbg)
+
     quit()
 # end of mods loop