Changeset 1022 in lmdz_wrf

Timestamp:

Aug 15, 2016, 2:38:53 PM (9 years ago)

Author:

lfita

Message:

Working copy up to statistics without 'turb' and redudec space computation of 'pinterp' (for 1D, 2D variables)

Location:

trunk/tools

Files:

• model_graphics.py (modified) (18 diffs)
• model_graphics_template.dat (modified) (3 diffs)

trunk/tools/model_graphics.py

@@ -275 +275 @@
     strfiles = gen.dictKeysVals_stringList(Files)
     strtestfiles = gen.dictKeysVals_stringList(TestFiles)
-
-    print 'Lluis' + fname + ': strtestfile:', strtestfiles
 
     Svarcompute = ''
@@ -347 +345 @@
             TestFiles = gen.stringList_dictKeysVals(vals[1])
         elif vals[0] == 'varcompute:':
-            Vvals = vals[1].split('|')
-            if Vvals[3] == 'None':
-                mod = None
-            else:
-                mod = Vvals[3].split(':')
-            if Vvals[4] == 'None':
-                diag = None
-            else:
-                diag = Vvals[4].split(':')
-
-            allvarcomp[Vvals[0]+'_'+Vvals[1]] = [Vvals[2], mod, diag, Vvals[5]]
+            for VvalsS in vals[1].split(','):
+                
+                Vvals = VvalsS.split('|')
+                if Vvals[3] == 'None':
+                    mod = None
+                else:
+                    mod = Vvals[3].split(':')
+                if Vvals[4] == 'None':
+                    diag = None
+                else:
+                    diag = Vvals[4].split(':')
+
+                allvarcomp[Vvals[0]+'_'+Vvals[1]] = [Vvals[2], mod, diag, Vvals[5]]
         elif vals[0] == 'itotv:':
             ivop = int(vals[1])
@@ -462 +462 @@
                 values = '-f ' + cf + " -d '" + dims + "' -v '" + diagn + '|' +      \
                   diagc + "'"
-                sout = sub.call('python ' + pyH + '/diagnostics.py ' + values,       \
-                  shell=True)
+                try:
+                    with gen.Capturing() as output:
+                        sout = sub.call('python ' +pyH+ '/diagnostics.py ' + values, \
+                          shell=True)
+                except:
+                    print 'python ' + pyH + '/diagnostics.py ' + values
+                    for sout in output: print sout
+                    quit(-1)
+                if db:
+                    for sout in output: print sout
+
                 sout = sub.call(' mv diagnostics.nc ' + ifilen, shell=True)
 
@@ -479 +488 @@
                   ' to allow pressure interpolation'
                 for rqv in requiredinterpvars:
-                    ncvar.fvaradd(idir+'/'+cf+','+rqv,ifilen)
+                    try:
+                        with gen.Capturing() as output:
+                            ncvar.fvaradd(idir+'/'+cf+','+rqv,ifilen)
+                    except:
+                        print 'fvaradd('+idir+'/'+cf+','+rqv+','+ifilen+')'
+                        for sout in output: print sout
+                        quit(-1)
+                    if db:
+                        for sout in output: print sout
 
             # adding CF lon,lat,time in WRF files
@@ -495 +512 @@
             print '  ' + fname + ": concatenating all variable files to create '" +  \
               fileon + "..."
-        ncvar.netcdf_fold_concatenation_HMT('./,time', CFvarn+'_'+headerf+SgP+       \
-          '_,-,.nc', 'all')
+        try:
+            with gen.Capturing() as output:
+                ncvar.netcdf_fold_concatenation_HMT('./,time', CFvarn+'_'+headerf+   \
+                  SgP+'_,-,.nc', 'all')
+        except:
+            print 'netcdf_fold_concatenation_HMT(./time, '+CFvarn+'_'+headerf+SgP+   \
+              '_,-,.nc, all)'
+            for sout in output: print sout
+            quit(-1)
+        if db:
+            for sout in output: print sout
+
         sout = sub.call('mv netcdf_fold_concatenated_HMT.nc ' + fileon, shell=True)
         if os.path.isfile(fileon):
@@ -547 +574 @@
     vdnx = minf.vardxn
     vdny = minf.vardyn
+    # Model dimension-variable dictionary
+    Modeldimvardict = {dnx: vdnx, dny: vdny, dnz: vdnz, dnt: vdnt}
 
     # Some experiment configuration values
@@ -575 +604 @@
 
     # For that variables which require vertical interpolation 'p' suffix to the file
-    #   header is added
+    #   header is added. 
+    # After some operations some CF dimension-variables might not be kept
+
     if gP != 'none':
         SgP = 'p'
@@ -583 +614 @@
         varnCFs = ['lon', 'lat', 'time']
 
-    # Mandatory CF variables to be in all files
-    SvarnCFs = ',' + ','.join(varnCFs)
+    # CF dimension-variable dictionary
+    CFdimvardict = {'lon': 'lon', 'lat': 'lat', 'pres': 'pres', 'time': 'time'}
 
     # Input file
@@ -595 +626 @@
     wrongstats = []
 
-    # Variables to be kept in the final file
-    varkeep = []
-
     opers = Opers.split('+')
     Fopers = ''
-    if db: print '    computing statistics of variable:', CFvarn, ' ...'
+    if db: print "    computing statistics of variable:'", CFvarn, "' operation '" + \
+      Opers + "'...'"
     for op in opers:
 
-        # File name from previous operations
+        # File name from previous operations, and name of the variable within the 
+        #   file (some operations change the name of the variable)
         if op == opers[0]:
             Fopers = op
@@ -614 +644 @@
 
         # Adding required variables for the vertical interpolation in all operations
+        SvarnCFs = ',' + ','.join(varnCFs)
+
         if gP != 'none':
             if gP == 'local':
-                CFvarnp = vninF + ',P,PB,PSFC,PH,PHB,HGT,T,QVAPOR,XLONG,XLAT,Times' + SvarnCFs
+                CFvarnp = vninF + ',P,PB,PSFC,PH,PHB,HGT,T,QVAPOR,XLONG,XLAT,Times'+ \
+                  SvarnCFs
             else:
                 CFvarnp = vninF + SvarnCFs
@@ -626 +659 @@
 
         hprevfile = prevfile[0:len(prevfile)-3]
+
+        if db: print '      ', op, "using '" + prevfile + "' "
+        # Just produce file in case output file does not exist
         if not os.path.isfile(fileon):
-            if db: print '      ', op, "using '" + prevfile + "' "
+            if db: 
+                print '  ' + fname + ": creation of file '" + fileon + "' ..."
+            dofile = True
+        else:
+            dofile = False
         
-            if op == 'acc':
-                # temporal accumulated values
-                print "  " + fname + ": kind '" + op + "' not ready !!"
-                wrongstats.appen(CFvarn + '_' + opers)
-                break
-            elif op == 'direct':
-                # no statistics
+        # Variables to be kept in the final file
+        varkeep = []
+
+        if op == 'acc':
+            # temporal accumulated values
+            print "  " + fname + ": kind '" + op + "' not ready !!"
+            wrongstats.appen(CFvarn + '_' + opers)
+            break
+        elif op == 'direct':
+            # no statistics
+            if dofile:
                 sout = sub.call('mv ' + prevfile + ' ' + fileon, shell=True)
-            elif op == 'last':
-                # last temporal value
-                vals='time,-9,0,0'
-                with gen.Capturing() as output:
-                    ncvar.DataSetSection(vals,prevfile)
+        elif op == 'last':
+            # last temporal value
+            vals='time,-9,0,0'
+            if dofile:
+                try:
+                    with gen.Capturing() as output:
+                        ncvar.DataSetSection(vals,prevfile)
+                except:
+                    print 'DataSetSection('+vals+',', prevfile, ')'
+                    for sout in output: print sout
+                    quit(-1)
+
+                if db:
+                    for sout in output: print sout
                 
                 sout = sub.call('mv ' + hprevfile + '_time_B-9-E0-I0.nc ' + fileon, \
@@ -652 +705 @@
                 otrackf.write("# " + CFvarn + " " + Fopers + "\n")
                 otrackf.write(pyins + "\n")
-            elif op =='Lmean':
-                # latitudinal mean values
-                print "  " + fname + ": kind '" + op + "' not ready !!"
-                wrongstats.appen(CFvarn + '_' + opers)
-                break
-            elif op =='Lsec':
-                # latitudinal section (latitudinal value must be given, [var]@[lat]) 
-                print "  " + fname + ": kind '" + op + "' not ready !!"
-                wrongstats.appen(CFvarn + '_' + opers)
-                break
-            elif op =='lmean':
-                # longitudinal mean values
-                print "  " + fname + ": kind '" + op + "' not ready !!"
-                wrongstats.appen(CFvarn + '_' + opers)
-                break
-            elif op =='lsec':
-                # longitudinal section (longitudinal value must be given, [var]@[lon])
-                print "  " + fname + ": kind '" + op + "' not ready !!"
-                wrongstats.appen(CFvarn + '_' + opers)
-                break
-            elif op == 'pinterp':
-                # pinterp: pressure interpolation (to the given $plevels)
-                vals=plevels + ',1,1'
-                ncvar.pinterp(vals,prevfile,vninF)
+
+        elif op =='Lmean':
+            # latitudinal mean values
+            print "  " + fname + ": kind '" + op + "' not ready !!"
+            wrongstats.appen(CFvarn + '_' + opers)
+            break
+        elif op =='Lsec':
+            # latitudinal section (latitudinal value must be given, [var]@[lat]) 
+            print "  " + fname + ": kind '" + op + "' not ready !!"
+            wrongstats.appen(CFvarn + '_' + opers)
+            break
+        elif op =='lmean':
+            # longitudinal mean values
+            print "  " + fname + ": kind '" + op + "' not ready !!"
+            wrongstats.appen(CFvarn + '_' + opers)
+            break
+        elif op =='lsec':
+            # longitudinal section (longitudinal value must be given, [var]@[lon])
+            print "  " + fname + ": kind '" + op + "' not ready !!"
+            wrongstats.appen(CFvarn + '_' + opers)
+            break
+        elif op == 'pinterp':
+            # pinterp: pressure interpolation (to the given $plevels)
+            vals=plevels + ',1,1'
+            if dofile:
+                try:
+                    with gen.Capturing() as output:
+                        ncvar.pinterp(vals,prevfile,vninF)
+                except:
+                    print 'pinterp('+vals+',', prevfile, ','+vninF+')'
+                    for sout in output: print sout
+                    quit(-1)
+
+                if db:
+                    for sout in output: print sout
+
                 sout = sub.call('mv pinterp.nc ' + fileon, shell=True)
 
@@ -690 +755 @@
                     ncvar.WRF_toCF(values, fileon)
 
-                # vertical interpolation variables are no more needed
-                CFvarnp = vninF
-                gP = 'none'
-
-            elif op == 'tmean':
-                # temporal mean values
-                vals='time|-1,time,mean,lon:lat:' + vdnz + ':time:pres'
-                dims='time@time,' + dnz + '@' + vdnz + ',lat@lat,lon@lon'
-                ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
+            # vertical interpolation variables are no more needed
+            CFvarnp = vninF
+            gP = 'none'
+
+        elif op == 'tmean':
+            # temporal mean values
+            vals='time|-1,time,mean,' + ':'.join(varnCFs) + ':' + vdnz
+            dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
+            if dofile:
+                try:
+                    with gen.Capturing() as output:
+                        ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
+                except:
+                    print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
+                    for sout in output: print sout
+                    quit(-1)
+
+                if db:
+                    for sout in output: print sout
+
                 sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
 
@@ -708 +784 @@
                 otrackf.write(pyins + "\n")
 
-                # lon, lat CF-variables do not have time, thus they are not kept in
-                #   output
-                for rqv in varnCFs:
-                    if rqv != 'time':
-                        ncvar.fvaradd(prevfile+','+rqv,fileon)
-                varkeep.append('timestats')
-
-#            elif op == 'turb':
-#                # turbulence values
-#                vals='turbulence|'${CFvarn}
-#                dims='time@time,'${dnz}'@'${vdnz}',lat@lat,lon@lon,pres@pres'
-#                pyout=`python ${pyHOME}/diagnostics.py -d ${dims} -v ${vals} -f ${cfiles}`
-
-#                # Keeping the operations
-#                pyins="python "${pyHOME}"/diagnostics.py -d "${dims}" -v '"${vals}
-#                pyins=${pyins}"' -f ${cfiles}"
-#                echo " " >> ${odir}/all_statsvars.inf
-#                echo "# ${CFvarn}" "${vark}" >> ${odir}/all_statsvars.inf
-#                echo ${pyins} >> ${odir}/all_statsvars.inf
-
-#                varkeep=':'${CFvarn}'turb'
-
-            elif op == 'xmean':
-                # x-axis mean values
-                vals='lon|-1,lon,mean,lon:lat:' + vdnz + ':time:pres'
-                dims='time@time,' + dnz + '@' + vdnz + ',lat@lat,lon@lon'
-                ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
+            # removing dimension variable-dimension 'time'
+            varnCFs.remove('time')
+
+            varkeep.append('timestats')
+
+#        elif op == 'turb':
+#            # turbulence values
+#            vals='turbulence|'${CFvarn}
+#            dims='time@time,'${dnz}'@'${vdnz}',lat@lat,lon@lon,pres@pres'
+#            pyout=`python ${pyHOME}/diagnostics.py -d ${dims} -v ${vals} -f ${cfiles}`
+
+#            # Keeping the operations
+#            pyins="python "${pyHOME}"/diagnostics.py -d "${dims}" -v '"${vals}
+#            pyins=${pyins}"' -f ${cfiles}"
+#            echo " " >> ${odir}/all_statsvars.inf
+#            echo "# ${CFvarn}" "${vark}" >> ${odir}/all_statsvars.inf
+#            echo ${pyins} >> ${odir}/all_statsvars.inf
+
+#            varkeep=':'${CFvarn}'turb'
+
+        elif op == 'xmean':
+            # x-axis mean values
+            vals='lon|-1,lon,mean,' + ':'.join(varnCFs) + ':' + vdnz
+            dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
+            if dofile:
+                try:
+                    with gen.Capturing() as output:
+                        ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
+                except:
+                    print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
+                    for sout in output: print sout
+                    quit(-1)
+
+                if db:
+                    for sout in output: print sout
+
                 sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
 
@@ -744 +829 @@
                 otrackf.write(pyins + "\n")
 
-                # time CF-variable do not have 'x', thus they are not kept in
-                #   output
-                rqv = 'time'
-                ncvar.fvaradd(prevfile+','+rqv,fileon)
-                varkeep.append('lonstats')
-
-            elif op == 'ymean':
-                # y-axis mean values
-                vals='lat|-1,lat,mean,lon:lat:' + vdnz + ':time:pres'
-                dims='time@time,' + dnz + '@' + vdnz + ',lat@lat,lon@lon'
-                ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
+            varkeep.append('lonstats')
+
+        elif op == 'ymean':
+            # y-axis mean values
+            vals='lat|-1,lat,mean,' + ':'.join(varnCFs) + ':' + vdnz
+            dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
+            if dofile:
+                try:
+                    with gen.Capturing() as output:
+                        ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
+                except:
+                    print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
+                    for sout in output: print sout
+                    quit(-1)
+
+                if db:
+                    for sout in output: print sout
+
                 sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
 
@@ -764 +856 @@
                 otrackf.write(pyins + "\n")
 
-                # time CF-variable do not have 'y', thus they are not kept in
-                #   output
-                rqv = 'time'
-                ncvar.fvaradd(prevfile+','+rqv,fileon)
-                varkeep.append('latstats')
-            elif op == 'zsum':
-                # vertical aggregated values
-                print "  " + fname + ": kind '" + op + "' not ready !!"
-                wrongstats.append(CFvarn + '_' + opers)
-                break
-            else:
-                print errmsg
-                print '  ' + fname + ": operation '" + op + "' not ready !!"
-                quit(-1)
-
-            # End of kind of operation
-
-            # Variable name in file (vninF) changed due to operation 
-            #   but only if previous operation does not the same 'statistic'
-            chvn = gen.dictionary_key_list(Opsurs, op)
-            if chvn is not None:
-                oldvninF = vninF
-                if vninF.find(chvn) != -1:
-                    Lchvn = len(chvn)
-                    LvninF = len(vninF)
-                    if vninF[LvninF-Lchvn:LvninF-Lchvn+1] != chvn: 
-                        vninF = vninF + chvn
-                else:
-                    vninF = vninF + chvn
-                CFvarnp = CFvarnp.replace(oldvninF,vninF)
+            varkeep.append('latstats')
+
+        elif op == 'zsum':
+            # vertical aggregated values
+            print "  " + fname + ": kind '" + op + "' not ready !!"
+            wrongstats.append(CFvarn + '_' + opers)
+            break
+        else:
+            print errmsg
+            print '  ' + fname + ": operation '" + op + "' not ready !!"
+            quit(-1)
+
+        # End of kind of operation
+
+        # Variable name in file (vninF) changed due to operation 
+        #   but only if previous operation does not the same 'statistic'
+        chvn = gen.dictionary_key_list(Opsurs, op)
+        if chvn is not None:
+            oldvninF = vninF
+            vninF = vninF + chvn
+            CFvarnp = CFvarnp.replace(oldvninF,vninF)
                 
+        if dofile:
             if len(varkeep) > 0:
                 varkeepS = ',' + ','.join(varkeep)
@@ -800 +884 @@
                 varkeepS = ''
 
+            # Adding such CF dimension variables which might not be in the 
+            #   post-operation file
+            try:
+                with gen.Capturing() as output:  
+                    varinfile = ncvar.ivars(fileon)
+            except:
+                print 'ivars('+fileon+')'
+                for sout in output: print sout
+                quit(-1)
+
+            for CFvn in varnCFs:
+                if not gen.searchInlist(varinfile, CFvn):
+                    if db:
+                        print '  ' + fname + ": recupering CF variable '" + CFvn + "'"
+                    try:
+                        with gen.Capturing() as output:
+                            ncvar.fvaradd(prevfile+','+CFvn, fileon)
+                    except:
+                        print 'fvaradd(', prevfile, ','+CFvn+','+fileon+')'
+                        for sout in output: print sout
+                        quit(-1)
+
+                    if db:
+                        for sout in output: print sout
+
             totalvarkeeps = CFvarnp + ',' + ','.join(varnCFs) + varkeepS
-            oclean = ncvar.cleaning_varsfile(totalvarkeeps,fileon)
+            try:
+                with gen.Capturing() as output:
+                    oclean = ncvar.cleaning_varsfile(totalvarkeeps,fileon)
+            except:
+                print 'cleaning_varsfile('+totalvarkeeps+','+fileon+')'
+                for sout in output: print sout
+                quit(-1)
+
+            if db:
+                for sout in output: print sout
 
     # End of operations
+    if db:
+        print fname + ": successful calculation of '" + vninF + "' from '" +        \
+          CFvarn + "' '" + Opers + "' !!"
+
     if len(wrongstats) > 1:
         print warnmsg
@@ -966 +1088 @@
             files, testfiles, allcompvar, Nvar = read_varcomp_file(owdir +           \
               '/varcompute.inf')
+
         # End of avoiding to repeat all the experiment search
 
         print "  For experiment '"+exp+"' is required to compute:", Nvar, "variables"
+
+        if dbg:
+            print 'Variables to compute _______', type(allcompvar)
+            gen.printing_dictionary(allcompvar)
 
 ###

trunk/tools/model_graphics_template.dat

@@ -24 +24 @@
 
 # Scratch files: should be any files previous work removed and start from the scratch?
-filescratch = true
+filescratch = false
 
 # Scratch figures: should be any figures previous work removed and start from the scratch?
@@ -30 +30 @@
 
 # Have new files been added and do they need to be processed?
-addfiles = false
+addfiles = true
 
 # Have new figures been added and do they need to be drawn?
@@ -94 +94 @@
 VAR_xmean = tas:uas:vas:hfss:hfls:wss
 #VAR_last+pinterp+xmean = ta:ua:va:hur:hus:turb
-VAR_pinterp+tmean+xmean = ta:ua:va:hur:hus:turb
+VAR_pinterp+tmean+xmean = ta:ua:va:hur:hus
 
 #varlast = tas:uas:vas:ps:pr:pracc:rsds:prw:evspsbl:hfss:hfls:hurs:huss:zmla:hufs:wakes:lwakeh:stherm:zmaxth:clt:cll:clm:clh:prc:prls:bils:wss