Changeset 2219 in lmdz_wrf

Timestamp:

Nov 7, 2018, 4:47:40 PM (6 years ago)

Author:

lfita

Message:

Adding:

• `compute_slices_stats': Function to compute stats of variables of a file splitting variables by slices along sets of ranges for a series of variables

Location:

trunk/tools

Files:

• nc_var.py (modified) (4 diffs)
• nc_var_tools.py (modified) (2 diffs)

trunk/tools/nc_var.py

@@ -64 +64 @@
 ## e.g. # nc_var.py -o compute_slice2Dstats -S 'XLAT,-63.,19.,2.,HGT,250.,7000.,500.,Time|Times:west_east|XLONG:south_north|XLAT' -f wrfout_d01_1995-01-01_00:00:00 -v T2,Q2
 ## e.g. # nc_var.py -o same_deltasign -f wrfout_d01_1995-01-01_00:00:00 -S 'Time:0|south_north:60|west_east:-1' -v T2,Q2,XLAT,XLONG
+## e.g. # nc_var.py -o compute_slices_stats -S 'XLONG,-74.,-36.,4.;XLAT,-63.,19.,2.;HGT,250.,7000.,500.@Time|Times:west_east|XLONG:south_north|XLAT@Time' -f wrfout_d01_1995-01-01_00:00:00 -v T2,Q2
 
 from optparse import OptionParser
@@ -91 +92 @@
 # compute_opervaralltime: Function to compute opervaralltime: operation of variable successible allover the time-steps
 # compute_opervartimes: Function to compute opervartimes: operation of variable for a given sub-set of time-steps
+# compute_slices_stats: Function to compute stats of variables of a file splitting variables by 
+#   slices along sets of ranges for a series of variables
 # compute_slice2Dstats: Function to compute stats of variables of a file following slices along 2 variables
 # compute_tevolboxtraj: Function to compute tevolboxtraj: temporal evolution at a given point along a box following a trajectory
@@ -208 +211 @@
   'chgtimestep', 'chvarname', 'cleaning_varsfile', 'compute_deaccum',                \
   'compute_opersvarsfiles',                                                          \
-  'compute_opervaralltime', 'compute_opervartimes', 'compute_slice2Dstats',          \
+  'compute_opervaralltime', 'compute_opervartimes', 'compute_slices_stats',          \
+  'compute_slice2Dstats',                                                            \
   'compute_tevolboxtraj',                                                            \
   'computevar_model', 'curve_section', 'DatesFiles',                                 \
@@ -348 +352 @@
 elif oper == 'compute_opervartimes':
     ncvar.compute_opervartimes(opts.values, opts.ncfile, opts.varname)
+elif oper == 'compute_slices_stats':
+    ncvar.compute_slices_stats(opts.values, opts.ncfile, opts.varname)
 elif oper == 'compute_slice2Dstats':
     ncvar.compute_slice2Dstats(opts.values, opts.ncfile, opts.varname)

trunk/tools/nc_var_tools.py

@@ -46 +46 @@
 # compute_opervaralltime: Function to compute opervaralltime: operation of variable successible allover the time-steps
 # compute_opervartimes: Function to compute opervartimes: operation of variable for a given sub-set of time-steps
+# compute_slices_stats: Function to compute stats of variables of a file splitting variables by 
+#   slices along sets of ranges for a series of variables
 # compute_slice2Dstats: Function to compute stats of variables of a file following slices along 2 variables
 # computevar_model: Function to provide the way to compute a CF-variable providing its name
@@ -26436 +26438 @@
 #compute_slice2Dstats(values, '/home/lluis/PY/wrfout_d01_1995-01-01_00:00:00', 'T2,Q2')
 
+def compute_slices_stats(values, ncfile, variable):
+    """ Function to compute stats of variables of a file splitting variables by 
+        slices along sets of ranges for a series of variables
+      values=[slicevarsinf]@[dimvars]@[slicestatsdim]
+        [slicevarsinf] ';' separated list of [varn1],[minvar1],[maxvar1],[slcevar1];
+          ...;[varnN],[minvarN],[maxvarN],[slcevarN]
+          varn[i]: name of the variable i
+          minvar[i]: minimum value to start the slices for variable i
+          maxvar[i]: maximum value to end the slices for variable i
+          slcevar[i]: length of slices in variable i
+        [dimvars]: ':' separated list of [dimn]|[vardimn] dimension name [dimn] and 
+          its associated dimension-variable to get its values to provide values for 
+          the final file
+        [slicestatsdim]: ',' list of name of dimensions to do not use for computing 
+          statistics of variables at each slice ('any' for not leaving any 
+          dimension out, resulting on scalar statistics)
+      ncfile= netCDF file to use
+      variable: ',' list of variables ('all' for all variables)
+    """
+    fname = 'compute_slices_stats'
+
+    if values == 'h':
+        print fname + '_____________________________________________________________'
+        print compute_slices_stats.__doc__
+        quit()
+
+    expectargs = '[slicevarsinf]@[dimvars]@[slicestatsdim]'
+    gen.check_arguments(fname, values, expectargs, '@')
+
+    varvalues0 = values.split('@')[0]
+    dimvars0 = values.split('@')[1]
+    slicestatsdim = values.split('@')[2]
+
+    varvalues = varvalues0.split(';')
+
+    varvals = {}
+    slcvarns = []
+    for varvls in varvalues:
+        expectargs = '[varni],[minvari],[maxvari],[slcevari]'
+        gen.check_arguments(fname, varvls, expectargs, ',')
+
+        varni = varvls.split(',')[0]
+        minvari = np.float(varvls.split(',')[1])
+        maxvari = np.float(varvls.split(',')[2])
+        slicevari = np.float(varvls.split(',')[3])
+        varvals[varni] = [minvari, maxvari, slicevari]
+        slcvarns.append(varni)
+    Nslcvars = len(slcvarns)
+
+    dimvars = gen.stringS_dictvar(dimvars0, Dc=':', DVs='|')
+    
+    onc = NetCDFFile(ncfile, 'r')
+
+    # Varibales to slice
+    if variable == 'all':
+        varns = onc.variables.keys()
+    else:
+        varns = gen.str_list(variable, ',')
+
+    # Getting dimensions
+    for dn in dimvars.keys():
+        if not gen.searchInlist(onc.dimensions,dn):
+            print errormsg
+            print '  ' +fname+ ": file '" + ncfile + "' does not have dimension: '"+ \
+              dn + "' !!"
+            dimns = list(onc.dimensions)
+            dimns.sort()
+            print '    available ones:', dimns
+            quit(-1)
+
+    slcvarnS = ''
+    slicesinf = {}
+    # Preparing slices and output file
+    for varn in slcvarns:
+        if not onc.variables.has_key(varn):
+            print errormsg
+            print '  ' + fname + ": file '" + ncfile + "' does not have variable " + \
+              "i: '" + varn + "' !!"
+            varns = onc.variables.keys()
+            varns.sort()
+            print '    available ones:', varns
+            quit(-1)
+
+        ovar = onc.variables[varn]
+        vu = ovar.units
+        dimnv = list(ovar.dimensions)
+
+        vvls = varvals[varn]
+        minvar = vvls[0]
+        maxvar = vvls[1]
+        slicevar = vvls[2]
+
+        dvar = (maxvar-minvar+slicevar)/slicevar
+        slices = np.arange(minvar, maxvar+slicevar, slicevar)
+        Nslices = slices.shape[0]
+
+        print '    ' + fname + ': slices ____'
+        print '      var:', varn, ':', Nslices, '(', minvar, ',', maxvar+slicevar,   \
+          ',', slicevar, ')'
+
+        # Slices var
+        sliceshape = [Nslices] + list(ovar.shape)
+        slcvar = np.zeros(tuple(sliceshape), dtype=bool)
+        slcvalsc = np.zeros((Nslices), dtype=np.float)
+        slcvals = np.zeros((Nslices,2), dtype=np.float)
+        varv = ovar[:]
+        for islc in range(Nslices-1):
+            slcvar[islc,] = ma.masked_inside(varv, slices[islc], slices[islc+1]).mask
+            slcvalsc[islc] = (slices[islc+1]+slices[islc+1])/2.
+            slcvals[islc,0] = slices[islc]
+            slcvals[islc,1] = slices[islc+1]
+
+        slicesinf[varn] = [Nslices, dimnv, slcvar]
+
+        if varn == slcvarns[0]:
+            onewnc = NetCDFFile(fname + '.nc', 'w')
+            newdim = onewnc.createDimension('slice_bnds', 2)
+            slcvarnS = varn
+        else:
+            if varn == slcvarns[Nslcvars-1]: slcvarnS = slcvarnS + ' & ' + varn
+            else: slcvarnS = slcvarnS + ', ' + varn
+
+        # dimensions
+        newdim = onewnc.createDimension('slice_'+varn, Nslices)
+
+        # variable dimensions
+        for dn in dimnv:
+            if not gen.searchInlist(onewnc.dimensions,dn): add_dims(onc, onewnc, [dn])
+
+        newvar = onewnc.createVariable('slice_'+varn, 'f', ('slice_'+varn))
+        newvar[:] = slcvalsc[:]
+        basicvardef(newvar, 'slice_'+varn, 'slices for variable ' + varn, vu)
+
+        newvar = onewnc.createVariable('slice_'+varn+'_bnds', 'f', ('slice_'+varn,  \
+          'slice_bnds'))
+        newvar[:] = slcvals[:]
+        basicvardef(newvar, 'slice_'+varn+'_bnds', 'boundaries of slices for ' +     \
+          'variable ' + varn, vu)
+
+        slcdims = ['slice_'+varn] + dimnv
+        slcshape = [Nslices] + list(ovar.shape)
+        slcv = np.zeros(tuple(slcshape), dtype=int)
+        for islc in range(Nslices):
+            vvslc = slcvar[islc,]
+            slcv[islc,] = np.where(vvslc, 1, 0)
+        newvar = onewnc.createVariable(varn+'sliced', 'i', tuple(slcdims))
+        newvar[:] = slcv[:]
+        basicvardef(newvar, varn+'sliced', 'sliced variable ' + varn, vu)
+
+    for varn in varns:
+        print '    ' + varn + ' ...'
+        ovar = onc.variables[varn]
+        varv = ovar[:]
+        vdimns = list(ovar.dimensions)
+        vshape = list(ovar.shape)
+        vu = ovar.units
+
+        for dn in vdimns:
+            if not gen.searchInlist(onewnc.dimensions,dn): add_dims(onc, onewnc, [dn])
+
+        # mask in var slice
+        maskvarslcs = {}
+        allNslices = []
+        slicedimn = []
+        for vslcn in slcvarns:
+            varslcv = slicesinf[vslcn]
+            Nslices = varslcv[0]
+            ovardims = varslcv[1]
+            slcvar = varslcv[2]
+
+            NOTcoinc = list(set(ovardims) - set(vdimns))
+            if len(NOTcoinc) > 0:
+                notcoincdim = {}
+                for dn in NOTcoinc: notcoincdim[dn] = 0
+            else:
+                notcoincdim = None
+            maskvarslice = []
+            for islc in range(Nslices):
+                varmask = gen.mat_dimreshape(list(ovardims), slcvar[islc,], vdimns,  \
+                  vshape, notcoincdim)
+                maskvarslice.append(varmask)
+
+            maskvarslcs[vslcn] = maskvarslice
+            allNslices.append(Nslices)
+            slicedimn.append('slice_'+vslcn)
+
+        newvarshape = allNslices + vshape
+        print '    Shape final sliced variable:', newvarshape
+
+        # variables
+  
+        # This can get quite huge !!
+        #newvard = slicedimn + vdimns
+        #newvar = onewnc.createVariable(varn+'sliced', 'f', tuple(newvard),           \
+        #  fill_value=gen.fillValueF)
+        #basicvardef(newvar, varn+'sliced', varn + 'sliced by '+ slcvarnS, vu)
+        #add_varattrs(onc, onewnc, [varn], [varn+'sliced'])
+
+        masknodims = []
+        vrank = np.arange(len(vdimns))
+        if slicestatsdim == 'any':
+            slcmin = np.ones(tuple(allNslices), dtype=ovar.dtype)*gen.fillValueF
+            slcmax = np.ones(tuple(allNslices), dtype=ovar.dtype)*gen.fillValueF
+            slcmean = np.ones(tuple(allNslices), dtype=ovar.dtype)*gen.fillValueF
+            slcstd= np.ones(tuple(allNslices), dtype=ovar.dtype)*gen.fillValueF
+            newvard = slicedimn
+        else:
+            nodims = gen.str_list(slicestatsdim, ',')
+            stvardims = vdimns + []
+            stvdn = []
+            stvid = []
+            idd = 0
+            vrank = list(vrank)
+            for nod in nodims:
+                if not gen.searchInlist(onc.dimensions, nod):
+                    print errormsg
+                    print '  ' +fname+ ": file '" + ncfile + "' does not have " +    \
+                      "dimension: '" + dn + "' !!"
+                    dimns = list(onc.dimensions)
+                    dimns.sort()
+                    print '    available ones:', dimns
+                    quit(-1)
+
+                if gen.searchInlist(vdimns, nod):
+                    stvdn.append(nod)
+                    stvid.append(vshape[idd])
+                    stvardims.remove(nod)
+                    vrank.pop(idd)
+                    masknodims.append(slice(0,vshape[idd]))
+                idd = idd + 1
+            vrank = np.array(vrank)
+
+            slcmin = np.ones(tuple(allNslices+stvid), dtype=ovar.dtype)*gen.fillValueF
+            slcmax = np.ones(tuple(allNslices+stvid), dtype=ovar.dtype)*gen.fillValueF
+            slcmean = np.ones(tuple(allNslices+stvid), dtype=ovar.dtype)*gen.fillValueF
+            slcstd= np.ones(tuple(allNslices+stvid), dtype=ovar.dtype)*gen.fillValueF
+            newvard = slicedimn + stvdn
+
+        newvarN = onewnc.createVariable(varn+'Nsliced', 'f', tuple(newvard),         \
+          fill_value=gen.fillValueF)
+        basicvardef(newvarN, varn+'Nsliced','number of values in slice of ' + varn + \
+          'sliced by '+ slcvarnS, vu)
+        add_varattrs(onc, onewnc, [varn], [varn+'Nsliced'])
+
+        newvarn = onewnc.createVariable(varn+'minsliced', 'f', tuple(newvard),       \
+          fill_value=gen.fillValueF)
+        basicvardef(newvarn, varn+'minsliced', 'minimum value of '+varn+'sliced by '+\
+          slcvarnS, vu)
+        add_varattrs(onc, onewnc, [varn], [varn+'minsliced'])
+
+        newvarx = onewnc.createVariable(varn+'maxsliced', 'f', tuple(newvard),       \
+          fill_value=gen.fillValueF)
+        basicvardef(newvarx, varn+'maxsliced', 'maximum value of '+varn+'sliced by '+\
+          slcvarnS, vu)
+        add_varattrs(onc, onewnc, [varn], [varn+'maxsliced'])
+
+        newvarm = onewnc.createVariable(varn+'meansliced', 'f', tuple(newvard),      \
+          fill_value=gen.fillValueF)
+        basicvardef(newvarm, varn+'meansliced', 'mean value of '+varn+'sliced by ' + \
+          slcvarnS, vu)
+        add_varattrs(onc, onewnc, [varn], [varn+'meansliced'])
+
+        newvars = onewnc.createVariable(varn+'stdsliced', 'f', tuple(newvard),       \
+          fill_value=gen.fillValueF)
+        basicvardef(newvars, varn+'stdsliced', 'standard deviation of '+varn+        \
+          'sliced by ' + slcvarnS, vu)
+        add_varattrs(onc, onewnc, [varn], [varn+'stdsliced'])
+
+        # Obviously we get memory problems... proceed slide by slide to fill the file
+        #newvarNmasked = np.ones(tuple(newvarshape), dtype=ovar.dtype)*gen.fillValueF
+        newvarNmasked = np.ones(tuple(vshape), dtype=ovar.dtype)*gen.fillValueF
+
+        # Masking!
+        allslices = gen.all_consecutive_combs(allNslices)
+        Nallslices = len(allslices)
+
+        for iallslc in range(Nallslices):
+            islcN = allslices[iallslc]
+            slcnewvar = []
+            iv = 0
+            for vslcn in slcvarns:
+                masksv = maskvarslcs[vslcn]
+                slcnewvar.append(islcN[iv])
+                if iv == 0: newmask = masksv[islcN[iv]]
+                else: newmask = newmask*masksv[islcN[iv]]
+                iv = iv + 1
+
+            for iid in vshape:
+               slcnewvar.append(slice(0,iid))
+
+            mavals = ma.array(varv, mask=~newmask)
+            #newvarNmasked[tuple(slcnewvar)] = mavals.filled(gen.fillValueF)
+            newvarNmasked[:] = mavals.filled(gen.fillValueF)
+
+            # This can get quite huge !!!
+            #newvar[tuple(slcnewvar)] = newvarNmasked[:]
+
+            manewvar = ma.masked_equal(newvarNmasked, gen.fillValueF)
+ 
+            if len(masknodims) > 0: 
+                islcst = list(islcN) + list(masknodims)
+                if iallslc == 0:
+                    print '  variable slice statisitcs not using:', nodims
+                    print '    size statistics variables:', allNslices+stvid
+                    print '    slice statistics variables:', islcst
+                    print '    running shapes of the variable:', vrank
+            else: islcst = islcN
+
+            newvarN[tuple(islcst)]=np.sum(~manewvar.mask, axis=tuple(vrank))
+
+            slcmin[tuple(islcst)] = np.min(manewvar, axis=tuple(vrank))
+            slcmax[tuple(islcst)] = np.max(manewvar, axis=tuple(vrank))
+            slcmean[tuple(islcst)] = np.mean(manewvar, axis=tuple(vrank))
+            slcstd[tuple(islcst)] = np.std(manewvar, axis=tuple(vrank))
+
+        newvarn[:] = slcmin
+        newvarx[:] = slcmax
+        newvarm[:] = slcmean
+        newvars[:] = slcstd
+
+        onewnc.sync()
+        # Adding dimension-variables
+        for dn in vdimns:
+            if not dimvars.has_key(dn):
+                print errormsg
+                print '  ' + fname + ": no dimension-variable provided for " +       \
+                  "dimension '" + dn + "' !!"
+                print '    provided ones _______'
+                gen.printing_dictionary(dimvars)
+                quit(-1)
+            if not onewnc.variables.has_key(dimvars[dn]): 
+                add_vars(onc, onewnc, [dimvars[dn]])
+                onewnc.sync()
+
+    # Add global attributes
+    add_global_PyNCplot(onewnc, 'nc_var_tools', fname, '1.0')
+    add_globattrs(onc,onewnc,'all')
+    onewnc.sync()
+    onc.close()
+    onewnc.close()
+    print fname + ": successfull written of file '" + fname + ".nc' !!"
+
+    return
+
+#values='XLONG,-74.,-36.,4.;XLAT,-63.,19.,4.;HGT,250.,7000.,500.@Time|Times:west_east|XLONG:south_north|XLAT@Time'
+#compute_slices_stats(values, '/home/lluis/PY/wrfout_d01_1995-01-01_00:00:00', 'T2,Q2')
+
 def same_deltasign(values, ncfile, variable):
     """ Function to determine if a given series of 1D values share the same sign of