Changeset 1499 in lmdz_wrf

Timestamp:

Apr 8, 2017, 9:59:57 PM (8 years ago)

Author:

lfita

Message:

Improving and adding documentation for `DataSetSection_multivars'

Location:

trunk/tools

Files:

• documentation/ncmanage/DataSetSection_multivars.html (added)
• documentation/ncmanage/ncmanage.html (modified) (1 diff)
• nc_var_tools.py (modified) (11 diffs)

trunk/tools/documentation/ncmanage/ncmanage.html

@@ -11 +11 @@
       <A CLASS="lc" HREF="compute_opersvarsfiles.html" TARGET="value">compute_opersvarsfiles</A><BR>
       <A CLASS="lc" HREF="curve_section.html" TARGET="value">curve_section</A><BR>
+      <A CLASS="lc" HREF="DataSection_multivars.html" TARGET="value">DataSection_multivars</A><BR>
       <A CLASS="lc" HREF="field_stats.html" TARGET="value">field_stats</A><BR>
       <A CLASS="lc" HREF="lonlat_polygon.html" TARGET="value">lonlat_polygon</A><BR>
       <A CLASS="lc" HREF="maskvar.html" TARGET="value">maskvar</A><BR>
       <A CLASS="lc" HREF="setvar_asciivalues.html" TARGET="value">setvar_asciivalues</A><BR>
-      <A CLASS="lc" HREF="varaddattr.html" TARGET="value">varaddattr</A><BR>
-      <A CLASS="lc" HREF="varaddattrk.html" TARGET="value">varaddatrk</A><BR>
+      <A CLASS="lc" HREF="addvarattr.html" TARGET="value">varaddattr</A><BR>
+      <A CLASS="lc" HREF="addvarattrk.html" TARGET="value">varaddatrk</A><BR>
     </DIV>
     <DIV CLASS="valmenu">

trunk/tools/nc_var_tools.py

@@ -169 +169 @@
 errormsg = 'ERROR -- error -- ERROR -- error'
 warnmsg = 'WARNING -- warning -- WARNING -- warning'
+infmsg = 'INFORMATION -- information -- INFORMATION -- information'
 
 ## Constants
@@ -13806 +13807 @@
       vn= variable name to include
       sliced= dictionary with the dimension name as key and [beg, end, int] as values
+        if beg=-9, means that values for the dimension corresponds for a series of individual values
     """
     fname = 'put_variable_slice'
@@ -13819 +13821 @@
     dnvs = []
     varslice = []
+
+    # Dictionary with the size of the sliced dimensions
+    newdimsizes = {}
 # Getting variable's dimensions
+    Nslices = 1
     for dnv in ov.dimensions:
         if gen.searchInlist(sliced.keys(),dnv):
-            newdimsize = sliced[dnv][1] - sliced[dnv][0]
-        else:
-            newdimsize = 10
+            if sliced[dnv][0] == -9:
+                newdimsize = len(sliced[dnv])-1
+                Nslices = newdimsize
+            else:
+                if sliced[dnv][2] != 0:
+                    newdimsize = len(range(sliced[dnv][0], sliced[dnv][1],           \
+                      sliced[dnv][2]))
+                else:
+                    newdimsize = 0
+            newdimsizes[dnv] = newdimsize
+            # Creation of dimensions according to the slice
+            if not gen.searchInlist(onc.dimensions, dnv):
+                od = inc.dimensions[dnv]
+                if od.isunlimited():
+                    newdim = onc.createDimension(dnv, None)
+                else:
+                    if newdimsize > 0: newdim = onc.createDimension(dnv, newdimsize)
         if not gen.searchInlist(onc.dimensions,dnv):
-# Avoiding montone dimensions!
+# Avoiding monotone dimensions!
             if newdimsize > 0:
                 od = inc.dimensions[dnv]
@@ -13834 +13854 @@
                     dsize = len(od)
                 newdim = onc.createDimension(dnv, dsize)
-        if gen.searchInlist(sliced.keys(),dnv):
-            if newdimsize > 0:
-                varslice.append(slice(sliced[dnv][0], sliced[dnv][1], sliced[dnv][2]))
-                dnvs.append(dnv)
-            else:
-                varslice.append(sliced[dnv][0])
-        else:
-            varslice.append(slice(0,len(inc.dimensions[dnv])))
-            dnvs.append(dnv)
-# Getting variable
+            newdimsizes[dnv] = newdimsize
+
+    # Getting variable
     varattrs = ov.ncattrs()
     if gen.searchInlist(varattrs, '_FillValue'):
@@ -13851 +13864 @@
     vartype = ov.dtype
 
-    newvar = onc.createVariable(vn, vartype, tuple(dnvs), fill_value=varfil)
-    finalvals = np.squeeze(ov[tuple(varslice)])
-    newvar[:] = finalvals
+    # List of slices
+    for idind in range(Nslices):
+        newvarslice = []
+        for dnv in ov.dimensions:
+            if gen.searchInlist(sliced.keys(),dnv):
+                if sliced[dnv][0] != -9:
+                    if newdimsizes[dnv] > 0:
+                        varslice.append(slice(sliced[dnv][0], sliced[dnv][1],        \
+                          sliced[dnv][2]))
+                        dnvs.append(dnv)
+                        newvarslice.append(slice(sliced[dnv][0], sliced[dnv][1],     \
+                          sliced[dnv][2]))
+                    else:
+                        varslice.append(sliced[dnv][0])
+                else:
+                    varslice.append(sliced[dnv][idind])
+                    newvarslice.append(idind)
+            else:
+                varslice.append(slice(0,len(inc.dimensions[dnv])))
+                dnvs.append(dnv)
+                newvarslice.append(slice(0,len(inc.dimensions[dnv])))
+
+        if idind == 0:
+            newvar = onc.createVariable(vn, vartype, tuple(dnvs), fill_value=varfil)
+ 
+        finalvals = np.squeeze(ov[tuple(varslice)])
+        newvar[tuple(newvarslice)] = finalvals
 
     for attrs in varattrs:
@@ -13865 +13902 @@
     """ Function to get a section (values along multiple variables) of a given data-set
       values= [varn1],[beg1],[end1],[int1]@[...[[varnM],[begM],[endM],[intM]]]
-        [varni]: name of the variable ('WRFtime', for WRF time varibale)
+        [varni]: name of the variable with exception of:
+          'dist;[var1],[val1]|[...,[varN],val2]] distance computed from sqrt[sum_i([vari]-[vali])^2)]
+             variables have to be of the same shape and share dimensions
         [begi],[endi],[inti]: beginning, end and interval along the variable
           [endi] = -1, maximum value
-          [inti] = -1, all the values within the range
+          [inti] = -1, all the values within the range (default)
                  = 0, single value. [begi] will be taken as the reference value
           NOTE: variables without section by the variables are taken allover their size
           NOTE2: if inti = 'str', it is assumed that the variable is a string-list of values
-            then [begi] must be the string to be used ('!', for spaces)
+            then [begi] must be the string to be used for the search ('!', for spaces)
       filen= netCDF with the data-set
       varn= ',' list of variables ('all', for all variables)
     """
+# Not working          'WRFtime', for WRF time varibale
+
     import numpy.ma as ma
     fname = 'DataSetSection_multivars'
 
 # Variables not to check in the file
-    NOcheck = ['WRFtime']
+    NOcheck = ['WRFtime','dist']
 
     if values == 'h':
@@ -13904 +13945 @@
     fintvs = np.zeros((Nvars), dtype=np.float)
 
+    # Dictionary with the indices with coincident values 
+    dimvals = {}
+
     for ivar in range(Nvars):
         val = values.split('@')[ivar]
         vn = val.split(',')[0]
-        if not gen.searchInlist(nciobj.variables,vn):
+        if not gen.searchInlist(nciobj.variables,vn) and                             \
+          not gen.searchInlist(NOcheck,vn) and vn[0:4] != 'dist':
             print errormsg
             print '  ' + fname + ": file '" + filen + "' does not have variable '" + \
@@ -13913 +13958 @@
             print '    it has:', list(nciobj.variables)
             quit(-1)
-
-        ostrvar = nciobj.variables[vn]
-        varns.append(vn)
-        if val.split(',')[3] == 'str':
-            strvalue = val.split(',')[1].replace('!',' ')
+        if val[0:4] == 'dist':
+            # Selection by distance
+            varns.append('dist')
+            vardist = val.split(';')[1].split('|')
+            varsdist = {}
+            iiv = 0
+            distv = 0.
+            # Computing distance and looking for the minimal distance
+            for vdn in vardist:
+                dvarn = vdn.split(',')[0]
+                dvalS = vdn.split(',')[1]
+                if not gen.searchInlist(nciobj.variables,dvarn):
+                    print errormsg
+                    print '  '+fname+": file '"+filen+"' does not have variable '" + \
+                      dvarn + "' !!"
+                    print '    it has:', list(nciobj.variables)
+                    quit(-1)
+                oval = nciobj.variables[dvarn]
+                Vvals = oval[:]
+                dval = gen.retype(dvalS,oval.dtype)
+
+                if iiv == 0:
+                    valsfirst = Vvals
+                    valsdims = oval.dimensions
+                    if dvalS[0:1] == '-':
+                        LdvalS = len(dvalS)
+                        distS = '('+ dvarn + '+' + dvalS[1:LdvalS+1] + ')^2'
+                    else:
+                        distS = '('+ dvarn + '-' + dvalS + ')^2'
+                    distv = (Vvals - dval)**2
+                else:
+                    # Checking for consistency among 
+                    gen.same_shape(valsfirst, Vvals)
+                    if dvalS[0:1] == '-':
+                        LdvalS = len(dvalS)
+                        distS = distS + ' + ('+ dvarn + '+' + dvalS[1:LdvalS+1] + ')^2'
+                    else:
+                        distS = distS + ' + ('+ dvarn + '-' + dvalS + ')^2'
+
+                    distv = distv + (Vvals - dval)**2
+                iiv = iiv + 1
+            mindist = np.min(distv)
+            iDistmindist = gen.multi_index_mat(distv, mindist)
+
+            print infmsg
+            print '  ' + fname + ": found a distance searching as dist=sqrt(" +      \
+              distS + ")"
+            varsdist[dvarn] = dval
+            print '    found:', len(iDistmindist),'values at distance:', mindist
+            iid = 0
+            for dimn in valsdims:
+                dvvs = []
+                for ifound in range(len(iDistmindist)):
+                    foundvals = iDistmindist[ifound]
+                    if len(dvvs) == 0: dvvs = [int(foundvals[iid])]
+                    # getting indices for a given dimension including non-repeated
+                    if not gen.searchInlist(dvvs, foundvals[iid]):
+                        dvvs.append(int(foundvals[iid]))
+                iid = iid + 1
+                # Including only that indices coincident with previous founds
+                if dimvals.has_key(dimn):
+                    foundvals = dimvals[dimn]
+                    coincvals = list(set(foundvals).intersection(set(dvvs)))
+                    if len(coincvals) == 0:
+                        print errormsg
+                        print '  ' + fname + ': no coincidencies among searches ' + \
+                          "found for dimension '" + dimn + "' !!"
+                        print '    revise criteria of selection of values'
+                        quit(-1)
+                    else:
+                        dimvals[dimn] = coincvals
+                else:
+                    dimvals[dimn] = dvvs
+              
+        elif val.split(',')[3] == 'str':
+            # String search
+            ostrvar = nciobj.variables[vn]
+            ostrvardims = ostrvar.dimensions
+            varns.append(vn)
+            istrvalue = val.split(',')[1].replace('!',' ')
+            estrvalue = val.split(',')[2].replace('!',' ')
             print warnmsg
             print '  ' + fname + ": String value for variable '" + val + "' !!"
             print '    assuming string variable as [Nvalues, Lstring]'
-            print "    from which only is required '" + strvalue + "'"
+            if istrvalue != estrvalue:
+                print "    from which is required from '" + istrvalue + "' to '" +   \
+                  estrvalue + "'"
+            else:
+                print "    from which only is required '" + istrvalue + "'"
             strvals = get_str_nc(ostrvar, ostrvar.shape[1])
-            if not gen.searchInlist(strvals, strvalue):
+            if not gen.searchInlist(strvals, istrvalue):
                 print errormsg
                 print '  ' + fname + ": variable '" + val + "' does not have " +     \
-                  "value= '" + strvalue + "' !!"
+                  "value= '" + istrvalue + "' !!"
                 print '    values:', strvals
                 quit(-1)
-            indstrvalue = strvals.index(strvalue)
+            if istrvalue != estrvalue:
+                if not gen.searchInlist(strvals, estrvalue):
+                    print errormsg
+                    print '  ' + fname + ": variable '" + val + "' does not have " + \
+                      "value= '" + estrvalue + "' !!"
+                    print '    values:', strvals
+                    quit(-1)
+
+            dimvals[ostrvardims[1]] = range(ostrvar.shape[1])
+            dimn = ostrvardims[0]
+            if istrvalue == estrvalue:
+                Nfound = strvals.count(istrvalue)
+                print infmsg
+                print '  '+fname+ ": variable '" + vn + "' has:", Nfound, "times " + \
+                  "value '" + istrvalue + "'"
+
+                # Values found for each dimension
+
+                # Assuming shape as said
+                copyvals = list(strvals)     
+                dvvs = []
+                for iif in range(Nfound):
+                    iidstr = copyvals.index(strvalue)
+                    dvvs.append(iidstr+iif)
+                    copyvals.pop(iidstr)
+            else:
+                ibeg = strvals.index(istrvalue)
+                iend = strvals.index(estrvalue)
+                dvvs = range(ibeg,iend)
+
+            # Including only that indices coincident with previous founds
+            if dimvals.has_key(dimn):
+                foundvals = dimvals[dimn]
+                coincvals = list(set(foundvals).intersection(set(dvvs)))
+                if len(coincvals) == 0:
+                    print errormsg
+                    print '  ' + fname + ': no coincidencies among searches found '+ \
+                      "for dimension '" + dimn + "' !!"
+                    print '    revise criteria of selection of values'
+                    quit(-1)
+                else:
+                    dimvals[dimn] = coincvals
+
+            indstrvalue = strvals.index(istrvalue)
+            endstrvalue = strvals.index(estrvalue)
             begvs[ivar] = np.float(indstrvalue)
             endvs[ivar] = np.float(indstrvalue)
             intvs[ivar] = -99.
         else:
-            begvs[ivar] = np.float(val.split(',')[1])
-            endvs[ivar] = np.float(val.split(',')[2])
-            intvs[ivar] = np.float(val.split(',')[3])
+            # Range search
+            ovar = nciobj.variables[vn]
+            vals = ovar[:]
+            valsdims = ovar.dimensions
+            varns.append(vn)
+            begvs[ivar] = gen.retype(val.split(',')[1], ovar.dtype)
+            endvs[ivar] = gen.retype(val.split(',')[2], ovar.dtype)
+            intvs[ivar] = -1
             fbegvs[ivar] = np.float(val.split(',')[1])
             fendvs[ivar] = np.float(val.split(',')[2])
             fintvs[ivar] = np.float(val.split(',')[3])
+
+            if endvs[ivar] == -1: endvs[ivar] = np.max(vals)
+            maskinf = ma.masked_less(vals, begvs[ivar])
+            masksup = ma.masked_greater(vals, endvs[ivar])
+
+            #if intvs[ivar] != -1:
+            #    print errormsg
+            #    print '  ' + fname + ': non-consecutive slices not ready!!'
+            #    quit(-1)
+            finalmask = maskinf.mask + masksup.mask
+            print 'Lluis: found;', np.sum(~finalmask)
+            # Indices of the found values
+            iDistmindist = gen.multi_index_mat(finalmask, False)
+
+            print '    found:', len(iDistmindist),"values in variable '" + vn +      \
+              "' as; beginning:", begvs[ivar], 'ending:', endvs[ivar], 'interval:',  \
+              intvs[ivar]
+            iid = 0
+            for dimn in valsdims:
+                dvvs = []
+                for ifound in range(len(iDistmindist)):
+                    foundvals = iDistmindist[ifound]
+                    # getting indices for a given dimension
+                    dvvs.append(int(foundvals[iid]))
+                iid = iid + 1
+                # Including only that indices coincident with previous founds
+                if dimvals.has_key(dimn):
+                    foundvals = dimvals[dimn]
+                    coincvals = list(set(foundvals).intersection(set(dvvs)))
+                    if len(coincvals) == 0:
+                        print errormsg
+                        print '  ' + fname + ': no coincidencies among searches ' + \
+                          "found for dimension '" + dimn + "' !!"
+                        print '    revise criteria of selection of values'
+                        quit(-1)
+                    else:
+                        dimvals[dimn] = coincvals
+                else:
+                    dimvals[dimn] = dvvs
+
             if intvs[ivar] != -1:
                 slicevalS = slicevalS + varns[ivar] + ' (' + str(fbegvs[ivar]) + ',' +   \
@@ -13947 +14161 @@
                   str(fendvs[ivar]) + ',1); '
 
+    print infmsg
+    print '  ' + fname + ': Indices found for each dimensions ____'
+    finalslice = {}
+    for dimn in dimvals.keys():
+        ddvals = dimvals[dimn]
+        print '    ' + dimn + ':', ddvals
+        if len(dimvals[dimn]) > 1:
+            Lddvals = len(ddvals)
+            if np.all(np.array(ddvals[1:Lddvals])-np.array(ddvals[0:Lddvals-1])) == 1:
+                finalslice[dimn] = [ddvals[0], ddvals[Lddvals-1], 1]
+            else:
+                finalslice[dimn] = [-9] + ddvals
+        else:
+            finalslice[dimn] = [ddvals[0], ddvals[0], 0]
+
+    print 'Lluis finalslice:', finalslice
+
     ncoobj = NetCDFFile(ofile,'w')
-
 # Output variables
 ##
@@ -13959 +14189 @@
             variables = [varn]
 
-# Looking for limits due to the variables
-##
-    slicedims = {}
-    for ivar in range(Nvars):
-        print '  ' + fname + ": masking with '" + varns[ivar] + "':",  begvs[ivar],  \
-          ',', endvs[ivar], ',', intvs[ivar]
-        if not gen.searchInlist(nciobj.variables,varns[ivar]) and not                \
-          gen.searchInlist(NOcheck,varns[ivar]):
+    for varn in variables:
+        if not gen.searchInlist(nciobj.variables, varn):
             print errormsg
-            print ' ' + fname + ": variable '" + varns[ivar] + "' is not in ' +      \
-              'input file!!"
+            print '  ' + fname + ": file '" + filen + "' does not have variable '" + \
+              dvarn + "' !!"
+            print '    it has:', list(nciobj.variables)
             quit(-1)
 
-        if varns[ivar] == 'WRFtime':
-            ovar = nciobj.variables['Times'][:]
-            vals = []
-            for it in range(ovar.shape[0]):
-                vals.append(datetimeStr_conversion(ovar[it,:], 'WRFdatetime',        \
-                  'YmdHMS'))
-                vals[it] = int(vals[it])
-                begvs[ivar] = int(begvs[ivar])
-                endvs[ivar] = int(endvs[ivar])
-                intvs[ivar] = int(intvs[ivar])
-
-            vals = np.array(vals)
-            vardims = ['Time']
-        else:
-            ovar = nciobj.variables[varns[ivar]]
-            vals = ovar[:]
-            vardims = ovar.dimensions
-
-        if intvs[ivar] == -99.:
-            ostrvar = nciobj.variables[varns[ivar]]
-            finalmask = np.ones((ostrvar.shape), dtype= bool)
-            finalmask[int(begvs[ivar]),:] = False
-        elif intvs[ivar] == 0.:
-#           Looking in the non efficient way
-            diffvar = np.abs(vals-fbegvs[ivar])
-            mindiffvar = np.min(diffvar)
-            imindiff = gen.index_mat(diffvar, mindiffvar)
-            if np.all(imindiff == -1): 
-                print errmsg
-                print '  ' + fname + ": no exact value '" + varns[ivar] + "'=",      \
-                  fbegvs[ivar], 'has been found!!'
-                quit(-1)
-            print '  ' + fname + ": '" + varns[ivar] + "'= ", fbegvs[ivar],          \
-              'indexes:', imindiff, 'distance:', mindiffvar
-            finalmask = np.ones(vals.shape, dtype=np.float)
-           
-            finalmask[tuple(imindiff)] = False
-        else:
-            if endvs[ivar] == -1: endvs[ivar] = np.max(vals)
-            maskinf = ma.masked_less(vals, begvs[ivar])
-            masksup = ma.masked_greater(vals, endvs[ivar])
-
-            if intvs[ivar] != -1:
-                print errormsg
-                print '  ' + fname + ': non-consecutive slices not ready!!'
-                quit(-1)
-            finalmask = maskinf.mask + masksup.mask
-            idn = 0
-        
-        iidim = 0
-        for dn in vardims:
-            ddn = len(nciobj.dimensions[dn])
-            dinds = np.arange(ddn)
-            rightvals = ~ma.array(vals, mask=finalmask).mask
-
-# Checking if dimension is already cut:
-            if slicedims.has_key(dn):
-                slicedval = slicedims[dn]
-                if slicedval[0] == 0 and slicedval[1] == ddn: 
-                    fulldim = True
-                else:
-                    fulldim = False
-            else:
-                fulldim = True
-
-            if not slicedims.has_key(dn) or fulldim:
-                if intvs[ivar] == -99.:
-                    if iidim == 0:
-                        slicedims[dn]=[int(begvs[ivar]), int(begvs[ivar]), None]
-                    else:
-                        slicedims[dn]=[0, ddn, None]
-                elif intvs[ivar] == 0.:
-                    if imindiff[iidim] == 0:
-                        slicedims[dn]=[0, ddn, None]
-                    else:
-                        slicedims[dn]=[imindiff[iidim], imindiff[iidim], None]
-                else:
-                    slicedims[dn]=[np.min(dinds), np.max(dinds)+1, None]
-
-            iidim = iidim + 1
-
-        print '  ' + fname + ': slicing variables with______'
-        print slicedims
- 
-# Creating dimensions
-## 
-    print '  ' + fname + ': adding dimensions...'
-    for nd in slicedims.keys():
-        objdim = nciobj.dimensions[nd]
-        if objdim.isunlimited():
-            dimsize = None
-        else:
-            dimsize = slicedims[nd][1] - slicedims[nd][0]
-        if dimsize > 0: newdim = ncoobj.createDimension(nd, dimsize)
-    ncoobj.sync()
-
-    for var in variables:
-        put_variable_slice(nciobj, ncoobj, var, slicedims)
+        ovar = nciobj.variables[varn]
+        vardims = ovar.dimensions
+
+        put_variable_slice(nciobj, ncoobj, varn, finalslice)
 
 # Global attributes
@@ -14078 +14209 @@
 
     attr = set_attribute(ncoobj, 'sliced_variables', slicevalS)
+    add_global_PyNCplot(ncoobj, main, fname, '1.1')
 
     nciobj.close()