Changeset 2343 in lmdz_wrf

Timestamp:

Feb 15, 2019, 8:58:17 PM (6 years ago)

Author:

lfita

Message:

Adding:

• `except_fillValue': Function to fill a file with fill_Value except a given value of a variable
  • Adding selection of kind of filling outside selection

Location:

trunk/tools

Files:

• nc_var.py (modified) (4 diffs)
• nc_var_tools.py (modified) (6 diffs)

trunk/tools/nc_var.py

@@ -71 +71 @@
 ## e.g. # nc_var.py -o compute_slices_stats_areaweighted -S 'XLONG,-74.,-36.4,4.;XLAT,-63.,19.,4.;HGT,500.,7000.,500.@Time|WRFtime:west_east|XLONG:south_north|XLAT@Time@west_east|lon_bnds,south_north|lat_bnds@XLONG|lat_bnds;lon_bnds,XLAT|lat_bnds;lon_bnds@Time' -f wrfout_bnds.nc -v T2,U10,V10,Q2
 ## e.g. # nc_var.py -o compute_slices_stats_areaweighted -S 'lat,-63.,19.,2.;orog,500.,7000.,500.;rangefaces,fixed,-2.5|-0.5,-0.5|0.5,0.5|2.5@time|time:lon|lon:lat|lat@time@lon|lon_bnds,lat|lat_bnds@lon|lon_bnds,lat|lat_bnds@lat,lon@time' -f /media/lluis/ExtDiskC_ext3/DATA/estudios/Andes/DATA/concatenated/historical/tasmin/tasmin_Amon_ACCESS1-0_historical_r1i1p1_185001-200512_Andes_19600101000000-19900101000000.nc -v tasmin
+## e.g. # nc_var.py -o except_fillValue -S 'orog:range,500.,7000.:None' -f /home/lluis/estudios/Andes/cmip_data/fx/orog_fx_ACCESS1-0_historical_r0i0p0.nc -v 'all'
 
 from optparse import OptionParser
@@ -118 +119 @@
 # dimToUnlimited: Operation to create an unlimited dimension from an existing one
 # dimVar_creation: Function to add a 1D variable with the size of a given dimension in a file
+# except_fillValue: Function to fill a file with fill_Value except a given value of a variable
 # fattradd: Adding attributes from a reference file
 # fdimadd: Adding dimension from another reference file 
@@ -229 +231 @@
   'DataSetSection', 'DataSetSection_multidims', 'DataSetSection_multivars',          \
   'DYNAMICO_toCF', 'dimrename', 'dimrm', 'dimToUnlimited', 'dimVar_creation',        \
-  'fattradd',                                                                        \
+  'except_fillValue', 'fattradd',                                                    \
   'fdimadd', 'fgaddattr', 'field_stats', 'field_stats_dim', 'file_creation',         \
   'file_oper_alongdims', 'filter_2dim',                                              \
@@ -397 +399 @@
 elif oper == 'dimVar_creation':
     ncvar.dimVar_creation(opts.values, opts.ncfile)
+elif oper == 'except_fillValue':
+    ncvar.except_fillValue(opts.values, opts.ncfile, opts.varname)
 elif oper == 'fattradd':
     ncvar.fattradd(var, opts.values, opts.ncfile)

trunk/tools/nc_var_tools.py

@@ -75 +75 @@
 # dimToUnlimited: Operation to create an unlimited dimension from an existing one
 # dimVar_creation: Function to add a 1D variable with the size of a given dimension in a file
+# except_fillValue: Function to fill a file with fill_Value except a given value of a variable
 # fattradd: Adding attributes from a reference file
 # fgaddattr: Adding global attributes from a reference file
@@ -29398 +29399 @@
 def except_fillValue(values, ncfile, variable):
     """ Function to fill a file with fill_Value except a given value of a variable
-      values= [varname]:[value]:[rmdims]
+      values= [varname]:[value]:[rmdims]:[fill]
         [varname]: name of the variable to use its values for the filling
         [value]: value of the variable from which grid points will not be filled by 
@@ -29406 +29407 @@
             range [ivaule],[evalue]
         [rmdims]: ',' separated list of [dimn]@[dimv] name of dimensions and value to
-          remove from [varname] to find the value
+          remove from [varname] to find the value ('None' for any)
           [dimn]: name of the dimension from [varname]
           [dimv]: value of [dimn] to use
+        [fill]: kind of filling for the values outside selection
+          'fill_value': using standard fill_values according to the type of variable
+          'value',[value]: using [value] according to the type of variable
       ncfile= netCDF file to fill
-      variable= name of the variables to use
+      variable= ',' list of name of the variables to fill ('all' for all variables)
     """
     fname = 'except_fillValue'
 
     availvalue = ["'exact',[value]", "'range',[ivalue],[evalue]"]
+    availfill = ["'fill_value'", "'value',[value]"]
 
     if values == 'h':
@@ -29421 +29426 @@
         quit()
 
-    expectargs = '[varname]:[value]:[rmdims]'
+    expectargs = '[varname]:[value]:[rmdims]:[fill]'
     gen.check_arguments(fname, values, expectargs, ':')
 
     varname = values.split(':')[0]
     value = values.split(':')[1]
-    rmdims = values.split(':')[2]
-
+    rmdims00 = values.split(':')[2]
+    fill = values.split(':')[3]
+
+    # Getting dimensions to remove from varname to make the selection
+    if rmdims00 != 'None':
+        rmdims0 = gen.str_list(rmdims00, ',')
+    else:
+        rmdims0 = None
+
+    # Getting kind of selection
     if value[0:5] == 'exact':
+        kind = 'exact'
         varvalue = value.split(',')[1]
+        print infmsg
+        print '  ' + fname + ": masking by '" + kind + "' not equal ", varvalue
     elif value[0:5] == 'range':
+        kind = 'range'
         ivarvalue = value.split(',')[1]
         evarvalue = value.split(',')[2]
+        print infmsg
+        print '  ' + fname + ": masking by '" + kind + "' within [", ivarvalue, ',', \
+          evarvalue, ']'
     else:
         print errormsg
@@ -29439 +29459 @@
         quit(-1)
 
-    
-
+    # Getting kind of filling
+    if fill[0:10] == 'fill_value':
+        kfill = 'fval'
+    elif fill[0:5] == 'value':
+        kfill = 'val'
+        kfillv = fill.split(',')[1]
+    else:
+        print errormsg
+        print '  ' + fname + ": kind of filling '" + fill +  "' not ready !!"
+        print '    available ones:', availfill
+        quit(-1)
+        
     onc = NetCDFFile(ncfile, 'r')
     if not onc.variables.has_key(varname):
@@ -29446 +29476 @@
         print '  ' + fname + ": file '" + + "' does not have variable '" + varname + \
           "' !!"
-        varns = list(onc.variables.keys())
-        varns.sort()
-        print '    available ones:', varns
+        Varns = list(onc.variables.keys())
+        Varns.sort()
+        print '    available ones:', Varns
         quit(-1)
 
+    if rmdims0 is not None:
+        rmdims = {}
+        for rmd in rmdims0:
+            dn = rmd.split('@')[0]
+            dv = rmd.split('@')[1]
+            if not gen.searchInlist(onc.dimensions,dn):
+                print errormsg    
+                print '  ' + fname + ": file '" + + "' does not have dimension '" +  \
+                  dn + "' !!"
+                dimns = list(onc.dimensions)
+                dimns.sort()
+                print '    available ones:', dimns
+                quit(-1)
+            rmdims[dn] = dv
+    else:
+        rmdims = None
+
     ovar = onc.variables[varname]
+    if rmdims is not None:
+        rmdns = list(rmdims.keys())
+        rmdvs = list(rmdims.values())
+        varv, rmdims, rmshape = ovar_reducedims(ovar, rmdns, rmdvs)
+    else:
+        varv = ovar[:]
+        rmdims = ovar.dimensions
+        rmshape = ovar.shape
+
+    if variable != 'all':
+        varns = gen.str_list(variable, ',')
+    else:
+        varns = onc.variables.keys()
+
+    # Filling the variables
+    ofilen = fname + '.nc' 
+    onewnc = NetCDFFile(ofilen, 'w')
     
+    # Getting variable of filling
+    idim = 0
+    newrmdims = []
+    for dn in rmdims:
+        onewnc.createDimension(dn+'fill', rmshape[idim])
+        newrmdims.append(dn+'fill')
+        idim = idim + 1
+
+    # Removing the variables which do not have the fill dimensions
+    newvarns = varns + []
+    for varn in varns:
+        if not onc.variables.has_key(varn):
+            print errormsg
+            print '  ' + fname + ": file '" + + "' does not have variable '" + varn+ \
+              "' !!"
+            Varns = list(onc.variables.keys())
+            Varns.sort()
+            print '    available ones:', Varns
+            quit(-1)
+        ovar = onc.variables[varn]
+        dvns = ovar.dimensions
+        for drmn in rmdims:
+            if not gen.searchInlist(dvns, drmn):
+                print infmsg
+                print '  ' +fname + ": variable '" + varn + "' does not have fill "+ \
+                  " dimension '" + drmn + "', removing variable from list"
+                newvarns.remove(varn)
+                break
+    varns = newvarns + []
+
+    if kind == 'exact':
+        varvalue = gen.typemod(varvalue, varv.dtype)
+        mavarv = ma.masked_not_equal(varv, varvalue)
+    elif kind == 'range':
+        ivarvalue = gen.typemod(ivarvalue, varv.dtype)
+        evarvalue = gen.typemod(evarvalue, varv.dtype)
+        mavarv = ma.masked_outside(varv, ivarvalue, evarvalue)
+    mamask = mavarv.mask
+
+    if ovar.dtype == type(1):
+        newvar= onewnc.createVariable(varname + 'fill', ovar.dtype, tuple(newrmdims),\
+          fill_value=gen.fillValueI)
+        mavarv.filled(gen.fillValueI)
+    elif ovar.dtype == type(1.):
+        newvar= onewnc.createVariable(varname + 'fill', ovar.dtype, tuple(newrmdims),\
+          fill_value=gen.fillValueF)
+        mavarv.filled(gen.fillValueF)
+    elif ovar.dtype == type(np.float32(1.)) or ovar.dtype == type(np.float64(1.)):
+        newvar= onewnc.createVariable(varname + 'fill', ovar.dtype, tuple(newrmdims),\
+          fill_value=gen.fillValueD)
+        mavarv.filled(gen.fillValueD)
+    else: 
+        print errormsg
+        print '  ' + fname + ': type of variable ', ovar.dtype, 'not ready !!'
+        print '    available ones:', type(1), type(1.), type(np.float32(1.)),        \
+          type(np.float64(1.))
+        quit(-1)
+
+    newvar[:] = mavarv
+    vattrs = ovar.ncattrs()
+    for attrn in vattrs:
+        if attrn != '_FillValue':
+            attrv = ovar.getncattr(attrn)
+            newvar.setncattr(attrn, attrv)
+    if kind == 'exact':
+        newvar.setncattr('only_exact',varvalue)
+    elif kind == 'range':
+        mavarv = ma.masked_outside(varv, ivarvalue, evarvalue)
+        newvar.setncattr('only_within_beg',ivarvalue)
+        newvar.setncattr('only_within_end',evarvalue)
+
+    onewnc.sync()
+
+    # getting variables and slicing
+    for varn in varns:
+        print varn + " ..."
+        ovar = onc.variables[varn]
+
+        vdims = list(ovar.dimensions)
+        for dn in vdims:
+            odim = onc.dimensions[dn]
+            if not gen.searchInlist(onewnc.dimensions, dn):
+                if odim.isunlimited(): onewnc.createDimension(dn, None)
+                else: onewnc.createDimension(dn, len(onc.dimensions[dn]))
+
+        vshape = ovar.shape
+        newdims = vdims + []
+        for dn in rmdims:
+            if gen.searchInlist(newdims, dn): newdims.remove(dn)
+
+        slicesvar = gen.provide_slices(vdims, vshape, newdims)
+        Nslices = len(slicesvar)
+
+        if ovar.dtype == type(1):
+            if kfill == 'fval':
+                fillv = gen.fillValueI
+            else:
+                fillv = int(kfillv)
+        elif ovar.dtype == type(1.):
+            if kfill == 'fval':
+                fillv = gen.fillValueF
+            else:
+                fillv = np.float(kfillv)
+        elif ovar.dtype== type(np.float32(1.)) or ovar.dtype== type(np.float64(1.)):
+            if kfill == 'fval':
+                fillv = gen.fillValueD
+            else:
+                if ovar.dtype== type(np.float32(1.)): fillv = np.float32(kfillv)
+                else: fillv = np.float64(kfillv)
+        else: 
+            print errormsg
+            print '  ' +fname+ ': fill type of variable ', ovar.dtype, 'not ready !!'
+            print '    available ones:', type(1), type(1.), type(np.float32(1.)),    \
+              type(np.float64(1.))
+            quit(-1)
+
+        if kfill == 'fval':
+            newvar = onewnc.createVariable(varn, ovar.dtype, tuple(vdims),           \
+              fill_value=fillv)
+        else:
+            newvar = onewnc.createVariable(varn, ovar.dtype, tuple(vdims))
+
+        for islc in range(Nslices):
+            slcv = slicesvar[islc]
+            filled = ma.masked_array(ovar[tuple(slcv)], mask=mamask)
+            fill = filled.filled(fillv)
+            newvar[tuple(slcv)] = fill
+
+        vattrs = ovar.ncattrs()
+        for attrn in vattrs:
+            if attrn != '_FillValue':
+                attrv = ovar.getncattr(attrn)
+                newvar.setncattr(attrn, attrv)
+        newvar.setncattr('fill', varname + 'fill')
+        onewnc.sync()
+
+        # Looking for coordinates (let's be pro!)
+        if gen.searchInlist(vattrs, 'coordinates'):
+            coords = ovar.getncattr('coordinates')
+            print infmsg
+            print '  ' + fname + ": variable '" + varn + "' with coordinates " +     \
+              "attribtue:" + coords + " !!"
+            print '  adding coordinates if they are not yet in file'
+            cordv = coords.split(' ')
+            add_vars(onc, onewnc, cordv)                
+            
+    # Global attributes
+    add_global_PyNCplot(onewnc, 'nc_var_tools', fname, '1.0')
+    onewnc.sync()
+
+    vattrs = onc.ncattrs()
+    for attrn in vattrs:
+        attrv = onc.getncattr(attrn)
+        onewnc.setncattr(attrn, attrv)
+    onewnc.sync()
+
+    onc.close()
+    onewnc.close()
+    
+    print fname + ": successful writting of file '" + ofilen + "' !!"
+
+    return
+
+#except_fillValue('orog:range,500.,7000.:None:value,0', '/home/lluis/estudios/Andes/cmip_data/fx/orog_fx_ACCESS1-0_historical_r0i0p0.nc', 'all')
 
 #quit()