Changeset 1597 in lmdz_wrf

Timestamp:

Aug 3, 2017, 12:51:19 AM (8 years ago)

Author:

lfita

Message:

Adding:

• `merge_files': Function to merge variables from two files

Location:

trunk/tools

Files:

• nc_var.py (modified) (5 diffs)
• nc_var_tools.py (modified) (2 diffs)

trunk/tools/nc_var.py

@@ -47 +47 @@
 ## e.g. # nc_var.py -o rm_FillValue -f new_ORCHIDEE_13PFTmap_2010_cmpi6_LUH2v2h.nc -S 0 -v mask
 ## e.g. # nc_var.py -o curve_section -f /home/lluis/PY/test.nc -S 'gridline,x,y,8.,8.,16.,16.,32' -v all
+## e.g. # nc_var.py -o merge_files -S 'plev|plev,time|time:merged.nc' -f 'ncobs/AliceSprings/snd_94326_198201-198201.nc,ncobs/AliceSprings/snd_94326_198202-198202.nc' -v 'plev,time'
 
 from optparse import OptionParser
@@ -113 +114 @@
 # lonlat_polygon: Function to define a lon/lat region giving the coordinates of the vertexs of a given polygon 
 # maskvar: Function to mask a variable using another variable to mask it. Only need to share at least 1 dimension with the same size (no need same name)
+# merge_files: Function to merge variables from two files
 # model_characteristics: Function to provide major characterisitcs of a given model output
 # mthDYNAMICO_toCF: Function to pass a mthDYNAMICO original file to CF-conventions
@@ -182 +184 @@
   'grmattr', 'idims', 'igattrs', 'increaseDimvar', 'isgattrs',                       \
   'isvattrs', 'ivars', 'ivattrs', 'LMDZ_toCF', 'lonlat_polygon', 'maskvar',          \
-  'model_characteristics',                                                           \
+  'merge_files', 'model_characteristics',                                            \
   'mthDYNAMICO_toCF', 'ncreplace', 'ncstepdiff', 'netcdf_concatenation',             \
   'netcdf_fold_concatenation',                                                       \
@@ -244 +246 @@
 # Operations which file name is not a real file
 NotCheckingFile = ['DatesFiles', 'compute_opersvarsfiles', 'file_creation',          \
-  'list_operations',                                                                 \
+  'list_operations', 'merge_files',                                                  \
   'model_characteristics', 'netcdf_concatenation', 'netcdf_fold_concatenation',      \
   'netcdf_fold_concatenation_HMT']
@@ -389 +391 @@
 elif oper == 'maskvar':
     ncvar.maskvar(opts.values, opts.ncfile, opts.varname)
+elif oper == 'merge_files':
+    ncvar.merge_files(opts.values, opts.ncfile, opts.varname)
 elif oper == 'model_characteristics':
     ncvar.model_characteristics(opts.values, opts.ncfile)

trunk/tools/nc_var_tools.py

@@ -92 +92 @@
 # lonlat_polygon: Function to define a lon/lat region giving the coordinates of the vertexs of a given polygon 
 # maskvar: Function to mask a variable using a mask. It is assumed that mask[...,dimM,dimJ,dimK] and var[...,dimM,dimJ,dimK] share the last dimensions
+# merge_files: Function to merge variables from two files
 # ModelChar: Class object to determine major characterisitcs of a given model output
 # model_characteristics: Functino to provide major characterisitcs of a given model output
@@ -20165 +20166 @@
 #lonlat_polygon('/home/lluis/PY/star.dat','/home/lluis/PY/wrfout_d01_1995-01-01_00:00:00', 'XLONG,XLAT')
 
+def merge_files(values, ncfiles, variable):
+    """ Function to merge variables from two files
+      values= [dimvar]:[mergedfilen]
+        [dimvarn]: ',' list of couples [dimvarn]|[dimn] of the name of dimension and the variable with its values
+          related to the variables to merge
+        [mergedfilen]: name of the merged file
+      ncfile= ',' list of file names to merge
+      variable= ',' list of names of variables to merge ('all' for all variables) 
+    """
+    fname = 'merge_files'
+
+    if values == 'h':
+        print fname + '_____________________________________________________________'
+        print merge_files.__doc__
+        quit()
+
+    expectargs = '[dimvar]:[mergedfilen]'
+    gen.check_arguments(fname, values, expectargs, ':')
+
+    dimvarns = gen.str_list(values.split(':')[0], ',')
+    mergedfilen = values.split(':')[1]
+
+    filen1 = ncfiles.split(',')[0]
+    filen2 = ncfiles.split(',')[1]
+
+    if not os.path.isfile(filen1):
+        print errormsg
+        print '  ' + fname + ": file named '" + filen1 + "' does not exist !!"
+        quit(-1)
+
+    if not os.path.isfile(filen2):
+        print errormsg
+        print '  ' + fname + ": file named '" + filen2 + "' does not exist !!"
+        quit(-1)
+
+    onc1 = NetCDFFile(filen1, 'r')
+    onc2 = NetCDFFile(filen2, 'r')
+
+    dimn1 = onc1.dimensions
+    dimn2 = onc2.dimensions
+    varn1 = onc1.variables.keys()
+    varn2 = onc2.variables.keys()
+
+    # Getting dimension variables
+    dimvar = {}
+    
+    for dimvarn in dimvarns:
+        dimn = dimvarn.split('|')[0]
+        dvn = dimvarn.split('|')[1]
+
+        if not gen.searchInlist(dimn1, dimn):
+            print errormsg
+            print '  ' +fname+ ": file '" + filen1 + "' does not have dimension: '"+ \
+              dimn
+            print '    available ones:', dimn1
+            quit(-1)
+
+        if not gen.searchInlist(dimn2, dimn):
+            print errormsg
+            print '  ' +fname+ ": file '" + filen2 + "' does not have dimension: '"+ \
+              dimn
+            print '    available ones:', dimn2
+            quit(-1)
+
+        if not gen.searchInlist(varn1, dvn):
+            print errormsg
+            print '  ' +fname+ ": file '" + filen1 + "' does not have variable: '" + \
+              dimn
+            print '    available ones:', varn1
+            quit(-1)
+
+        if not gen.searchInlist(varn2, dvn):
+            print errormsg
+            print '  ' +fname+ ": file '" + filen2 + "' does not have variable: '" + \
+              dimn
+            print '    available ones:', varn2
+            quit(-1)
+
+        dimvar[dimn] = dvn
+
+    if variable == 'all':
+        # concident variables
+        setvarn1 = set(varn1)
+        setvarn2 = set(varn2)
+        coincidentvarn = list(setvarn1.intersection(setvarn2))
+    else:
+        coincidentvarn = gen.str_list(variable, ',')
+
+    # Creation of merged file
+    onewnc = NetCDFFile(mergedfilen, 'w')
+
+    # Looping on coincident variables
+    for varn in coincidentvarn:
+        print '  ' + fname + ": merging on variable '" + varn + "' ...."
+        ovar1 = onc1.variables[varn]
+        ovar2 = onc2.variables[varn]
+
+        dimvn1 = ovar1.dimensions
+        dimvn2 = ovar2.dimensions
+
+        # Assuming same name of dimensions. Too busy to generalize it... ;)
+        NOcoindimns = list(set(dimvn1).symmetric_difference(set(dimvn2)))
+        if len(NOcoindimns) != 0:
+            print errormsg
+            print '  ' + fname + ": variable to merge '" + varn + "' have " +        \
+              "different dimensions in files!!"
+            print "    file '" + filen1 + "' dimensions: ", dimvn1
+            print "    file '" + filen2 + "' dimensions: ", dimvn2
+            quit(-1)
+
+        # Getting values of the dimensions
+        dimvals = {}
+        for dimn in dimvn1:
+            odvar1 = onc1.variables[dimvar[dimn]]
+            odvar2 = onc2.variables[dimvar[dimn]]
+
+            vdd1 = odvar1[:]
+            vdd2 = odvar2[:]
+            if len(vdd1.shape) != 1:
+                print errormsg
+                print '  ' + fname + ": dimension-variable '" + dimvar[dimn] +       \
+                  "' is not 1D!!"
+                print "    shape", vdd1.shape
+                quit(-1)
+ 
+            lvdd1 = list(vdd1)
+            lvdd2 = list(vdd2)
+
+            # Looking for non coincident values
+            notvdd = list(set(lvdd2).difference(set(lvdd1)))
+            Lnotvdd = len(notvdd)
+
+            dimv = np.zeros((len(vdd1)+Lnotvdd), dtype=vdd1.dtype)
+            if Lnotvdd == 0:
+                newdimv = np.array(lvdd1)
+            else:
+                # Getting the right order and sense
+                notvdd.sort()
+                if lvdd1[1] < lvdd1[0]: 
+                    notvdd.reverse()
+
+                # Getting dimension values starting with filen1 as reference
+                print infmsg
+                print '    ' + fname + ": dimension '" + dimvar[dimn] + "' with ",   \
+                  Lnotvdd, "different values !!"
+                print '    size from file1:', len(lvdd1), 'file2:', len(lvdd2)
+
+                # Filling dimension
+                newdimv = np.array(gen.InsertVal_list_sign(lvdd1, notvdd))
+
+            # Creation of dimension
+            Lnewdim = len(newdimv)
+            if not gen.searchInlist(onewnc.dimensions, dimn):
+                print infmsg
+                print '    ' + fname + ": creation of dimension '" + dimn+ "' (" +   \
+                  str(Lnewdim) + ") !!"
+
+                if onc1.dimensions[dimn].isunlimited():
+                    newdim = onewnc.createDimension(dimn, None)
+                else:
+                    newdim = onewnc.createDimension(dimn, Lnewdim)
+
+            # Creation of variable-dimension
+            if not onewnc.variables.has_key(dimvar[dimn]):
+                print infmsg
+                print '    ' + fname + ": creation of variable '" + dimvar[dimn] +   \
+                  "'", newdimv.shape, "!!"
+                if gen.searchInlist(odvar1.ncattrs(), '_FillValue'):
+                    newvar = onewnc.createVariable(dimvar[dimn], nctype(vdd1.dtype), \
+                      (dimn), fill_value=ovar1.getncattr('_FillValue'))
+                else:
+                    newvar = onewnc.createVariable(dimvar[dimn], nctype(vdd1.dtype), \
+                      (dimn))
+                newvar[:] = newdimv[:]
+                for attrn in odvar1.ncattrs():
+                    attrv = odvar1.getncattr(attrn)
+                    if attrn != '_FillValue': newattr = set_attribute(newvar, attrn, \
+                       attrv)
+
+            onewnc.sync()
+
+        # Looking for the values of the variables and filling netCDF file
+        vvar1 = ovar1[:]
+        vvar2 = ovar2[:]
+        Lv1 = len(vvar1)
+        Lv2 = len(vvar2)
+
+        v1dim = ovar1.dimensions[0]
+        odimv = onewnc.variables[dimvar[v1dim]]
+        dimv = odimv[:]
+        Lnewdim = len(dimv)
+        
+        newvarvals = np.zeros((Lnewdim), vdd1.dtype)
+        iv1 = 0
+        iv2 = 0
+        for iv in range(Lnewdim):
+            if dimv[iv] == vdd1[iv1]: 
+                newvarvals[iv] = vvar1[iv1]
+                iv1 = np.min([iv1 + 1, Lv1-1])
+            else: 
+                newvarvals[iv] = vvar2[iv2]
+                iv2 = np.min([iv2 + 1, Lv2-1])
+
+        # Creation of variable
+        if not onewnc.variables.has_key(varn):
+            print infmsg
+            print '    '+fname+": creation of variable '" +varn+  "' (" +dimn+ ") !!"
+            if gen.searchInlist(ovar1.ncattrs(), '_FillValue'):
+                newvar = onewnc.createVariable(dimvar[dimn], nctype(vdd1.dtype),     \
+                  (dimn), fill_value=ovar1.getncattr('_FillValue'))
+            else:
+                newvar = onewnc.createVariable(dimvar[dimn], nctype(vdd1.dtype),     \
+                  (dimn))
+        newvar[:] = newdimv
+        for attrn in ovar1.ncattrs():
+            attrv = ovar1.getncattr(attrn)
+            if attrn != '_FillValue': newattr = set_attribute(newvar, attrn, attrv)
+
+        onewnc.sync()
+
+    # Global attributes
+    gattrs = onc1.ncattrs()
+    for attrn in gattrs:
+        attrv = onc1.getncattr(attrn)
+        newattr = set_attribute(onewnc, attrn, attrv)
+    onewnc.sync()
+
+    # Adding PyNCplot
+    add_global_PyNCplot(onewnc, main, fname, '0.1')
+    onewnc.sync()
+
+    onewnc.close()
+    print fname + ": Variables succesfuly  merged in file '" + mergedfilen + "' !!"
+    return
 
 #quit()