Changeset 905 in lmdz_wrf for trunk/tools

Timestamp:

Jun 19, 2016, 9:46:06 PM (8 years ago)

Author:

lfita

Message:

Adding `netcdf_fold_concatenation_HMT': Function to concatenate netCDF files in a given folder for a given set of variables giving Header, Middle, Tail for the name files

Location:

trunk/tools

Files:

• nc_var.py (modified) (3 diffs)
• nc_var_tools.py (modified) (2 diffs)

trunk/tools/nc_var.py

@@ -40 +40 @@
   'ivattrs', 'maskvar', 'model_characteristics',                                     \
   'ncreplace', 'ncstepdiff', 'netcdf_concatenation', 'netcdf_fold_concatenation',    \
-  'Partialmap_Entiremap', 'Partialmap_EntiremapFor', 'Partialmap_EntiremapForExact', \
+  'netcdf_fold_concatenation_HMT', 'Partialmap_Entiremap',                           \
+  'Partialmap_EntiremapFor', 'Partialmap_EntiremapForExact',                         \
   'pinterp', 'remapnn',                                                              \
   'seasmean', 'sellonlatbox', 'sellonlatlevbox', 'selvar', 'setvar_asciivalues',     \
@@ -97 +98 @@
 # Operations which file name is not a real file
 NotCheckingFile = ['DatesFiles', 'file_creation', 'list_operations',                 \
-  'model_characteristics', 'netcdf_concatenation', 'netcdf_fold_concatenation']
+  'model_characteristics', 'netcdf_concatenation', 'netcdf_fold_concatenation',      \
+  'netcdf_fold_concatenation_HMT']
 
 ####### ###### ##### #### ### ## #
@@ -216 +218 @@
 elif oper == 'netcdf_fold_concatenation':
     ncvar.netcdf_fold_concatenation(opts.values, opts.ncfile, opts.varname)
+elif oper == 'netcdf_fold_concatenation_HMT':
+    ncvar.netcdf_fold_concatenation_HMT(opts.values, opts.ncfile, opts.varname)
 elif oper == 'opersvarsfiles':
     ncvar.compute_opersvarsfiles(opts.values, opts.varname)

trunk/tools/nc_var_tools.py

@@ -75 +75 @@
 # netcdf_concatenation: Function to concatenate netCDF files for a given set of variable
 # netcdf_fold_concatenation: Function to concatenate netCDF files in a given folder for a given set of variables
+# netcdf_fold_concatenation_HMT: Function to concatenate netCDF files in a given folder for a given set of variables giving Header, Middle, Tail for the name files
 # operation_alongdims: Function to operate along different dimensions of a variable
 # operdim: Function to operate along a series of dimensions
@@ -10066 +10067 @@
 #dimVar_creation('bottom_top', 'wrfout_d01_1979-12-01_00:00:00_south_north_B3-E3-I1_west_east_B26-E26-I1.nc')
 
+def netcdf_fold_concatenation_HMT(values, ncfile, varn):
+    """ Function to concatenate netCDF files in a given folder for a given set of variables 
+      giving Header, Middle, Tail for the name files
+    netcdf_fold_concateation(values, ncfile, varn)
+      [values]= [fold],[dimname]
+        [fold]: folder with the location of the netCDF files
+        [dimname]: dimension along which files should be concatenated
+        [tail]: tail of the files to concatenate (optional)
+      [ncfile]= [header],[middle],[tail]
+        [header] = header of the name of the files to concatenate [ncfile]*[middle]*[tail]
+        [middle] = middle of the name of the files to concatenate [ncfile]*[middle]*[tail]
+        [tail] = tail of the name of the files to concatenate [ncfile]*[middle]*[tail]
+
+      [varn]= ',' separated list of variables to concatenate 
+          [var1],[var2],[...[varN]] or 'all' for all variables
+    """
+    import subprocess as sub
+    fname='netcdf_fold_concatenate_HMT'
+
+    ofile = 'netcdf_fold_concatenated_HMT.nc'
+
+    if values == 'h':
+        print fname + '_____________________________________________________________'
+        print netcdf_fold_concatenation_HMT.__doc__
+        quit()
+
+    fold = values.split(',')[0]
+    condim = values.split(',')[1]
+
+    confiles = gen.files_folder_HMT(fold,ncfile.split(',')
+    Nfiles = len(confiles)
+
+    print '  ' + fname +': concatenating:', Nfiles, 'files'
+    if Nfiles == 0:
+        print errormsg
+        print '  ' + fname + ": there are no files as '" + fold + '/' + ncfile + "*' !!"
+        quit(-1)
+    print confiles
+
+# Opening all files
+    ncobjs = []
+    for filen in confiles:
+        print 'charging: ',filen
+        ncobjs.append(NetCDFFile(fold + '/' + filen, 'r'))
+
+# Looking for the new length of the concatenation dimension
+    if not ncobjs[0].dimensions.has_key(condim):
+        print errormsg
+        print '  ' + fname + ": files do not have dimensions '" + condim + "' !!!"
+        quit(-1)
+
+    totcondim = 0
+    for ifile in range(Nfiles):
+        ncobj = ncobjs[ifile]
+        totcondim = totcondim + len(ncobjs[ifile].dimensions[condim])
+
+    print "total concatenated dimension '" + condim + "': ", totcondim
+
+# concatenated file creation
+##
+    newnc = NetCDFFile(ofile, 'w')
+
+# dimension creation 
+##
+    if ncobjs[0].dimensions[condim].isunlimited():
+        newnc.createDimension(condim, None)
+    else:
+        newnc.createDimension(condim, totcondim)
+# Fake variable for the dimension
+    print 'condim:',condim
+    dims = []
+    dims.append(condim)
+    newvar = newnc.createVariable('int' + condim, 'i4', tuple(dims))
+    newvar[:] = range(totcondim)
+
+# Looping variables
+##
+    if varn == 'all':
+        desvars = ncobjs[0].variables
+    else:
+        desvars = varn.split(',')
+
+    for dvar in desvars:
+        if not ncobjs[0].variables.has_key(dvar):
+            print errormsg
+            print '  ' + fname + ": files do not have variable '" + dvar + "' !!!"
+            quit(-1)
+
+    for dvar in desvars:
+        print '  ' + fname + ": concatenating '" + dvar + "'..."
+        objvar = ncobjs[0].variables[dvar]
+# creation of dimensions
+        vardims = objvar.dimensions
+        for ndim in vardims:
+            if not newnc.dimensions.has_key(ndim):
+                vardimo = ncobjs[0].dimensions[ndim]
+                if vardimo.isunlimited():
+                    newnc.createDimension(ndim, None)
+                else:
+                    newnc.createDimension(ndim, len(vardimo))
+# creation of variable
+        kvar = objvar.dtype
+        newvar = newnc.createVariable(dvar, kvar, objvar.dimensions)
+        varattrs = objvar.ncattrs()
+        for atr in varattrs:
+            attrv = objvar.getncattr(atr)
+            newattr = set_attribute(newvar, atr, attrv)
+
+# variable values
+#        vartotv = np.zeros((newvar.shape), dtype=kvar)
+        objvar = ncobjs[0].variables[dvar]
+
+        if gen.searchInlist(list(objvar.dimensions),condim):
+            begslicetot = 0
+            for ifile in range(Nfiles):
+                slicevartot = []
+                if ncobjs[ifile].variables.has_key(dvar):
+                    objvar = ncobjs[ifile].variables[dvar]
+                    for dimn in objvar.dimensions:
+                        ldimfile = len(ncobjs[ifile].dimensions[dimn])
+                        if dimn == condim:
+                            slicevartot.append(slice(begslicetot,begslicetot+ldimfile))
+                            begslicetot = begslicetot + ldimfile
+                        else:
+                            slicevartot.append(slice(0,ldimfile))
+                    newvar[tuple(slicevartot)] = objvar[:]
+                    newnc.sync()
+                else:
+                    print errormsg
+                    print '  ' + fname + ": file '" + fold + '/' + confiles[ifile] + \
+                      "' does not have variable '" + dvar + "' !!"
+                    quit(-1)
+        else:
+            newvar[:] = objvar[:]
+            newnc.sync()
+
+#        newvar[:] = vartotv
+
+    gattrs = ncobjs[0].ncattrs()
+    for attr in gattrs:
+        attrv = ncobjs[0].getncattr(attr)
+        newattr = set_attribute(newnc, attr, attrv)
+
+    for ifile in range(Nfiles):
+        ncobjs[ifile].close()
+
+    newnc.sync()
+    newnc.close()
+
+    print "Successfull creation of concatenated file '" + ofile + "' !!!"
+
+    return
+
 def netcdf_fold_concatenation(values, ncfile, varn):
     """ Function to concatenate netCDF files in a given folder for a given set of variables
     netcdf_fold_concateation(values, ncfile, varn)
-      [values]= [fold],[dimname] folder with the location of the netCDF files
+      [values]= [fold],[dimname]
         [fold]: folder with the location of the netCDF files
         [dimname]: dimension along which files should be concatenated
+        [tail]: tail of the files to concatenate (optional)
       [ncfile]= header of the name of the files to concatenate [ncfile]*
       [varn]= ',' separated list of variables to concatenate