Changeset 2061 in lmdz_wrf

Timestamp:

Aug 9, 2018, 10:00:17 PM (7 years ago)

Author:

lfita

Message:

Adding:

• join_sounding_obsfiles: Function to join 1D variables from files generated by UWyoming_snd_nc.py' script to create a single file

Location:

trunk/tools

Files:

• nc_var.py (modified) (5 diffs)
• nc_var_tools.py (modified) (2 diffs)

trunk/tools/nc_var.py

@@ -27 +27 @@
 ## e.g. # nc_var.py -o get_time -S '1422759600;seconds!since!1949-12-01!00:00:00' -f /home/lluis/PY/wrfout_d01_1995-01-01_00:00:00 -v WRFtime
 ## e.g. # nc_var.py -o join_singlestation_obsfiles -S 'obs/sfc_CAM:OBSnetcdf' -v all
+## e.g. # nc_var.py -o join_sounding_obsfiles -S .:UWyoming_snd_1 -v all
 
 ## e.g. ccrc468-17 # ./nc_var.py -v time -f 123/CCRC_NARCliM_Sydney_All_1990-1999_pr10max.nc -o out -S 1:-1
@@ -128 +129 @@
 # ivars: Give all the variable names of a file
 # ivattrs: Give all the attributes of a variable and its type
-# join_singlestation_obsfiles: Function to join files from 'single-station' `create_OBSnetcdf.py' script to create a single file
+# join_singlestation_obsfiles: Function to join files from 'single-station' 
+#   `create_OBSnetcdf.py' script to create a single file
+# join_sounding_obsfiles: Function to join 1D variables from files generated by 
+#   `UWyoming_snd_nc.py' script to create a single file
 # LMDZ_toCF: Function to pass a LMDZ original file to CF-conventions
 # lonlat_polygon: Function to define a lon/lat region giving the coordinates of the vertexs of a given polygon 
@@ -206 +210 @@
   'grmattr', 'idims', 'ifile', 'igattrs', 'increaseDimvar', 'isgattrs',              \
   'isvattrs', 'itime', 'ivars', 'ivattrs', 'join_singlestation_obsfiles',            \
-  'LMDZ_toCF', 'lonlat_polygon', 'maskvar', \
+  'join_sounding_obsfiles', 'LMDZ_toCF', 'lonlat_polygon', 'maskvar',                \
   'merge_files', 'model_characteristics',                                            \
   'mthDYNAMICO_toCF', 'ncreplace', 'ncstepdiff', 'netcdf_concatenation',             \
@@ -269 +273 @@
 # Operations which file name is not a real file
 NotCheckingFile = ['DatesFiles', 'compute_opersvarsfiles', 'file_creation',          \
-  'join_singlestation_obsfiles', 'list_operations', 'merge_files',                   \
+  'join_singlestation_obsfiles', 'join_sounding_obsfiles', 'list_operations',        \
+  'merge_files',                                                                     \
   'model_characteristics', 'netcdf_concatenation', 'netcdf_fold_concatenation',      \
   'netcdf_fold_concatenation_HMT']
@@ -415 +420 @@
 elif oper == 'join_singlestation_obsfiles':
     ncvar.join_singlestation_obsfiles(opts.values, opts.varname)
+elif oper == 'join_sounding_obsfiles':
+    ncvar.join_sounding_obsfiles(opts.values, opts.varname)
 elif oper == 'list_operations':
 # From: http://www.diveintopython.net/power_of_introspection/all_together.html

trunk/tools/nc_var_tools.py

@@ -103 +103 @@
 # ivars: Give all the variable names of a file
 # ivattrs: Give all the attributes of a variable and its type
-# join_singlestation_obsfiles: Function to join files from 'single-station' `create_OBSnetcdf.py' script to create a single file
+# join_singlestation_obsfiles: Function to join files from 'single-station' 
+#   `create_OBSnetcdf.py' script to create a single file
+# join_sounding_obsfiles: Function to join 1D variables from files generated by 
+#   `UWyoming_snd_nc.py' script to create a single file
 # load_ncvariable_lastdims: Function to load the last [Nlastdims] dimensions of a variable [varn] from a netCDF object at the other dimensions at [prevdims]
 # lonlatProj: Function to compute longitudes and latitudes for a given projection following subroutine calc_resolution_in from ORCHIDEE/src_global/module_InterpWeight
@@ -24875 +24878 @@
     return
 
+def join_sounding_obsfiles(values, variable):
+    """ Function to join 1D variables from files generated by `UWyoming_snd_nc.py' 
+        script to create a single file
+      values=[namefoldobs]:[headerF]
+        [namefoldobs]: name of the foler holding all single-stations netCDF files
+        [headerF]: header of the names of the files with the observations
+      variable= ',' separated list of variables names ('all' for all variables 
+        encountred within all files)
+    """
+    fname = 'join_sounding_obsfiles'
+
+    if values == 'h':
+        print fname + '_____________________________________________________________'
+        print join_sounding_obsfiles.__doc__
+        quit()
+
+    expectargs = '[namefoldobs]:[headerF]'
+    gen.check_arguments(fname, values, expectargs, ':')
+
+    namefoldobs = values.split(':')[0]
+    headerF = values.split(':')[1]
+
+    fobs = gen.files_folder(namefoldobs, headerF)
+
+    # Getting number stations
+    ifile = 0
+    namevals = []
+    refvals = []
+    lonvals = []
+    latvals = []
+    heightvals = []
+    timevals = []
+    varvals = {}
+    for fobsn in fobs:
+        filen =  namefoldobs + '/' + fobsn
+        if not os.path.isfile(filen):
+            print errormsg
+            print '  ' + fname + ": file '" + filen + "' does not exist !!"
+            print "    files found in folder '" + namefoldobs + "':", fobs 
+            quit(-1)
+
+        onc = NetCDFFile(filen, 'r')
+        if gen.searchInlist(onc.dimensions,'time'): dimt = onc.dimensions['time']
+        elif gen.searchInlist(onc.dimensions,'CFtime'): dimt=onc.dimensions['CFtime']
+        else:
+            print errormsg
+            print '  ' + fname + ": file '" + filen + "' does not have any 'time'" + \
+              "dimension !!"
+            print "    available ones:", onc.dimensions
+            quit(-1)
+
+        if ifile == 0:
+            varns = []
+            if variable == 'all':
+                allvarns = onc.variables.keys()
+                allvarns.remove('pres')
+                for vn in allvarns: 
+                    ovn = onc.variables[vn]
+                    # Getting 1D variables
+                    rank = len(ovn.shape)
+                    if rank == 1: 
+                        varns.append(vn)
+                        vattrs = ovn.ncattrs()
+                        atv = []
+                        for atr in vattrs:
+                            atv.append(ovn.getncattr(atr))
+                        if gen.searchInlist(vattrs, '_FillValue'): 
+                            varfill = ovn._FillValue
+                        else: varfill = None
+                        varvals[vn]= [ovn.dtype, vattrs, atv, ovn.dimensions, varfill]
+
+            else: 
+                allvarns = gen.str_list(variable, ',')
+                for vn in allvarns: 
+                    if onc.variables.has_key(vn):
+                        ovn = onc.variables[vn]
+                        # Getting 1D variables
+                        rank = len(ovn.shape)
+                        if rank == 1: 
+                            varns.append(vn)
+                            vattrs = ovn.ncattrs()
+                            atv = []
+                            for atr in vattrs:
+                                atv.append(ovn.getncattr(atr))
+                            if gen.searchInlist(vattrs, '_FillValue'): 
+                                varfill = ovn._FillValue
+                            else: varfill = None
+                            varvals[vn]= [ovn.dtype,vattrs,atv,ovn.dimensions,varfill]
+
+            if onc.variables.has_key('time'): 
+                timevals = list(onc.variables['time'][:])
+            elif onc.variables.has_key('CFtime'): 
+                timevals = list(onc.variables['CFtime'][:])
+            else:
+                print errormsg
+                print '  '+fname+": file '" + filen + "' does not have any 'time'" + \
+                  "expected variable ('time', 'CFtime') !!"
+                varnsorted = onc.variables.has_key(varn)
+                varnsorted.sort()
+                print '    available ones:', varnsorted
+                quit(-1)
+        else:
+            if variable == 'all':
+                allvarns = onc.variables.keys()
+                allvarns.remove('pres')
+            else:
+                allvarns = gen.str_list(variable, ',')
+            for vn in allvarns: 
+                if not gen.searchInlist(allvarns,vn):
+                    ovn = onc.variables[vn]
+                    rank = len(ovn.shape)
+                    if rank == 1: 
+                        varns.append(vn)
+                        vattrs = ovn.ncattrs()
+                        atv = []
+                        for atr in vattrs:
+                            atv.append(ovn.getncattr(atr))
+                        if gen.searchInlist(vattrs, '_FillValue'): 
+                            varfill = ovn._FillValue
+                        else: varfill = None
+                        varvals[vn]= [ovn.dtype, vattrs, atv, ovn.dimensions, varfill]
+
+            if onc.variables.has_key('time'): 
+                tvals = onc.variables['time'][:]
+            elif onc.variables.has_key('CFtime'): 
+                tvals =onc.variables['CFtime'][:]
+            else:
+                print errormsg
+                print '  ' +fname+ ": file '" +filen+ "' does not have any 'time'" + \
+                  "expected variable ('time', 'CFtime') !!"
+                varnsorted = onc.variables.has_key(varn)
+                varnsorted.sort()
+                print '    available ones:', varnsorted
+                quit(-1)
+            NOtvals, NOits = gen.list_differences(timevals, list(tvals))
+            if len(NOtvals) > 0:
+                print '  ' + fname + ": file '" + filen + "' adds additional times:",\
+                  NOtvals
+                for tv in NOtvals: timevals.append(tv)
+        stn = onc.getncattr('Station_name')
+        stref = onc.getncattr('Station_ref')
+        stlon = onc.getncattr('Station_longitude')
+        stlat = onc.getncattr('Station_latitude')
+        sthgt = onc.getncattr('Station_elevation')
+
+        namevals.append(stn)
+        refvals.append(stref)
+        lonvals.append(stlon)
+        latvals.append(stlat)
+        heightvals.append(sthgt)
+        print '    ' + fname + ": found values for station '" + namevals[ifile] +    \
+          "' at:", lonvals[ifile], ',', latvals[ifile], ' ...'
+
+        onc.close()
+        ifile = ifile + 1
+        
+    Nfile = ifile
+    print '  1D-rank variables to join:', varvals.keys()
+
+    # Creation of joined file
+    ofile = 'joined_soundingstations.nc'
+
+    onewnc = NetCDFFile(ofile, 'w')
+    # dimensions
+    newdim = onewnc.createDimension('Nstations', Nfile)
+    newdim = onewnc.createDimension('StrLength', 200)
+    newdim = onewnc.createDimension('time', None)
+
+    # Creation of dimension-variables
+    newvar = onewnc.createVariable('stsref', 'c', ('Nstations', 'StrLength'))
+    basicvardef(newvar, 'station_reference', 'Reference of stations', '-')
+    newvals = writing_str_nc(newvar, refvals, 200)
+
+    newvar = onewnc.createVariable('stsname', 'c', ('Nstations', 'StrLength'))
+    basicvardef(newvar, 'station_name', 'Name of stations', '-')
+    newvals = writing_str_nc(newvar, namevals, 200)
+
+    newvar = onewnc.createVariable('stslon', 'f8', ('Nstations'))
+    basicvardef(newvar, 'lon', 'Longitude of stations', 'degrees_east')
+    newvar[:] = lonvals[:]
+
+    newvar = onewnc.createVariable('stslat', 'f8', ('Nstations'))
+    basicvardef(newvar, 'lat', 'Latitude of stations', 'degrees_north')
+    newvar[:] = latvals[:]
+
+    newvar = onewnc.createVariable('stsheight', 'f', ('Nstations'))
+    basicvardef(newvar, 'height', 'height of stations', 'm')
+    newvar[:] = heightvals[:]
+
+    # Time axis
+    timevalsort = timevals
+    timevalsort.sort()
+    newvart = onewnc.createVariable('ststimes', 'f', ('time'))
+    newvart[:] = timevalsort[:]
+    newvart.setncattr('standrad_name','time')
+    newvart.setncattr('long_name','Time')
+
+    ifile = 0
+    for fobsn in fobs:
+        filen =  namefoldobs + '/' + fobsn
+        if not os.path.isfile(filen):
+            print errormsg
+            print '  ' + fname + ": file '" + filen + "' does not exist !!"
+            print "    files found in folder '" + namefoldobs + "':", fobs 
+            quit(-1)
+
+        onc = NetCDFFile(filen, 'r')
+        ot = onc.variables['time']
+        otv = list(ot[:])
+        coint, itcoinsort, itcoinotv = gen.list_coincidences(timevalsort, otv)
+        itcoinsort.sort()
+
+        if ifile == 0:
+            tu = ot.units
+            newvart.setncattr('units',tu)
+            for varn in varns:
+                dicvn = varvals[varn]
+                vartype = dicvn[0]
+                attrns = dicvn[1]
+                attrvs = dicvn[2]
+                vardims = dicvn[3]
+                varfill = dicvn[4]
+                newdims = list(vardims) + ['Nstations']
+                for vard in vardims:
+                    if not gen.searchInlist(onewnc.dimensions, vard):
+                        print '  ' + fname + ": adding dimension '" + vard + "' !!"
+                        dobj = onc.dimensions[vard]
+                        if dobj.isunlimited():
+                            newdim = onewnc.createDimension(vard, None)
+                        else:
+                            lend = len(dobj)
+                            newdim = onewnc.createDimension(vard, lend)
+                if varfill is not None:
+                    newvar = onewnc.createVariable(varn, vartype, tuple(newdims),    \
+                      fill_value = varfill)
+                else:
+                    newvar = onewnc.createVariable(varn, vartype, tuple(newdims))
+
+                Nattrs = len(attrns)
+                for iattr in range(Nattrs):
+                    attrn = attrns[iattr]
+                    if attrn != '_FillValue':
+                        attrv = attrvs[iattr]
+                        newattr = newvar.setncattr(attrn, attrv)
+                # Global attributes
+                for attrn in onc.ncattrs():
+                    attrv = onc.getncattr(attrn)
+                    onewnc.setncattr(attrn, attrv)
+                newvar = onewnc.variables[varn]
+
+                if not onc.variables.has_key(varn):
+                    print warnmsg
+                    print '  ' + fname + ": file '" + filen + "' does not have " +   \
+                      "variable '" + varn + "' filling with missing values !!"
+                else:
+                    ovar = onc.variables[varn]
+                    varv = ovar[:]
+                    if gen.searchInlist(newvar.dimensions, 'time'):
+                        if len(newvar.shape) == 4: newvar[itcoinsort,:,:,ifile] = varv[:]
+                        elif len(newvar.shape) == 3: newvar[itcoinsort,:,ifile] = varv[:]
+                        elif len(newvar.shape) == 2: newvar[itcoinsort,ifile] = varv[:]
+                    else:
+                        if len(newvar.shape) == 4: newvar[:,:,:,ifile] = varv[:]
+                        elif len(newvar.shape) == 3: newvar[:,:,ifile] = varv[:]
+                        elif len(newvar.shape) == 2: newvar[:,ifile] = varv[:]
+        else:
+            for varn in varns:
+                newvar = onewnc.variables[varn]
+
+                dicvn = varvals[varn]
+                vartype = dicvn[0]
+                attrns = dicvn[1]
+                attrvs = dicvn[2]
+                vardims = dicvn[3]
+                varfill = dicvn[4]
+
+                if not onc.variables.has_key(varn):
+                    print warnmsg
+                    print '  ' + fname + ": file '" + filen + "' does not have " +   \
+                      "variable '" + varn + "' filling with missing values !!"
+                else:
+                    ovar = onc.variables[varn]
+                    varv = ovar[:]
+                    if gen.searchInlist(newvar.dimensions, 'time'):
+                        if len(newvar.shape) == 4: newvar[itcoinsort,:,:,ifile] = varv[:]
+                        elif len(newvar.shape) == 3: newvar[itcoinsort,:,ifile] = varv[:]
+                        elif len(newvar.shape) == 2: newvar[itcoinsort,ifile] = varv[:]
+                    else:
+                        if len(newvar.shape) == 4: newvar[:,:,:,ifile] = varv[:]
+                        elif len(newvar.shape) == 3: newvar[:,:,ifile] = varv[:]
+                        elif len(newvar.shape) == 2: newvar[:,ifile] = varv[:]
+            onewnc.sync()
+ 
+        onc.close()
+        ifile = ifile + 1
+
+    # Global attributes
+    add_global_PyNCplot(onewnc, 'nc_var_tools', fname, '1.0')
+    onewnc.sync()
+    onewnc.close()
+    print fname + ": succesfull creation of file '" + ofile + "' !!"
+
+    return
+
 def stats_lonlat_box(vals, lonv, latv, nlon, xlon, nlat, xlat):
     """ Function to provide the statistics of a variable from a lon,lat box