Changeset 2684 in lmdz_wrf for trunk

Timestamp:

Jul 26, 2019, 4:19:04 AM (5 years ago)

Author:

lfita

Message:

Adding

• Jumping of lines from file (not working?)
• Addition of new extra descriptive variables

Location:

trunk/tools

Files:

• create_OBSnetcdf.py (modified) (12 diffs)
• extra_FLUXNET-ECB-METHOD-HH.code (added)
• extra_FLUXNET-ERA-QC-HH.code (added)
• extra_FLUXNET-ERA-drivers-QC-HH.code (added)
• extra_FLUXNET-QC-HH.code (added)
• extra_FLUXNET-SD-METHOD-HH.code (added)

trunk/tools/create_OBSnetcdf.py

@@ -611 +611 @@
     return values
 
-def read_datavalues(dataf, comchar, colchar, fmt, oper, miss, varns, dbg):
+def read_datavalues(dataf, comchar, colchar, fmt, jBl, oper, miss, varns, dbg):
     """ Function to read from an ASCII file values in column
       dataf= data file
@@ -618 +618 @@
       dbg= debug mode or not
       fmt= list of kind of values to be found
+      jBl= number of lines to jump from the beginning of file
       oper= list of operations to perform
       miss= missing value
@@ -640 +641 @@
     for line in ofile:
         line = line.replace('\n','').replace(chr(13),'')
-        if not searchInlist(comchar,line[0:1]) and len(line) > 1:
+        print iline, 'jBl', jBl
+        if not searchInlist(comchar,line[0:1]) and len(line) > 1 and iline > jBl-1:
             # Getting values
             if colchar[0:4] != 'None': 
@@ -687 +689 @@
             for ivar in range(Nvals):
                 if dbg: 
-                    print iline, varns[ivar],'value:',values[ivar],miss,opers[ivar], \
-                      fmt[ivar]
+                    print iline, ',', ivar, '  ', varns[ivar],'value:',values[ivar], \
+                      miss,opers[ivar], fmt[ivar]
 
                 if iline == 0:
@@ -1203 +1205 @@
     return
 
+def EXTRAcodevar(unitsn, onc, Lstrcode=1024):
+    """ Function to add a variabe providing description of a units based on an extra
+        code
+      unitsn= name of the units (all codes derived units must be labelled as
+        'extra_code_[ref]' associated to a file valled extra_[ref].code)
+      onc= netCDF object file to add the description
+      Lstrcode= length of description of codes
+
+      extra_[ref].code must have the structure: ('#' for comments)
+        reference|web page, document reference with the code
+        short_description|main description of the code
+        long_description|long description of the code
+        codeTYPE|type of the code (andy of 'D', 'F', 'I', 'S')
+        @| Values line giving the start of the values
+        [val1]|[meaning of first value]
+        (...)
+        [valN]|[meaning of last value]
+    """
+    fname = 'EXTRAcodevar'
+
+# From http://stackoverflow.com/questions/4934806/how-can-i-find-scripts-directory-with-python
+    folder = os.path.dirname(os.path.realpath(__file__))
+
+    code = unitsn.split('_')[2]
+
+    infile = folder + '/extra_' + code +'.code'
+
+    if not os.path.isfile(infile):
+        print warnmsg
+        print '  ' + fname + ": EXTRA code file '" + infile + "' does not exist !!"
+        return
+    # main expected values
+    descvals = ['reference', 'short_description', 'long_description',    \
+      'codeTYPE', '@']
+
+    availcodetype = ['D', 'F', 'I', 'S']
+
+    codevalues = {}
+    ocode = open(infile, 'r')
+    inivals = False
+    codvals = []
+    codmeanings = []
+    for line in ocode:
+        if len(line) > 1 and line[0:1] != '#':
+            linev = line.replace('\n','').replace('\t','    ').replace('\r','')
+            if not inivals:
+                Sv = linev.split('|')[0]
+                Vv = linev.split('|')[1]
+                if searchInlist(descvals, Sv): 
+                    if Sv != '@': codevalues[Sv] = Vv
+                    else: inivals = True
+            else:
+                Svv = linev.split('|')[0]
+                Vvv = linev.split('|')[1]
+                codvals.append(Svv)
+                codmeanings.append(Vvv)
+                        
+    # Creating variable
+    if not searchInlist(onc.dimensions, 'Lstringcode'):
+        print '  ' + fname + ": Adding string length dimension 'Lstringcode' for " + \
+          " code descriptions"
+        newdim = onc.createDimension('Lstringcode', Lstrcode)
+    Ncodes = len(codvals)
+    codedimn = 'extra_code_' + str(code)
+    if not searchInlist(onc.dimensions, codedimn):
+        print '  ' + fname + ": Adding '" + codedimn + "' dimension for code " +     \
+          "description"
+        newdim = onc.createDimension(codedimn, Ncodes)
+    onc.sync()
+
+    # Variable with the value of the code
+    if not onc.variables.has_key(codedimn):
+        if codevalues['codeTYPE'] == 'D':
+            newvar = onc.createVariable(codedimn, 'f8', (codedimn))
+            for iv in range(Ncodes): newvar[iv] = np.float64(codvals[iv])
+        elif codevalues['codeTYPE'] == 'F':
+            newvar = onc.createVariable(codedimn, 'f', (codedimn))
+            for iv in range(Ncodes): newvar[iv] = np.float(codvals[iv])
+        elif codevalues['codeTYPE'] == 'I':
+            newvar = onc.createVariable(codedimn, 'i', (codedimn))
+            for iv in range(Ncodes): newvar[iv] = int(codvals[iv])
+        elif codevalues['codeTYPE'] == 'S':
+            newvar = onc.createVariable(codedimn, 'c', (codedimn, 'Lstringcode'))
+            writing_str_nc(newvar, codvals, Lstrcode)
+        else:
+            print errormsg
+            print '  ' + fname + ": codeTYPE '" + codevalues['codeTYPE'] + "' not" + \
+              " ready !!"
+            print '   available ones:', availcodetype
+            quit(-1)
+
+        for descv in descvals:
+            if descv != '@' and descv != 'codeTYPE':
+                newvar.setncattr(descv, codevalues[descv])
+    # Variable with the meaning of the code
+    if not onc.variables.has_key(codedimn+'_meaning'):
+        print '  '+fname+": Adding '" + codedimn + "_meaning' variable for code " +  \
+          "description"
+        newvar = onc.createVariable(codedimn+'_meaning','c',(codedimn,'Lstringcode'))
+        writing_str_nc(newvar, codmeanings, Lstrcode)
+        newvar.setncattr('description', 'meaning of EXTRA code ' + str(code))
+
+    onc.sync()
+
+    return
+
 def add_global_PyNCplot(ObjFile, pyscript, funcname, version):
     """ Function to add global attributes from 'PyNCplot' to a given netCDF
@@ -1253 +1361 @@
 parser.add_option("-f", "--file", dest="obsfile",
   help="observational file to use", metavar="FILE")
+parser.add_option("-j", "--jumpBlines", dest="jumpBlines",
+  help="number of lines to jump from the beggining of file", metavar="VALUE")
 parser.add_option("-g", "--debug", dest="debug",
-  help="whther debug is required ('false', 'true')", metavar="VALUE")
+  help="whether debug is required ('false', 'true')", metavar="VALUE")
 parser.add_option("-k", "--kindObs", dest="obskind", type='choice', choices=kindobs, 
   help=strkObs, metavar="VALUE")
@@ -1277 +1387 @@
 else:
     charcomments = opts.charcom.split(':')    
+
+if opts.jumpBlines is None:
+    print warnmsg
+    print '  ' + main + ': No number of lines to jump from beggining of file provided!!'
+    print '    assuming no need!'
+    jumpBlines = 0
+else:
+    jumpBlines = int(opts.jumpBlines)
+
 
 if opts.fdesc is None:
@@ -1342 +1461 @@
     if obskind == 'single-station':
         if opts.stloc is None:
-            print errornmsg
+            print errormsg
             print '  ' + main + ': No station location provided !!'
             quit(-1)
@@ -1367 +1486 @@
         Nshow = len(formats)
         for ivar in range(Nshow):
-            print '      ',description['varN'][ivar],':', description['varLN'][ivar],\
+            print ivar,'      ',description['varN'][ivar],':', description['varLN'][ivar],\
                '[', description['varU'][ivar], '] fmt:', formats[ivar]
         print '      missing values for:', description['varN'][Nshow:Nvariables+1]
@@ -1373 +1492 @@
         Nshow = Nvariables
         for ivar in range(Nshow):
-            print '      ',description['varN'][ivar],':', description['varLN'][ivar],\
+            print ivar,'      ',description['varN'][ivar],':', description['varLN'][ivar],\
                '[', description['varU'][ivar], '] fmt:', formats[ivar]
         print '      excess of formats for:', formats[Nshow:len(formats)+1]
@@ -1381 +1500 @@
 # Reading values
 ##
-datavalues = read_datavalues(opts.obsfile, charcomments, endcol, formats, 
+datavalues = read_datavalues(opts.obsfile, charcomments, endcol, formats, jumpBlines, 
   description['varOPER'], description['MissingValue'], description['varN'], debug)
 
@@ -1443 +1562 @@
     if description['varU'][ivar][0:8] == 'wmo_code':
         WMOcodevar(description['varU'][ivar], objfile)
+    # Getting new variables to describe certain units as codeEXTRA_[ref] from an 
+    #   external file
+    if description['varU'][ivar][0:10] == 'extra_code':
+        EXTRAcodevar(description['varU'][ivar], objfile)
 
 # Extra variable descriptions/attributes