import numpy as np from netCDF4 import Dataset as NetCDFFile import os import re main = 'nc_var_tools.py' errormsg='ERROR -- error -- ERROR -- error' warnmsg='WARNING -- warning -- WARNING -- warning' fillValue = 1.e20 def filexist(filen, emsg, fmsg): """ Check existence of a file filen= file name emsg= general error message fmsg= message before generic text of inexistence of file """ if not os.path.isfile(filen): print emsg print ' ' + fmsg + ' file "' + filen + '" does not exist !!' print emsg quit(-1) def varinfile(ncf, filen, emsg, vmsg, varn): """ Check existence of a variable inside a netCDF file ncf= netCDF object filen= file name emsg= general error message vmsg= message before generic text of inexistence of variable varn= variable name """ if not ncf.variables.has_key(varn): print emsg print ' file "' + filen + '" does not have ' + vmsg + ' variable "' + varn + '" !!' print emsg ncf.close() quit(-1) class testvarinfile(object): """ Check existence of a variable inside a netCDF file ncf= netCDF object filen= file name emsg= general error message vmsg= message before generic text of inexistence of variable varn= variable name self.exist= boolean value of existence self.message= error message """ def __init__(self, ncf, filen, emsg, vmsg, varn): if ncf is None: self.exist= None self.message = None else: if not ncf.variables.has_key(varn): self.exist = False self.message = emsg + '\n' + ' file "' + filen + '" does not have "' + vmsg + '" variable "' + varn + '" !!\n' + \ emsg else: self.exist = True def attrinvar(varobj, emsg, vmsg, attrn): """ Check existence of an attribute in a netCDF variable object varobj= netCDF variable object emsg= general error message vmsg= message before generic text of inexistence of variable attrrn= attribute name """ varattrs = varobj.ncattrs() if not isInlist(varattrs, attrn): print emsg print ' variable "' + varobj.name + '" does not have attribute ' + attrn + '" !!' print emsg ncf.close() quit(-1) def reduce_spaces(string): """ Function to give words of a line of text removing any extra space """ values = string.replace('\n','').split(' ') vals = [] for val in values: if len(val) > 0: vals.append(val) return vals def reduce_tabulars(string): """ Function to give words of a line of text removing any extra space and tabular separation """ values = string.replace('\n','').split('\t') vals = [] for val in values: if len(val) > 0: vals.append(val) return vals def valmodoper(varVal, valuesS): """ Function to run the modification of a variable varVAl= matrix with the values valuesS= [modins],[[modval1],...,[modvalN]] modification instruction, value with which modify [modins]: Modifiers: sumc: add [modval1] subc: remove [modval1] mulc: multiply by [modval1] divc: divide by [modval1] lowthres: if [val] < [modval1]; val = [modval2] upthres: if [val] > [modval1]; val = [modval2] potc: [val] ** [modval1] """ fname='valmodoper' if valuesS == 'h': print fname + '_____________________________________________________________' print valmodoper.__doc__ quit() valsS = valuesS.split(',') modins = valsS[0] modval = float(valsS[1]) if modins == 'sumc': varVal[:] = varVal[:] + modval elif modins == 'subc': varVal[:] = varVal[:] - modval elif modins == 'mulc': varVal[:] = varVal[:] * modval elif modins == 'divc': varVal[:] = varVal[:] / modval elif modins == 'lowthres': varVal2 = np.where(varVal[:] < float(valsS[1]), float(valsS[2]), varVal[:]) varVal[:] = varVal2 elif modins == 'upthres': varVal2 = np.where(varVal[:] > float(valsS[1]), float(valsS[2]), varVal[:]) varVal[:] = varVal2 elif modins == 'potc': varVal[:] = varVal[:] ** modval else: print errormsg print ' valmodoper: Operation to modify values ' + modins + ' is not defined !!!' print errormsg quit(-1) return varVal def valmod(values, ncfile, varn): """ Function to modify the value of a variable values = modins,[modval1],[...,[modvalN]] modins = instruction: 'sumc', add [modval1] 'subc', substraction [modval1] 'mulc', multiply by [modval1] 'divc', divide by [modval1] 'lowthres': modify all values below [modval1] to [modval2] 'upthres': modify all values above [modval1] to [modval2] 'potc': [val] ** [modval1] ncfile = netCDF file name varn = name of the variable """ fname='valmod' if values == 'h': print fname + '_____________________________________________________________' print valmod.__doc__ quit() vals = values.split(',') modins = vals[0] modval = float(vals[1]) if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ": File '" + ncfile + "' does not have variable '" + \ varn + "' !!" print errormsg ncf.close() quit(-1) var = ncf.variables[varn] varshape = var.shape vartype = var.dtype Ndims = len(varshape) varshapevals = list(varshape) Nvars = len(varshape) if not Nvars == 0: if Nvars == 1: varvals = np.zeros(varshape[0], dtype=vartype) elif Nvars == 2: varvals = np.zeros((varshape[0], varshape[1]), dtype=vartype) elif Nvars == 3: varvals = np.zeros((varshape[1], varshape[2]), dtype=vartype) elif Nvars == 4: varvals = np.zeros((varshape[2], varshape[3]), dtype=vartype) elif Nvars == 5: varvals = np.zeros((varshape[3], varshape[4]), dtype=vartype) elif Nvars == 6: varvals = np.zeros((varshape[4], varshape[5]), dtype=vartype) else: print errormsg print ' fvaradd: variable size ',Nvars,' is not ready!!!!' # Allocating all the necessary memory (just in case) if Nvars <= 2: varvals = var[:] var[:] = valmodoper(varvals, values) elif Nvars == 3: for i in range(varshape[0]): varvals[:] = var[i,:,:] var[i,:,:] = valmodoper(varvals, values) elif Nvars == 4: for i in range(varshape[0]): for j in range(varshape[1]): varvals[:] = var[i,j,:,:] var[i,j,:,:] = valmodoper(varvals, values) elif Nvars == 5: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): varvals[:] = var[i,j,k,:,:] var[i,j,k,:,:] = valmodoper(varvals, values) elif Nvars == 6: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): for l in range(varshape[3]): varvals[:] = var[i,j,k,l,:,:] var[i,j,k,l,:,:] = valmodoper(varvals, values) ncf.sync() ncf.close() def rangedim(end, shape): """Gives the instruction to retrieve values from a dimension of a variable >>> print rangedim(-1, 15) 15 """ if end == -1: return shape else: return end def varaddref(values, ncfile, varn): """ Function to add a variable in an existing file copying characteristics from an existing one values = [variable ref]:[attr name]@[value][:[attr2]@[value2], ...]:[value/file with values] add a new variable [varn] with dimension and attributes from an already existing [variable ref] with attributes [[attr name]@[value][:[attr2]@[value2], ...]] in the file [netcdf] and value [value/file with values] netcdf = netCDF file name varn = new variable name """ fname = 'varaddref' if values == 'h': print fname + '_____________________________________________________________' print varaddref.__doc__ quit() varvalues = values.split(':') Nvarvalues = len(varvalues) varprev = varvalues[0] newattrs = {} for iattr in range(Nvarvalues - 2): attrv = varvalues[iattr+1] newattrs[attrv.split('@')[0]] = attrv.split('@')[1] ncf = NetCDFFile(ncfile,'a') if ncf.variables.has_key(varn): print errormsg print ' varaddref: File already has the varible ' + varn + ' !!!' print errormsg ncf.close() quit(-1) if not ncf.variables.has_key(varprev): print errormsg print ' varaddref: File does not have variable ' + varprev + ' !!!!' print errormsg ncf.close() quit(-1) varp = ncf.variables[varprev] varpVal = varp[:] varpshape = varp.shape Nvarpshape = len(varpshape) varpattrs = varp.ncattrs() varptype = varp.dtype varpdims = varp.dimensions varpndims = varp.ndim ## print ' shape of the variable used as reference: ',varpshape # # variable characteristics if len(varpshape) == 4: dimpt = varpshape[0] dimpz = varpshape[1] dimpy = varpshape[2] dimpx = varpshape[3] elif len(varpshape) == 3: dimpt = varpshape[0] dimpy = varpshape[1] dimpx = varpshape[2] elif len(varpshape) == 3: dimpy = varpshape[0] dimpx = varpshape[1] newvar = ncf.createVariable(varn, varptype, dimensions=varpdims) newvar = ncf.variables[varn] # # Values varplen=1 for idim in range(varpndims): varplen=varplen*varpshape[idim] if varptype == 'float' or varptype == 'float32' or varptype == 'float64' or vartype == np.float(1.) or vartype == np.float32(1.): newvals = np.reshape(np.arange(varplen), varpshape)*1. else: newvals = np.reshape(np.arange(varplen), varpshape) if not os.path.isfile(varvalues[Nvarvalues-1]): ## print ' Using constant value' newvals[:] = float(varvalues[Nvarvalues-1]) else: ## print ' Using 2-D values from ' + varvalues[Nvarvalues-1] asciif = open(varvalues[Nvarvalues-1], 'r') fdimx = len(asciif.readlines()) asciif.seek(0) if len(varpshape) == 4: if not fdimx == dimpx: print errormsg print ' varaddref: Provided file has dimx=', fdimx, ' and variable has dimx=', dimpx, ' !!!' print errormsg ncf.close() quit(-1) iline = 0 idx = 0 for fline in asciif: line = fline.replace('\n','') yvals = [] for iyval in line.split(' '): yvals.append(float(iyval)) if iline == 0: fdimy = len(yvals) if not fdimy == dimpy: print errormsg print ' varaddref: Provided file has dimy=', fdimy, ' and variable has dimy= ', dimpy, ' !!!' print errormsg ncf.close() quit(-1) for it in range(dimpt): for iz in range(dimpz): newvals[it,iz,:,idx] = yvals idx = idx+1 iline = iline+1 elif len(varpshape) == 3: if not fdimx == dimpx: print errormsg print ' varaddref: Provided file has dimx=', fdimx, ' and variable has dimx=', dimpx, ' !!!' print errormsg ncf.close() quit(-1) iline = 0 idx = 0 for fline in asciif: line = fline.replace('\n','') yvals = [] for iyval in line.split(' '): yvals.append(float(iyval)) if iline == 0: fdimy = len(yvals) if not fdimy == dimpy: print errormsg print ' varaddref: Provided file has dimy=', fdimy, ' and variable has dimy= ',dimpy, ' !!!' print errormsg ncf.close() quit(-1) for it in range(dimpt): newvals[it,:,idx] = yvals idx = idx+1 iline = iline+1 elif len(varpshape) == 2: if not fdimx == dimpx: print errormsg print ' varaddref: Provided file has dimx=', fdimx, ' and variable has dimx=', dimpx, ' !!!' print errormsg ncf.close() quit(-1) iline = 0 idx = 0 for fline in asciif: line = fline.replace('\n','') yvals = [] for iyval in line.split(' '): yvals.append(float(iyval)) if iline == 0: fdimy = len(yvals) if not fdimy == dimpy: print errormsg print ' varaddref: Provided file has dimy=', fdimy, ' and variable has dimy= ',dimpy, ' !!!' print errormsg ncf.close() quit(-1) newvals[:,idx] = yvals idx = idx+1 iline = iline+1 asciif.close() newvar[:] = newvals # # Attributes for iattr in range(len(varpattrs)): attrn = varpattrs[iattr] attrval = varp.getncattr(attrn) if attrn in newattrs: newattr = newvar.setncattr(attrn, newattrs[attrn]) else: newattr = newvar.setncattr(attrn, attrval) ncf.sync() ncf.close() def varoutold(values, ncfile, varn): """ Function when we want to output variable values values = [optsIrange]:[optsErange] [optsIrange]: val1,val2,...,valN inital value for the 'N' dimensions of the variable [optsErange]: val1,val2,...,valN ending value for the 'N' dimensions of the variable ncfile = netCDF file name varn = variable name """ import numpy as np from netCDF4 import Dataset as NetCDFFile optsIrange = values.split(':')[0] optsErange = values.split(':')[1] inirange=optsIrange.split(',') endrange=optsErange.split(',') irange = [int(val) for val in inirange] erange = [int(val) for val in endrange] if not len(irange) == len(erange): print errormsg print ' varout: Different number of values in each range!' print ' varout: initial range: ' + optsIrange print ' varout: ending range: ' + optsErange print errormsg quit(-1) else: ndims=len(irange) if not os.path.isfile(ncfile): print errormsg print ' varout: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'r') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' varout: File "' + ncfile + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) var = ncf.variables[varn] varshape = var.shape Nvarshape = len(varshape) if not Nvarshape == ndims: print errormsg print ' varout: Provided number of values of the range ' + ndims + ' is different of the shape of the variable ' + Nvarshape + ' !!!' print errormsg ncf.close() quit(-1) if Nvarshape == 1: varValrange = var[irange[0]:rangedim(erange[0], varshape[0])] elif Nvarshape == 2: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1]), varshape[1]] elif Nvarshape == 3: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1], varshape[1]), irange[2]:rangedim(erange[2], varshape[2])] elif Nvarshape == 4: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1], varshape[1]), irange[2]:rangedim(erange[2], varshape[2]), \ irange[3]:rangedim(erange[3], varshape[3])] elif Nvarshape == 5: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1], varshape[1]), irange[2]:rangedim(erange[2], varshape[2]), \ irange[3]:rangedim(erange[3], varshape[3]), irange[4]:rangedim(erange[4], varshape[4])] elif Nvarshape == 6: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1], varshape[1]), irange[2]:rangedim(erange[2], varshape[2]), \ irange[3]:rangedim(erange[3], varshape[3]), irange[4]:rangedim(erange[4], varshape[4]), \ irange[5]:rangedim(erange[5], varshape[5])] ncf.close() Nshaperange = len(varValrange.shape) Noneshape = 0 for ir in range(Nshaperange): if varValrange.shape[ir] == 1: Noneshape = Noneshape + 1 if Noneshape == Nshaperange - 1: for i in range(len(varValrange)): print '%2s %f' % ( 'NC', varValrange[i] ) def varout(values, ncfile, varn): """ Function when we want to output variable values values = [dimname1]:[valdim1]|[dimname2]:[valdim2]|[...,[dimnameN]:[valdimN]] [valdim]: * [integer]: which value of the dimension * -1: all along the dimension * -9: last value of the dimension * [beg]:[end] slice from [beg] to [end] ncfile = netCDF file name varn = variable name """ fname = 'varout' if values == 'h': print fname + '_____________________________________________________________' print varout.__doc__ quit() ncobj = NetCDFFile(ncfile, 'r') if not ncobj.variables.has_key(varn): print errormsg print ' ' + fname + ': file "' + ncfile + '" does not have variable "' + \ varn + '" !!' quit(-1) vobj = ncobj.variables[varn] newvar, newdims = slice_variable(vobj, values) Nnewdims = len(newdims) if Nnewdims == 1: for i in range(newvar.shape[0]): print '%2s %f' % ( 'NC',newvar[i] ) elif Nnewdims == 2: for i0 in range(newvar.shape[0]): row = numVector_String(newvar[i0,:],' ') print 'NC ' + row else: for i0 in range(newvar.shape[Nnewdims-2]): varslice = [] dimsinf = '' for idim in range(Nnewdims - 2): for d in range(newvar.shape[idim]): if idim == 0: dimsinf = newdims[idim] + ': ' + str(d) else: dimsinf = dimsinf + ' ' + newdims[idim] + ': ' + str(d) varslice.append(d) varslice.append(i0) varslice.append(0,newvar.shape[Nnewdims-1]) row = numVector_String(newvar[tuple(varslice)],' ') print 'NCinf ' + dimsinf + '_________________' print 'NC ' + row ncobj.close() return def chdimname(values, ncfile, varn): """ Changing the name of the dimension values = [olddimname]:[newdimname] [olddimname]: old name of the dimension [newdimname]: new name of the dimension ncfile = netCDF file name varn = variable name """ fname = 'chdimname' if values == 'h': print fname + '_____________________________________________________________' print chdimname.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' chdimname: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') olddimname=values.split(':')[0] newdimname=values.split(':')[1] if not ncf.dimensions.has_key(olddimname): print warnmsg print ' chdimname: File "' + ncfile + '" does not have dimension "' + olddimname + '" !!!!' ncf.close() quit() if not olddimname == newdimname and not ncf.dimensions.has_key(newdimname): newname = ncf.renameDimension(olddimname, newdimname) ncf.sync() ncf.close() else: print warnmsg print ' chdimname: File "' + ncfile + '" already has dimension name "' + newdimname + '" ' print ' chdimname: modifying all the variables which use the old dimension' filevars = ncf.variables for fvarn in filevars: if ncf.variables.has_key(fvarn): fvar = ncf.variables[fvarn] fvardims = fvar.dimensions if searchInlist(fvardims, olddimname): print ' variable "' + fvarn + '" uses dimension "' + olddimname + '" ' varinf = variable_inf(fvar) newdims = tuple(fvardims) change = {olddimname: newdimname} # From http://stackoverflow.com/questions/9067043/python-replace-list-values-using-a-tuple newdims = tuple([ change.get(x,x) for x in fvardims ]) newvar = ncf.createVariable(fvarn + 'tmpdname', varinf.dtype, newdims, fill_value=varinf.FillValue) varv = fvar[:] newvar[:] = varv for attrn in varinf.attributes: attrv = fvar.getncattr(attrn) newar = newvar.setncattr(attrn, attrv) fvar = ncf.renameVariable(fvarn, fvarn + 'rmdname') ncf.sync() newvar = ncf.renameVariable(fvarn + 'tmpdname', fvarn) ncf.sync() ncf.close() varrm(ncfile, fvarn + 'rmdname') ncf = NetCDFFile(ncfile,'a') def chvarname(values, ncfile, varn): """Changing the name of the variable values = new variable name ncfile = netCDF file varn = name of the variable """ fname = 'chvarname' if values == 'h': print fname + '_____________________________________________________________' print chvarname.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print warnmsg print ' ' + fname + ': File does not have variable "' + varn + '" !!!!' ncf.close() quit(-1) if not varn == values and not ncf.variables.has_key(values): newname = ncf.renameVariable(varn, values) ncf.sync() ncf.close() def searchInlist(listname, nameFind): """ Function to search a value within a list listname = list nameFind = value to find >>> searInlist(['1', '2', '3', '5'], '5') True """ for x in listname: if x == nameFind: return True return False def set_attribute(ncv, attrname, attrvalue): """ Sets a value of an attribute of a netCDF variable. Removes previous attribute value if exists ncv = object netcdf variable attrname = name of the attribute attrvalue = value of the attribute """ attvar = ncv.ncattrs() if searchInlist(attvar, attrname): attr = ncv.delncattr(attrname) attr = ncv.setncattr(attrname, attrvalue) return attr def set_attributek(ncv, attrname, attrval, attrkind): """ Sets a value of an attribute of a netCDF variable with a kind. Removes previous attribute value if exists ncvar = object netcdf variable attrname = name of the attribute attrvalue = value of the attribute atrtrkind = kind of attribute: 'S', string ('!' as spaces); 'I', integer; 'Inp32', numpy integer 32; 'R', real; ' npfloat', np.float; 'D', np.float64 """ if attrkind == 'S': attrvalue = str(attrval.replace('!', ' ')) elif attrkind == 'I': attrvalue = int(attrval) elif attrkind == 'Inp32': attrvalue = np.int32(attrval) elif attrkind == 'R': attrvalue = float(attrval) elif attrkind == 'npfloat': attrvalue = np.float(attrval) elif attrkind == 'D': attrvalue = np.float64(attrval) else: print errormsg print ' set_attributek: "' + attrkind + '" kind of attribute is not ready!' quit(-1) attvar = ncv.ncattrs() if searchInlist(attvar, attrname): attr = ncv.delncattr(attrname) attr = ncv.setncattr(attrname, attrvalue) return attr def basicvardef(varobj, vstname, vlname, vunits): """ Function to give the basic attributes to a variable varobj= netCDF variable object vstname= standard name of the variable vlname= long name of the variable vunits= units of the variable """ attr = varobj.setncattr('standard_name', vstname) attr = varobj.setncattr('long_name', vlname) attr = varobj.setncattr('units', vunits) return def gaddattr(values, ncfile): """ Add a global attribute to a netCDF. Removes previous attribute if it exist values = [attrname]|[attrvalue ('!' as spaces)] ncfile = netCDF file """ if not os.path.isfile(ncfile): print errormsg print ' gaddattr: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') attrvals=values.split('|') attrn=attrvals[0] attrv=attrvals[1].replace('!', ' ') ncf = set_attribute(ncf, attrn, attrv) ncf.sync() ncf.close() def gaddattrk(values, ncfile): """ Add a global attribute to a netCDF caring about the type. Removes previous attribute if it exist values = [attrname]|[attrvalue]|[attrk] attrname = name of the attribute attrvalue = value of the attribute attrk = 'S', string ('!' as spaces); 'I', integer; 'Inp32', numpy integer 32; 'R', real; 'D', double ncfile = netCDF file """ if not os.path.isfile(ncfile): print errormsg print ' gaddattrk: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') attrvals=values.split('|') attrn=attrvals[0] attrv0=attrvals[1] attrk=attrvals[2] if attrk == 'S': attrv = str(attrv0.replace('!', ' ')) elif attrk == 'I': attrv = int(attrv0) elif attrk == 'Inp32': attrv = np.int32(attrv0) elif attrk == 'R': attrv = float(attrv0) elif attrk == 'D': attrv = np.float64(attrv0) else: print errormsg print ' gaddattrk: "' + attrk + '" kind of attribute is not ready!' ncf.close() quit(-1) ncf = set_attribute(ncf, attrn, attrv) ncf.sync() ncf.close() def varaddattr(values, ncfile, varn): """ Add an attribute to a variable. Removes previous attribute if it exists values = [attrname]|[attrvalue('!' as spaces)] ncfile = netCDF file name varn = name of the variable """ fname = 'varaddattr' if values == 'h': print fname + '_____________________________________________________________' print varaddattr.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname +': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname +': File "' + ncfile + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) attrvals=values.split('|') attrn=attrvals[0] attrv=attrvals[1].replace('!', ' ') var = ncf.variables[varn] var = set_attribute(var, attrn, attrv) ncf.sync() ncf.close() def varaddattrk(values, ncfile, varn): """ Add an attribute to a variable caring about the type values = [attrname]|[attrvalue]|[attrk] attrname = name of the attribute attrvalue = value of the attribute attrk = 'S', string ('!' as spaces); 'I', integer; 'R', real; 'D', double; 'npR', numpy Real, 'npR32', numpy Real-32 ncfile = netCDF file varn = variable name """ fname = 'varaddattrk' if values == 'h': print fname + '_____________________________________________________________' print varaddattrk.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ': File "' + ncfile + '"does not have variable ' + varn + ' !!!!' print errormsg ncf.close() quit(-1) attrvals=values.split('|') attrn=attrvals[0] attrv0=attrvals[1] attrk=attrvals[2] var = ncf.variables[varn] if attrk == 'S': attrv = str(attrv0.replace('!', ' ')) elif attrk == 'I': attrv = int(attrv0) elif attrk == 'R': attrv = float8(attrv0) elif attrk == 'D': attrv = np.float64(attrv0) elif attrk == 'npR': attrv = np.float(attrv0) elif attrk == 'npR32': attrv = np.float32(attrv0) else: print errormsg print ' ' + fname + ': "' + attrk + '" kind of attribute is not ready!' ncf.close() quit(-1) var = set_attribute(var, attrn, attrv) ncf.sync() ncf.close() def varrmattr(values, ncfile, varn): """ Removing an attribute from a variable values = attribute name ncfile = netCDF file name varn = variable name """ fname = 'varrmattr' if values == 'h': print fname + '_____________________________________________________________' print varrmattr.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname +': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname +': File "' + ncfile + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) var = ncf.variables[varn] attvar = var.ncattrs() if searchInlist(attvar, values): attr = var.delncattr(values) else: print warnmsg print ' ' + fname +': "' + varn + '" does not have attribute: ' + values ncf.sync() ncf.close() def grmattr(values, ncfile): """ Removing a global attribute values = attribute name ncfile = netCDF file """ if not os.path.isfile(ncfile): print errormsg print ' grmattr: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') attvar = ncf.ncattrs() if searchInlist(attvar, values): attr = ncf.delncattr(values) else: print warnmsg print ' grmattr: "' + ncfile + '" does not have attribute: ' + values ncf.sync() ncf.close() def fvaradd(values, ncfile): """ Adding variable (and all its attributes and dimensions) from a reference file to a file values = [netCDFref]:[varnref] netCDFref = netCDF file name as reference for the variable to add varnref = name of the variable from [netCDFref] to be added ncfile = netCDF file name """ fname = 'fvaradd' if values == 'h': print fname + '_____________________________________________________________' print fvaradd.__doc__ quit() refnc = values.split(':')[0] refvar = values.split(':')[1] if not os.path.isfile(ncfile): print errormsg print ' fvaradd: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) if not os.path.isfile(refnc): print errormsg print ' fvaradd: Reference file "' + refnc + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') refvars = ncref.variables if searchInlist(refvars, refvar): refvarv = ncref.variables[refvar] else: print errormsg print ' fvaradd: File "' + refnc + '" does not have variable: ' + refvar ncf.close() ncref.close() quit(-1) vardims = refvarv.dimensions vartype = refvarv.dtype varattr = refvarv.ncattrs() # Checking dimensions ## newdims = ncf.dimensions for rdim in vardims: if not searchInlist(newdims, rdim): print ' fvaradd: Adding dimension ' + rdim ncf.close() ncref.close() fdimadd(refnc + ',' + rdim, ncfile) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') # Checking fill value ## if searchInlist(varattr, '_FillValue'): varfil = refvarv._FillValue else: varfil = False print ' fvaradd: Adding refvar:', refvar, 'shape: ', refvarv.shape var = ncf.createVariable(refvar, vartype, vardims, fill_value=varfil) varshape = refvarv.shape Nvars = len(varshape) if not Nvars == 0: if Nvars == 1: varvals = np.zeros(varshape[0], dtype=vartype) elif Nvars == 2: varvals = np.zeros((varshape[0], varshape[1]), dtype=vartype) elif Nvars == 3: varvals = np.zeros((varshape[0], varshape[1], varshape[2]), dtype=vartype) elif Nvars == 4: varvals = np.zeros((varshape[0], varshape[1], varshape[2], varshape[3]), dtype=vartype) elif Nvars == 5: varvals = np.zeros((varshape[0], varshape[1], varshape[2], varshape[3], varshape[4]), dtype=vartype) elif Nvars == 6: varvals = np.zeros((varshape[0], varshape[1], varshape[2], varshape[3], varshape[4], varshape[5]), dtype=vartype) else: print errormsg print ' fvaradd: variable size ',Nvars,' is not ready!!!!' # Allocating all the necessary memory (just in case) var[:] = varvals if Nvars <= 2: varvals = refvarv[:] var[:] = varvals elif Nvars == 3: for i in range(varshape[0]): varvals[i,:,:] = refvarv[i,:,:] var[i,:,:] = varvals[i,:,:] elif Nvars == 4: for i in range(varshape[0]): for j in range(varshape[1]): varvals[i,j,:,:] = refvarv[i,j,:,:] var[i,j,:,:] = varvals[i,j,:,:] elif Nvars == 5: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): varvals[i,j,k,:,:] = refvarv[i,j,k,:,:] var[i,j,k,:,:] = varvals[i,j,k,:,:] elif Nvars == 6: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): for l in range(varshape[3]): varvals[i,j,k,l,:,:] = refvarv[i,j,k,l,:,:] var[i,j,k,l,:,:] = varvals[i,j,k,l,:,:] newvar = ncf.variables[refvar] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = refvarv.getncattr(attr) if not searchInlist(newvarattrs, attr): newvar.setncattr(attr, attrv) ncf.sync() ncf.close() ncref.close() def fdimadd(values, ncfile): """ Adding dimension from another reference file values = [refnc],[refdim] refnc = netCDF file name as reference refdim = name of the dimension to be added ncfile = netCDF file name """ fname='fdimadd' refnc = values.split(',')[0] refdim = values.split(',')[1] if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) if not os.path.isfile(refnc): print errormsg print ' ' + fname + ': Reference file "' + refnc + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') refdims = ncref.dimensions if searchInlist(refdims, refdim): refdimv = ncref.dimensions[refdim] else: print errormsg print ' ' + fname + ': File "' + refnc + '" does not have dimension: "' + refdim + '"' ncf.close() ncref.close() quit(-1) if refdimv.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(refdimv) print ' ' + fname + ': Adding refdim:', refdim, 'size:', dimsize dim = ncf.createDimension(refdim, dimsize) ncf.sync() ncf.close() ncref.close() def fattradd(var, values, ncfile): """ Adding attributes from a reference file var = variable to which has to be added the attribute values = [refnc]:[refvar] refnc = netCDF file name as reference refvar = variable from the reference file ncfile = netCDF file name """ refnc = values.split(':')[0] refvar = values.split(':')[1] if not os.path.isfile(ncfile): print errormsg print ' fattradd: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) if not os.path.isfile(refnc): print errormsg print ' fattradd: Reference file "' + refnc + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') vars = ncf.variables if searchInlist(vars, var): varv = ncf.variables[var] else: print ' fattradd: File "' + ncfile + '" does not have variable: "' + var + '"' ncf.close() ncref.close() quit(-1) refvars = ncref.variables if searchInlist(refvars, refvar): refvarv = ncref.variables[refvar] else: print ' fattradd: File "' + refnc + '" does not have variable: "' + refvar + '"' ncf.close() ncref.close() quit(-1) refvarattrs = refvarv.ncattrs() Nattrs = len(refvarattrs) print ' fattradd: Adding ', Nattrs,' atributes from:', refvar for attr in refvarattrs: attrv = refvarv.getncattr(attr) atvar = set_attribute(varv, attr, attrv) ncf.sync() ncf.close() ncref.close() def fgaddattr(values, ncfile): """ Adding global attributes from a reference file values = netCDF file name as reference ncfile = netCDF file name """ refnc = values if not os.path.isfile(ncfile): print errormsg print ' fgaddattr: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) if not os.path.isfile(refnc): print errormsg print ' fgaddattr: Reference file "' + refnc + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') refgattrs = ncref.ncattrs() Nattrs = len(refgattrs) print ' fgaddattr: Adding ', Nattrs,' global atributes' for attr in refgattrs: attrv = ncref.getncattr(attr) atvar = set_attribute(ncf, attr, attrv) ncf.sync() ncf.close() ncref.close() def varrm(ncfile, var): """ Removing a variable from a file ncfile = netCDF object file var = variable name to remove """ import shutil as shu fname = 'varrm' if var == 'h': print fname + '_____________________________________________________________' print varrm.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' varrm: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') ncvars = ncf.variables ncf.close() if not searchInlist(ncvars, var): print ' varrm: File "' + ncfile + '" does not have variable: "' + var + '"' ncf.close() quit(-1) tmpncf = NetCDFFile('tmp_py.nc' , 'w') gtmpattr = set_attribute(tmpncf, 'copy', 'temporal') tmpncf.sync() tmpncf.close() for varn in ncvars: if not varn == var: fvaradd(ncfile + ':' + varn, 'tmp_py.nc') fgaddattr(ncfile, 'tmp_py.nc') shu.copyfile('tmp_py.nc', ncfile) os.remove('tmp_py.nc') def ivars(ncfile): """Give all the variable names of a file ncfile= netCDFfile name from which all variables will be given """ fname = 'ivars' if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'r') ncvars = ncf.variables allvars='' for var in ncvars: print var allvars=allvars + ':' + var print ' # allvars= ' + allvars ncf.close() def igattrs(ncfile): """Give all the global attributes of a file """ if not os.path.isfile(ncfile): print errormsg print ' igattrs: File "' + ncfile + '" does not exist !!' print errormsg quit() ncf = NetCDFFile(ncfile,'r') ncattrs = ncf.ncattrs() allattrs='' allattrsdic = {} for attr in ncattrs: attrv=ncf.getncattr(attr) if type(attrv) == type(str('1')): attrk = 'S' elif type(attrv) == type(unicode('1')): attrk = 'S' elif type(attrv) == type(int(1)): attrk = 'I' elif type(attrv) == type(np.int(1)): attrk = 'Inp' elif type(attrv) == type(np.int32(1)): attrk = 'Inp32' elif type(attrv) == type(float(1.)): attrk = 'R' elif type(attrv) == type(np.float32(1.)): attrk = 'Rnp32' elif type(attrv) == type(np.float64(1.)): attrk = 'Rnp64' else: print errormsg print ' igattr: Reading attribute "', type(attrv), '" not ready! value:', attrv ncf.close() quit(-1) print attr, '|', attrv, '|', attrk allattrsdic[attr] = [attrv, attrk] allattrs=allattrs + ':' + attr + '|' + str(attrv) + '|' + attrk print '####### ###### ##### #### ### ## #' print '# allgattrs= ' + allattrs ncf.close() return allattrsdic def isgattrs(values, ncfile): """Give a single global attribute of a file and its type values = attribute name ncfile = netCDF file name output: attribute name, '|', attribute value, '|', attribute kind ('S', string '!' as spaces; 'I', integer; 'R', real; 'D', double ) """ if not os.path.isfile(ncfile): print errormsg print ' isgattrs: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'r') ncattrs = ncf.ncattrs() if not searchInlist(ncattrs, values): print ' isgattrs: File "' + ncfile + '" does not have global attribute: "' + values + '" ' ncf.close() quit(-1) attrv=ncf.getncattr(values) if type(attrv) == type(str('1')): attrk = 'S' attrv = attrv.replace(' ','!') elif type(attrv) == type(unicode('1')): attrk = 'S' attrv = attrv.replace(' ','!') elif type(attrv) == type(int(1)): attrk = 'I' elif type(attrv) == type(np.int(1)): attrk = 'I' elif type(attrv) == type(np.int32(1)): attrk = 'I' elif type(attrv) == type(float(1.)): attrk = 'R' elif type(attrv) == type(np.float32(1.)): attrk = 'R' elif type(attrv) == type(np.float64(1.)): attrk = 'D' else: print errormsg print ' isgattr: Reading attribute "', type(attrv), '" not ready! value:', attrv ncf.close() quit(-1) print values, '|', attrv, '|', attrk ncf.close() return [values, attrv, attrk] def ivattrs(ncfile, varn): """Give all the attributes of a variable and its type ncfile = netCDF file name var = variable name output: attribute name, '|', attribute value, '|', attribute kind ('S', string '!' as spaces; 'I', integer; 'R', real; 'D', double ) """ if not os.path.isfile(ncfile): print errormsg print ' ivattrs: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'r') ncvars = ncf.variables if not searchInlist(ncvars, varn): print ' ivattrs: File "' + ncfile + '" does not have variable: "' + varn + '" ' ncf.close() quit(-1) varval = ncf.variables[varn] ncattrs = varval.ncattrs() allattrs='' allattrsdic = {} for attr in ncattrs: attrv=varval.getncattr(attr) if type(attrv) == type(str('1')): attrk = 'S' attrv = attrv.replace(' ','!') elif type(attrv) == type(unicode('1')): attrk = 'S' attrv = attrv.replace(' ','!') elif type(attrv) == type(int(1)): attrk = 'I' elif type(attrv) == type(np.int(1)): attrk = 'I' elif type(attrv) == type(np.int32(1)): attrk = 'I' elif type(attrv) == type(float(1.)): attrk = 'R' elif type(attrv) == type(np.float32(1.)): attrk = 'R' elif type(attrv) == type(np.float64(1.)): attrk = 'D' else: print errormsg print ' ivattrs: Reading attribute "', type(attrv), '" not ready! value:', attrv ncf.close() quit(-1) print attr, '|', attrv, '|', attrk allattrsdic[attr] = [attrv, attrk] allattrs=allattrs + ':' + attr + '|' + str(attrv) + '|' + attrk print '####### ###### ##### #### ### ## #' print '# allvattrs= ' + allattrs ncf.close() return allattrsdic def isvattrs(values, ncfile, varn): """Give a single attribute of a variable values = attribute name ncfile = netCDF file name varn = variable name """ if not os.path.isfile(ncfile): print errormsg print ' isvattrs:File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'r') ncvars = ncf.variables if not searchInlist(ncvars, varn): print errormsg print ' isvattrs:"' + ncfile + '" does not have variable: "' + varn + '" ' ncf.close() quit(-1) varval = ncf.variables[varn] ncattrs = varval.ncattrs() if not searchInlist(ncattrs, values): print errormsg print ' isvattrs:' + ncfile + ' does not have global attribute: "' + values + '" ' ncf.close() quit(-1) attrv=varval.getncattr(values) if type(attrv) == type(str('1')): attrk = 'S' elif type(attrv) == type(unicode('1')): attrk = 'S' elif type(attrv) == type(int(1)): attrk = 'I' elif type(attrv) == type(np.int(1)): attrk = 'I' elif type(attrv) == type(np.int32(1)): attrk = 'I' elif type(attrv) == type(float(1.)): attrk = 'R' elif type(attrv) == type(np.float(1.)): attrk = 'R' elif type(attrv) == type(np.float32(1.)): attrk = 'R' elif type(attrv) == type(np.float64(1.)): attrk = 'D' else: print errormsg print ' isvattr: Reading attribute "', type(attrv), '" not ready! value:', attrv ncf.close() quit(-1) print values, '|', attrv, '|', attrk ncf.close() return [values, attrv, attrk] def grattr(values, ncfile): """ Function to read a global atribute values = attribute name ncfile = netCDF file name """ ncf = NetCDFFile(ncfile,'r') glob_attrs = ncf.ncattrs() attrPos = searchInlist(glob_attrs, values) if not attrPos: print errormsg print ' grattr: File "' + ncfile + '" does not have attribute "' + values + '" !!!!' print errormsg ncf.close() quit(-1) print ncf.getncattr(values) ncf.close() def vrattr(values, ncfile, varn): """ Function to remove an atribute from a variable values = attribute name ncfile = netCDF file name varn = variable name """ fname = 'vrattr' ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ": File '" + ncfile + "' does not have variable '" + \ varn + "' !!!!" ncf.close() quit(-1) var = ncf.variables[varn] var_attrs = var.ncattrs() if not searchInlist(var_attrs, values): print errormsg print ' ' + fname + ": Variable '" + varn + "' does not have attribute '" + \ values + "' !!!!" ncf.close() quit(-1) else: attr = var.delncattr(values) ncf.close() return def datetimeStr_datetime(StringDT): """ Function to transform a string date ([YYYY]-[MM]-[DD]_[HH]:[MI]:[SS] format) to a date object >>> datetimeStr_datetime('1976-02-17_00:00:00') 1976-02-17 00:00:00 """ import datetime as dt fname = 'datetimeStr_datetime' dateD = np.zeros((3), dtype=int) timeT = np.zeros((3), dtype=int) dateD[0] = int(StringDT[0:4]) dateD[1] = int(StringDT[5:7]) dateD[2] = int(StringDT[8:10]) trefT = StringDT.find(':') if not trefT == -1: # print ' ' + fname + ': refdate with time!' timeT[0] = int(StringDT[11:13]) timeT[1] = int(StringDT[14:16]) timeT[2] = int(StringDT[17:19]) if int(dateD[0]) == 0: print warnmsg print ' ' + fname + ': 0 reference year!! changing to 1' dateD[0] = 1 newdatetime = dt.datetime(dateD[0], dateD[1], dateD[2], timeT[0], timeT[1], timeT[2]) return newdatetime def dateStr_date(StringDate): """ Function to transform a string date ([YYYY]-[MM]-[DD] format) to a date object >>> dateStr_date('1976-02-17') 1976-02-17 """ import datetime as dt dateD = StringDate.split('-') if int(dateD[0]) == 0: print warnmsg print ' dateStr_date: 0 reference year!! changing to 1' dateD[0] = 1 newdate = dt.date(int(dateD[0]), int(dateD[1]), int(dateD[2])) return newdate def timeStr_time(StringDate): """ Function to transform a string date ([HH]:[MI]:[SS] format) to a time object >>> datetimeStr_datetime('04:32:54') 04:32:54 """ import datetime as dt timeT = StringDate.split(':') if len(timeT) == 3: newtime = dt.time(int(timeT[0]), int(timeT[1]), int(timeT[2])) else: newtime = dt.time(int(timeT[0]), int(timeT[1]), 0) return newtime def timeref_datetime(refd, timeval, tu): """ Function to transform from a [timeval] in [tu] units from the time referece [tref] to datetime object refd: time of reference (as datetime object) timeval: time value (as [tu] from [tref]) tu: time units >>> timeref = date(1949,12,1,0,0,0) >>> timeref_datetime(timeref, 229784.36, hours) 1976-02-17 08:21:36 """ import datetime as dt import numpy as np ## Not in timedelta # if tu == 'years': # realdate = refdate + dt.timedelta(years=float(timeval)) # elif tu == 'months': # realdate = refdate + dt.timedelta(months=float(timeval)) if tu == 'weeks': realdate = refd + dt.timedelta(weeks=float(timeval)) elif tu == 'days': realdate = refd + dt.timedelta(days=float(timeval)) elif tu == 'hours': realdate = refd + dt.timedelta(hours=float(timeval)) elif tu == 'minutes': realdate = refd + dt.timedelta(minutes=float(timeval)) elif tu == 'seconds': realdate = refd + dt.timedelta(seconds=float(timeval)) elif tu == 'milliseconds': realdate = refd + dt.timedelta(milliseconds=float(timeval)) else: print errormsg print ' timeref_datetime: time units "' + tu + '" not ready!!!!' quit(-1) return realdate def timeref_datetime_mat(refd, timeval, tu): """ Function to transform from a [timeval] in [tu] units from the time referece [tref] to matrix with: year, day, month, hour, minute, second refd: time of reference (as datetime object) timeval: time value (as [tu] from [tref]) tu: time units >>> timeref = date(1949,12,1,0,0,0) >>> timeref_datetime(timeref, 229784.36, hours) [1976 2 17 8 36 21] """ import datetime as dt import numpy as np realdates = np.zeros(6, dtype=int) ## Not in timedelta # if tu == 'years': # realdate = refdate + dt.timedelta(years=float(timeval)) # elif tu == 'months': # realdate = refdate + dt.timedelta(months=float(timeval)) if tu == 'weeks': realdate = refd + dt.timedelta(weeks=float(timeval)) elif tu == 'days': realdate = refd + dt.timedelta(days=float(timeval)) elif tu == 'hours': realdate = refd + dt.timedelta(hours=float(timeval)) elif tu == 'minutes': realdate = refd + dt.timedelta(minutes=float(timeval)) elif tunits == 'seconds': realdate = refd + dt.timedelta(seconds=float(timeval)) elif tunits == 'milliseconds': realdate = refd + dt.timedelta(milliseconds=float(timeval)) else: print errormsg print ' timeref_datetime: time units "' + tu + '" not ready!!!!' quit(-1) realdates[0] = int(realdate.year) realdates[1] = int(realdate.month) realdates[2] = int(realdate.day) realdates[3] = int(realdate.hour) realdates[4] = int(realdate.second) realdates[5] = int(realdate.minute) return realdates def realdatetime_CFcompilant(times, Srefdate, tunits): """ Function to transform a matrix with real time values ([year, month, day, hour, minute, second]) to a netCDF one times= matrix with times Srefdate= reference date ([YYYY][MM][DD][HH][MI][SS] format) tunits= units of time respect to Srefdate >>> realdatetime_CFcompilant(np.array([ [1976, 2, 17, 8, 20, 0], [1976, 2, 18, 8, 20, 0]], dtype=int), '19491201000000', 'hours') [ 229784.33333333 229808.33333333] """ import datetime as dt yrref=int(Srefdate[0:4]) monref=int(Srefdate[4:6]) dayref=int(Srefdate[6:8]) horref=int(Srefdate[8:10]) minref=int(Srefdate[10:12]) secref=int(Srefdate[12:14]) refdate=dt.datetime(yrref, monref, dayref, horref, minref, secref) dimt=times.shape[0] cfdates = np.zeros((dimt), dtype=np.float64) if tunits == 'weeks': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = (cfdate.days + cfdate.seconds/(3600.*24.))/7. elif tunits == 'days': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days + cfdate.seconds/(3600.*24.) elif tunits == 'hours': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*24. + cfdate.seconds/3600. elif tunits == 'minutes': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*24.*60. + cfdate.seconds/60. elif tunits == 'seconds': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*24.*3600. + cfdate.seconds elif tunits == 'milliseconds': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*1000.*24.*3600. + cfdate.seconds*1000. elif tunits == 'microseconds': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*1000000.*24.*3600. + cfdate.seconds*1000000. else: print errormsg print ' ' + fname + ': time units "' + tunits + '" is not ready!!!' quit(-1) return cfdates def netCDFdatetime_realdatetime(units, tcalendar, times): """ Function to transfrom from netCDF CF-compilant times to real time """ import datetime as dt txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Calendar type ## is360 = False if tcalendar is not None: print ' netCDFdatetime_realdatetime: There is a calendar attribute' if tcalendar == '365_day' or tcalendar == 'noleap': print ' netCDFdatetime_realdatetime: No leap years!' isleapcal = False elif tcalendar == 'proleptic_gregorian' or tcalendar == 'standard' or tcalendar == 'gregorian': isleapcal = True elif tcalendar == '360_day': is360 = True isleapcal = False else: print errormsg print ' netCDFdatetime_realdatetime: Calendar "' + tcalendar + '" not prepared!' quit(-1) # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: print ' netCDFdatetime_realdatetime: refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) dimt = len(times) # datetype = type(dt.datetime(1972,02,01)) # realdates = np.array(dimt, datetype) # print realdates ## Not in timedelta # if tunits == 'years': # for it in range(dimt): # realdate = refdate + dt.timedelta(years=float(times[it])) # realdates[it] = int(realdate.year) # elif tunits == 'months': # for it in range(dimt): # realdate = refdate + dt.timedelta(months=float(times[it])) # realdates[it] = int(realdate.year) # realdates = [] realdates = np.zeros((dimt, 6), dtype=int) if tunits == 'weeks': for it in range(dimt): realdate = refdate + dt.timedelta(weeks=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'days': for it in range(dimt): realdate = refdate + dt.timedelta(days=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'hours': for it in range(dimt): realdate = refdate + dt.timedelta(hours=float(times[it])) # if not isleapcal: # Nleapdays = cal.leapdays(int(refdate.year), int(realdate.year)) # realdate = realdate - dt.timedelta(days=Nleapdays) # if is360: # Nyears360 = int(realdate.year) - int(refdate.year) + 1 # realdate = realdate -dt.timedelta(days=Nyears360*5) # realdates[it] = realdate # realdates = refdate + dt.timedelta(hours=float(times)) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'minutes': for it in range(dimt): realdate = refdate + dt.timedelta(minutes=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'seconds': for it in range(dimt): realdate = refdate + dt.timedelta(seconds=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'milliseconds': for it in range(dimt): realdate = refdate + dt.timedelta(milliseconds=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'microseconds': for it in range(dimt): realdate = refdate + dt.timedelta(microseconds=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] else: print errormsg print ' netCDFdatetime_realdatetime: time units "' + tunits + '" is not ready!!!' quit(-1) return realdates class cls_time_information(object): """ Classs to provide information about variable time ncfu = netCDF unit name tname = name of the variable time in [ncfu] self.calendar: calendar of the variable time self.unitsval: units value as it appears in attributes self.units: units of variable time self.Urefdate: reference date as it appears in the original 'units' section self.Srefdate: reference date as string [YYYY][MM][DD][HH][MI][SS] self.refdate: reference date self.attributes: attributes of time variable self.dimt: dimension of the time variable self.dt: distance between first two time-steps in self.tunits self.firstTu: first time value in self.units self.firstTt: first time value as datetime object self.firstTS: first time value as string [YYYY][MM][DD][HH][MI][SS] self.firstTm: first time value as matrix (from datetime; [Y], [M], [D], [H], [M], [S]) self.lastTu: last time value in self.units self.firstTt: last time value as datetime object self.lastTS: last time value as string [YYYY][MM][DD][HH][MI][SS] self.lastTm: last time value as matrix (from datetime; [Y], [M], [D], [H], [M], [S]) """ import datetime as dt def __init__(self, ncfu, tname): if ncfu is None: self.unitsval = None self.units = None self.calendar = None self.Urefdate = None self.Srefdate = None self.refdate = None self.attributes = None self.dimt = None self.dt = None self.firstTu = None self.firstTt = None self.firstTS = None self.firstTm = None self.lastTu = None self.lastTt = None self.lastTS = None self.lastTm = None else: times = ncfu.variables[tname] attvar = times.ncattrs() self.attributes = attvar if not searchInlist(attvar, 'units'): print errormsg print ' cls_time_information: time variable "', tname, '" does not have attribute: "units"' quit(-1) else: units = times.getncattr('units') self.unitsval = units if not searchInlist(attvar, 'calendar'): print warnmsg print ' cls_time_information: time variable "', tname, '" does not have attribute: "calendar"' self.calendar = '-' else: self.calendar = times.getncattr('calendar') txtunits = units.split(' ') self.units = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: # print ' refdate with time!' self.refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) self.Urefdate = txtunits[len(txtunits) - 2] + ' ' + Srefdate else: self.refdate = datetimeStr_datetime(Srefdate + '_00:00:00') self.Urefdate = Srefdate self.Srefdate = self.refdate.strftime("%Y%m%d%H%M%S") timev = times[:] self.dimt = times.shape[0] self.dt = timev[1]-timev[0] self.firstTu = timev[0] self.firstTt = timeref_datetime(self.refdate, self.firstTu, self.units) self.firstTS = self.firstTt.strftime("%Y%m%d%H%M%S") self.firstTm = timeref_datetime_mat(self.refdate, self.firstTu, self.units) self.lastTu = timev[self.dimt-1] self.lastTt = timeref_datetime(self.refdate, self.lastTu, self.units) self.lastTS = self.lastTt.strftime("%Y%m%d%H%M%S") self.lastTm = timeref_datetime_mat(self.refdate, self.lastTu, self.units) def time_information(ncfu, tname): """ Function to provide information about variable time ncfu = netCDF unit name tname = name of the variable time in [ncfu] """ times = ncfu.variables[tname] timeinf = [] attvar = times.ncattrs() if not searchInlist(attvar, 'units'): print errormsg print ' "time" does not have attribute: "units"' quit(-1) else: units = times.getncattr('units') txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: # print ' refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) timeinf.append(tunits) timeinf.append(Srefdate) timeinf.append(refdate) return timeinf def CFtimes_datetime(ncfile, tname): """ Provide date/time array from a file with a series of netCDF CF-compilant time variable ncfile = netCDF file name tname = name of the variable time in [ncfile] output: array(dimt, 0) = year array(dimt, 1) = month array(dimt, 2) = day array(dimt, 3) = hour array(dimt, 4) = minute array(dimt, 5) = second """ import datetime as dt times = ncfile.variables[tname] timevals = times[:] attvar = times.ncattrs() if not searchInlist(attvar, 'units'): print errormsg print ' CFtimes_datetime: "time" does not have attribute: "units"' quit(-1) else: units = times.getncattr('units') txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: # print ' refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) dimt = len(timevals) realdates = np.zeros((dimt, 6), dtype=int) ## Not in timedelta # if tunits == 'years': # for it in range(dimt): # realdate = refdate + dt.timedelta(years=float(times[it])) # realdates[it] = int(realdate.year) # elif tunits == 'months': # for it in range(dimt): # realdate = refdate + dt.timedelta(months=float(times[it])) # realdates[it] = int(realdate.year) if tunits == 'weeks': for it in range(dimt): realdate = refdate + dt.timedelta(weeks=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'days': for it in range(dimt): realdate = refdate + dt.timedelta(days=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'hours': for it in range(dimt): realdate = refdate + dt.timedelta(hours=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'minutes': for it in range(dimt): realdate = refdate + dt.timedelta(minutes=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'seconds': for it in range(dimt): realdate = refdate + dt.timedelta(seconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'milliseconds': for it in range(dimt): realdate = refdate + dt.timedelta(milliseconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) else: print errormsg print ' CFtimes_datetime: time units "' + tunits + '" not ready!!!!' quit(-1) return realdates class variable_inf(object): """ Class to provide the information from a given variable var = object netCDF variable self.name: name of the variable self.dtype: type of the variable self.attributes: list with the name of attributes self.FillValue: value of the missing value self.dimns: name of the dimensions of the variable self.dims: dimensions of the variable self.Ndims: number of dimensions self.dimx: length of dimension along x-axis self.dimy: length of dimension along y-axis self.sname: standard name self.lname: long name self.corr: attribute 'coordinates' self.units: units of the variable """ def __init__(self, var): if var is None: self.name = None self.dimns = None self.dims = None self.Ndims = None self.dimx = None self.dimy = None self.sname = None self.lname = None self.corr = None self.units = None self.FillValue = None self.dtype = None self.attributes = None else: self.name = var._name self.dimns = var.dimensions self.dtype = var.dtype self.attributes = var.ncattrs() self.dims = var.shape if searchInlist(self.attributes, 'standard_name'): self.sname = var.getncattr('standard_name') else: # print ' variable_inf.classpy: variable "' + self.name + '" does not have attribute "standard_name"' self.sname = None if searchInlist(self.attributes, 'long_name'): self.lname = var.getncattr('long_name') else: # print ' variable_inf.classpy: variable "' + self.name + '" does not have attribute "long_name"' self.lname = None if searchInlist(self.attributes, 'coordinates'): self.coor = var.getncattr('coordinates') else: # print ' variable_inf.classpy: variable "' + self.name + '" does not have attribute "coordinates"' self.coor = None if searchInlist(self.attributes, 'units'): self.units = var.getncattr('units') else: # print ' variable_inf.classpy: variable "' + self.name + '" does not have attribute "units"' self.units = None if searchInlist(self.attributes, '_FillValue'): self.FillValue = var.getncattr('_FillValue') else: # print ' variable_inf.classpy: variable "' + self.name + '" does not have attribute "_FillValue"' self.FillValue = None self.Ndims = len(self.dims) if self.Ndims == 1: self.dimx=self.dims[0] if self.Ndims == 2: self.dimy=self.dims[0] self.dimx=self.dims[1] if self.Ndims == 3: self.dimy=self.dims[1] self.dimx=self.dims[2] if self.Ndims == 4: self.dimy=self.dims[2] self.dimx=self.dims[3] def subyrs(values, ncfile, varn): """ Function to retrieve a series of years from a file values = [year1]:[[year2]:...[yearn]] values for years [year1], [year2], ... [yearn] [yearI]-[yearE] values for the period between [yearI] and [yearN] ncfile = netCDF file name varn = variable name """ import datetime as dt import calendar as cal ofile = 'subsetyrs.nc' ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' subyrs: File does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) times = ncf.variables['time'] timevals = times[:] realdates = CFtimes_datetime(ncf, 'time') dimt = len(timevals) # Checking years ## isper = values.find('-') desiredvalues = np.array(dimt, dtype=bool) desiredvaluesi = np.array(dimt, dtype=bool) desiredvaluese = np.array(dimt, dtype=bool) if not isper == -1: # Years list given as a period [YYYYi]-[YYYYf] print ' subyrs: There is a period of years "' + values + '"' iyr = int(values.split('-')[0]) eyr = int(values.split('-')[1]) desiredvaluesi = np.array(realdates[:,0] >= iyr) desiredvaluese = np.array(realdates[:,0] <= eyr) desiredvalues = desiredvaluesi*desiredvaluese else: yrs = values.split(':') nyr = 1 for iyr in range(len(yrs)): desiredvaluesi = np.array(realdates[:,0] == int(yrs[iyr])) if nyr == 1: desiredvalues = desiredvaluesi else: desiredvalues = desiredvaluesi+desiredvalues nyr = nyr + 1 Ndesiredvalues = len(realdates[desiredvalues]) print ' subyrs: N values: ', Ndesiredvalues if Ndesiredvalues == 0: print errormsg print ' subyrs: No values found for "' + values + '"!!' ncf.close() quit(-1) # Variable values (assuming time as first dimension) ## var = ncf.variables[varn] # varvals = var[:] vardims = var.shape varshape = len(vardims) print ' subyrs: Shape of data: ',varshape, ':' , vardims if varshape == 1: vardesiredvalues = np.arange(Ndesiredvalues) vardesiredvalues = var[desiredvalues] elif varshape == 2: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]).reshape(Ndesiredvalues,vardims[1]) vardesiredvalues = var[desiredvalues, :] elif varshape == 3: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]).reshape(Ndesiredvalues, \ vardims[1],vardims[2]) vardesiredvalues = var[desiredvalues, :, :] elif varshape == 4: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]).reshape(Ndesiredvalues, \ vardims[1],vardims[2],vardims[3]) vardesiredvalues = var[desiredvalues, :, :, :] elif varshape == 5: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]*vardims[4]).reshape(Ndesiredvalues, \ vardims[1],vardims[2],vardims[3],vardims[4]) vardesiredvalues = var[desiredvalues, :, :, :, :] elif varshape == 6: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]*vardims[4]* \ vardims[5]).reshape(Ndesiredvalues,vardims[1],vardims[2],vardims[3],vardims[4],vardims[5]) vardesiredvalues = var[desiredvalues, :, :, :, :, :] else: print errormsg print ' subyrs: ', varshape, ' shape of matrix not prepared !' ncf.close() quit(-1) # print ' shape of desired values: ', vardesiredvalues.shape # Creation of file ## ncfo = NetCDFFile( ofile, 'w') vardims = var.dimensions vartype = var.dtype varattr = var.ncattrs() # Checking dimensions ## newdims = ncfo.dimensions for rdim in vardims: if not searchInlist(newdims, rdim): if not rdim == 'time': print ' subyrs: Adding dimension ' + rdim ncfo.sync() ncf.close() ncfo.close() fdimadd(ncfile + ',' + rdim, ofile) ncf = NetCDFFile(ncfile,'r') ncfo = NetCDFFile(ofile,'a') else: ncfo.createDimension('time', None) # Checking fill value ## if searchInlist(varattr, '_FillValue'): varfil = var._FillValue else: varfil = False newvar = ncfo.createVariable(varn, vartype, vardims, fill_value=varfil) if not varshape == 0: newvar[:] = vardesiredvalues newvar = ncfo.variables[varn] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = var.getncattr(attr) if not searchInlist(newvarattrs, attr): newvar.setncattr(attr, attrv) vardims = times.dimensions vartype = times.dtype varattr = times.ncattrs() newvar = ncfo.createVariable('time', vartype, vardims, fill_value=varfil) newvar = ncfo.variables['time'] newvar[:] = timevals[desiredvalues] ncf.close() ncfo.sync() ncfo.close() fattradd('time', ncfile + ':time', ofile) fvaradd(ncfile + ':lon', ofile) fvaradd(ncfile + ':lat', ofile) ncfo = NetCDFFile(ofile,'a') newvar = ncfo.variables['time'] newvarattrs = newvar.ncattrs() if searchInlist(newvarattrs, 'bounds'): if newvar.getncattr('bounds') == 'time_bnds': ncf = NetCDFFile(ncfile,'r') tbnds = ncf.variables['time_bnds'] vardims = tbnds.dimensions vartype = tbnds.dtype varattr = tbnds.ncattrs() ncfo.createDimension('bnds', 2) newvar = ncfo.createVariable('time_bnds', vartype, vardims, fill_value=varfil) newvar[:] = tbnds[desiredvalues,:] ncf.close() ncfo.sync() ncfo.close() fattradd('time_bnds', ncfile + ':time_bnds', ofile) else: ncfo.close() else: ncfo.close() fgaddattr(ncfile, ofile) print ' subyrs: File "' + ofile + '" with a subset of ' + values + ' has been created' def submns(values, ncfile, varn): """ Function to retrieve a series of months from a file values = [mon1]:[[mion2]:...[monn]] values for months [mon1], [mon2], ... [monn] [monI]-[monE] values for the period between [monI] and [monN] ncfile = netCDF file name varn = variable name """ import datetime as dt import calendar as cal ofile = 'subsetmns.nc' ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' submns: File "' + ncfile + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) times = ncf.variables['time'] timevals = times[:] realdates = CFtimes_datetime(ncf, 'time') dimt = len(timevals) # Checking months ## isper =values.find('-') desiredvalues = np.array(dimt, dtype=bool) desiredvaluesi = np.array(dimt, dtype=bool) desiredvaluese = np.array(dimt, dtype=bool) if not isper == -1: # Months list given as a period [MMi]-[MMf] print ' submns: There is a period of months "' + values + '"' imn = int(values.split('-')[0]) emn = int(values.split('-')[1]) desiredvaluesi = np.array(realdates[:,1] >= imn) desiredvaluese = np.array(realdates[:,1] <= emn) desiredvalues = desiredvaluesi*desiredvaluese else: mns = values.split(':') nmn = 1 for imn in range(len(mns)): desiredvaluesi = np.array(realdates[:,1] == int(mns[imn])) if nmn == 1: desiredvalues = desiredvaluesi else: desiredvalues = desiredvaluesi+desiredvalues nmn = nmn + 1 Ndesiredvalues = len(realdates[desiredvalues]) print ' submns: N values: ', Ndesiredvalues if Ndesiredvalues == 0: print errormsg print ' submns: No values found for "' + values + '"!!' ncf.close() quit(-1) # Variable values (assuming time as first dimension) ## var = ncf.variables[varn] # varvals = var[:] vardims = var.shape varshape = len(vardims) # print ' submns: Shape of data: ',varshape, ':' , vardims if varshape == 1: vardesiredvalues = np.arange(Ndesiredvalues) vardesiredvalues = var[desiredvalues] elif varshape == 2: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]).reshape(Ndesiredvalues,vardims[1]) vardesiredvalues = var[desiredvalues, :] elif varshape == 3: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]).reshape(Ndesiredvalues, \ vardims[1],vardims[2]) vardesiredvalues = var[desiredvalues, :, :] elif varshape == 4: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]).reshape(Ndesiredvalues, \ vardims[1],vardims[2],vardims[3]) vardesiredvalues = var[desiredvalues, :, :, :] elif varshape == 5: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]*vardims[4]).reshape(Ndesiredvalues, \ vardims[1],vardims[2],vardims[3],vardims[4]) vardesiredvalues = var[desiredvalues, :, :, :, :] elif varshape == 6: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]*vardims[4]* \ vardims[5]).reshape(Ndesiredvalues,vardims[1],vardims[2],vardims[3],vardims[4],vardims[5]) vardesiredvalues = var[desiredvalues, :, :, :, :, :] else: print errormsg print ' submns: ', varshape, ' shape of matrix not prepared !' ncf.close() quit(-1) # print ' shape of desired values: ', vardesiredvalues.shape # Creation of file ## ncfo = NetCDFFile( ofile, 'w') vardims = var.dimensions vartype = var.dtype varattr = var.ncattrs() # Checking dimensions ## newdims = ncfo.dimensions for rdim in vardims: if not searchInlist(newdims, rdim): if not rdim == 'time': print ' submns: Adding dimension ' + rdim ncfo.sync() ncf.close() ncfo.close() fdimadd(ncfile + ',' + rdim, ofile) ncf = NetCDFFile(ncfile,'r') ncfo = NetCDFFile(ofile,'a') else: ncfo.createDimension('time', None) # Checking fill value ## if searchInlist(varattr, '_FillValue'): varfil = var._FillValue else: varfil = False newvar = ncfo.createVariable(varn, vartype, vardims, fill_value=varfil) if not varshape == 0: newvar[:] = vardesiredvalues newvar = ncfo.variables[varn] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = var.getncattr(attr) if not searchInlist(newvarattrs, attr): newvar.setncattr(attr, attrv) vardims = times.dimensions vartype = times.dtype varattr = times.ncattrs() newvar = ncfo.createVariable('time', vartype, vardims, fill_value=varfil) newvar = ncfo.variables['time'] newvar[:] = timevals[desiredvalues] ncf.close() ncfo.sync() ncfo.close() fattradd('time', ncfile + ':time', ofile) fvaradd(ncfile + ':lon', ofile) fvaradd(ncfile + ':lat', ofile) ncfo = NetCDFFile(ofile,'a') newvar = ncfo.variables['time'] newvarattrs = newvar.ncattrs() if searchInlist(newvarattrs, 'bounds'): if newvar.getncattr('bounds') == 'time_bnds': ncf = NetCDFFile(ncfile,'r') tbnds = ncf.variables['time_bnds'] vardims = tbnds.dimensions vartype = tbnds.dtype varattr = tbnds.ncattrs() ncfo.createDimension('bnds', 2) newvar = ncfo.createVariable('time_bnds', vartype, vardims, fill_value=varfil) newvar[:] = tbnds[desiredvalues,:] ncf.close() ncfo.sync() ncfo.close() fattradd('time_bnds', ncfile + ':time_bnds', ofile) else: ncfo.close() else: ncfo.close() fgaddattr(ncfile, ofile) print ' submns: File "' + ofile + '" with a subset of ' + values + ' has been created' class statsValWeigthed(object): """Weigthed Statistics class providing: vals = values (can be a matrix) wgs = weights (can be a matrix) self.meanv: mean weigthed value self.mean2v: mean quadratic weigthed value self.stdv: weigthed standard deviation self.Nokvalue non None values of a list of values self.meanwgt: mean of the weigths self.mean2wgt: cuadratic mean of the weigths self.stdwgt: standard deviation of the weigths """ def __init__(self, vals, wgs): if vals is None: self.Nv = None self.meanv = None self.mean2v = None self.stdv = None self.Nokvalues = None self.meanwgt = None self.mean2wgt = None self.stdwgt = None else: values = vals.flatten() weights = wgs.flatten() self.Nv=len(values) self.meanv=0. self.mean2v=0. self.stdv=0. self.meanwgt = 0. self.mean2wgt = 0. self.stdwgt = 0. self.Nokvalues = 0 for inum in range(self.Nv): if not values[inum] is None: self.Nokvalues = self.Nokvalues + 1 self.meanv = self.meanv+values[inum]*weights[inum] self.mean2v = self.mean2v+values[inum]*weights[inum]*values[inum] self.meanwgt = self.meanwgt+weights[inum] self.mean2wgt = self.mean2wgt+weights[inum]*weights[inum] self.meanv = self.meanv/float(self.meanwgt) self.mean2v = self.mean2v/float(self.meanwgt) self.stdv = np.sqrt(self.mean2v-self.meanv*self.meanv) self.meanwgt = self.meanwgt/float(self.Nokvalues) self.mean2wgt = self.mean2wgt/float(self.Nokvalues) self.stdwgt = np.sqrt(self.mean2wgt-self.meanwgt*self.meanwgt) class statsValWeighted_missVal(object): """Weighted Statistics taking into account a missing value class providing: vals = values (can be a matrix) wgs = weights (can be a matrix) missVal= missing value self.meanv: mean weigthed value self.mean2v: mean quadratic weigthed value self.stdv: weigthed standard deviation self.Nokvalue non None values of a list of values self.meanwgt: mean of the weigths self.mean2wgt: cuadratic mean of the weigths self.stdwgt: standard deviation of the weigths self.quantilesv: quantiles of the weighted values """ def __init__(self, vals, wgs, missVal): fname='statsValWeigthed_missVal' if vals is None: self.Nv = None self.meanv = None self.mean2v = None self.stdv = None self.Nokvalues = None self.meanwgt = None self.mean2wgt = None self.stdwgt = None self.quantilesv = None else: Npt = 1 for idim in range(len(vals.shape)): Npt = Npt * vals.shape[idim] if np.sum(vals >= missVal) == Npt: print errormsg print ' ' + fname + ' all values get missing!!' print errormsg quit(-1) vals0 = np.where(vals >= missVal, None, vals) values = vals0.flatten() weights = wgs.flatten() self.Nv=Npt self.meanv=0. self.mean2v=0. self.stdv=0. self.meanwgt = 0. self.mean2wgt = 0. self.stdwgt = 0. self.Nokvalues = 0 valswgt = values for inum in range(self.Nv): if not values[inum] is None: self.Nokvalues = self.Nokvalues + 1 valswgt[inum] = valswgt[inum]*weights[inum] self.meanv = self.meanv+values[inum]*weights[inum] self.mean2v = self.mean2v+values[inum]*weights[inum]*values[inum] self.meanwgt = self.meanwgt+weights[inum] self.mean2wgt = self.mean2wgt+weights[inum]*weights[inum] self.meanv = self.meanv/float(self.meanwgt) self.mean2v = self.mean2v/float(self.meanwgt) self.stdv = np.sqrt(self.mean2v-self.meanv*self.meanv) valswgt = valswgt/np.float(self.meanwgt) self.meanwgt = self.meanwgt/float(self.Nokvalues) self.mean2wgt = self.mean2wgt/float(self.Nokvalues) self.stdwgt = np.sqrt(self.mean2wgt-self.meanwgt*self.meanwgt) valsq=Quantiles(valswgt, 20) self.quantilesv=valsq.quantilesv class stats2Val(object): """two variables Statistics class providing: vals1 = variable 1 vals2 = variable 2 power = power of the polynomial fitting to apply between both variables self.min[var], self.max[var], self.mean[var], self.mean2[var], self.std[var] of [var] = var1+var2[v1Av2], var1-var2[v1Sv2], var1/var2[v1Dv2], var1*var2[v1Pv2] self.Nokvalues1: number of correct values of variable 1 self.Nokvalues2: number of correct values of variable 2 self.Nokvalues12: number of correct coincident values of variable 1 and variable 2 self.mae=mean(abs(var1-var2)) self.rmse=sqrt((var1-var2)**2) self.correlation (and p-value) self.linRegress: linear regression [trend, intercept, regression coefficient, p_value, standard error] self.polRegress: polinomial Regresion of degree [power] [coef**[power], coef**[power-1], ...., coef**0] """ def __init__(self, vals1, vals2, power): import numpy as np from scipy import stats as sts if vals1 is None: self.Nv = None self.Nokvalues1 = None self.Nokvalues2 = None self.Nokvalues12 = None self.NDvalNone = None self.minv1Av2 = None self.maxv1Av2 = None self.meanv1Av2 = None self.mean2v1Av2 = None self.stdv1Av2 = None self.minv1Sv2 = None self.maxv1Sv2 = None self.meanv1Sv2 = None self.mean2v1Sv2 = None self.stdv1Sv2 = None self.minv1Dv2 = None self.maxv1Dv2 = None self.meanv1Dv2 = None self.mean2v1Dv2 = None self.stdv1Dv2 = None self.minv1Pv2 = None self.maxv1Pv2 = None self.meanv1Pv2 = None self.mean2v1Pv2 = None self.stdv1Pv2 = None self.mae = None self.rmse = None self.corr = None self.linRegress = None self.polRegress = None self.polRegressResidual = None self.polRegressRes = None self.polRegressSingVal = None else: values1 = vals1.flatten() values2 = vals2.flatten() if not len(values1) == len(values2): print errormsg print ' stats2Val: lengths of variables differ!! Lvar1: ', len(values1), ' Lvar2: ',len(values2),' statistics between them can not be computed!' quit(-1) self.Nv=len(values1) self.minv1Av2=10000000000. self.maxv1Av2=-self.minv1Av2 self.meanv1Av2=0. self.mean2v1Av2=0. self.stdv1Av2=0. self.minv1Sv2=self.minv1Av2 self.maxv1Sv2=-self.minv1Av2 self.meanv1Sv2=0. self.mean2v1Sv2=0. self.stdv1Sv2=0. self.minv1Dv2=self.minv1Av2 self.maxv1Dv2=-self.minv1Av2 self.meanv1Dv2=0. self.mean2v1Dv2=0. self.stdv1Dv2=0. self.minv1Pv2=self.minv1Av2 self.maxv1Pv2=-self.minv1Av2 self.meanv1Pv2=0. self.mean2v1Pv2=0. self.stdv1Pv2=0. self.mae = 0. self.rmse = 0. self.corr = np.array([0., 0.]) self.linRegress = np.zeros(5, float) self.polRegress = np.zeros(power+1, float) self.polRegressResidual = 0. self.polRegressSingVal = np.zeros(power+1, float) # v1 [+ / - / / / *] v2 ## self.Nokvalues1 = 0 self.Nokvalues2 = 0 self.Nokvalues12 = 0 self.NDvalNone = 0 for inum in range(self.Nv): if not values1[inum] is None: self.Nokvalues1 = self.Nokvalues1 + 1 if not values2[inum] is None: self.Nokvalues2 = self.Nokvalues2 + 1 if not values1[inum] is None and not values2[inum] is None: self.Nokvalues12 = self.Nokvalues12 + 1 Aval = values1[inum] + values2[inum] Sval = values1[inum] - values2[inum] Pval = values1[inum] * values2[inum] if np.isinf(values1[inum] / values2[inum]) or np.isnan(values1[inum] / values2[inum]): if self.NDvalNone < 1: print warnmsg print ' stats2Val: val1/val2 inf or Nan!!!!' Dval = None self.NDvalNone = self.NDvalNone + 1 else: Dval = values1[inum] / values2[inum] self.mae = self.mae + abs(Sval) self.rmse = self.rmse + Sval**2 if Aval < self.minv1Av2: self.minv1Av2 = Aval if Aval > self.maxv1Av2: self.maxv1Av2 = Aval if Sval < self.minv1Sv2: self.minv1Sv2 = Sval if Sval > self.maxv1Sv2: self.maxv1Sv2 = Sval if not Dval is None and Dval < self.minv1Dv2: self.minv1Dv2 = Dval if not Dval is None and Dval > self.maxv1Dv2: self.maxv1Dv2 = Dval if Pval < self.minv1Pv2: self.minv1Pv2 = Pval if Pval > self.maxv1Pv2: self.maxv1Pv2 = Pval self.meanv1Av2 = self.meanv1Av2+Aval self.mean2v1Av2 = self.mean2v1Av2+Aval*Aval self.meanv1Sv2 = self.meanv1Sv2+Sval self.mean2v1Sv2 = self.mean2v1Sv2+Sval*Sval if not Dval is None: self.meanv1Dv2 = self.meanv1Dv2+Dval self.mean2v1Dv2 = self.mean2v1Dv2+Dval*Dval self.meanv1Pv2 = self.meanv1Pv2+Pval self.mean2v1Pv2 = self.mean2v1Pv2+Pval*Pval ## print 'Nokvalues1: ', self.Nokvalues1, 'Nokvalues2: ', self.Nokvalues2, 'Nokvalues12: ', float(self.Nokvalues12), 'NDvalNone: ',self.NDvalNone self.meanv1Av2 = self.meanv1Av2/float(self.Nokvalues12) self.mean2v1Av2 = self.mean2v1Av2/float(self.Nokvalues12) self.stdv1Av2 = np.sqrt(self.mean2v1Av2-self.meanv1Av2*self.meanv1Av2) self.meanv1Sv2 = self.meanv1Sv2/float(self.Nokvalues12) self.mean2v1Sv2 = self.mean2v1Sv2/float(self.Nokvalues12) self.stdv1Sv2 = np.sqrt(self.mean2v1Sv2-self.meanv1Sv2*self.meanv1Sv2) if self.Nokvalues12 - self.NDvalNone == 0: self.meanv1Dv2 = None self.mean2v1Dv2 = None self.stdv1Dv2 = None print warnmsg print ' stats2Val: all values of val1/val2 are None!' else: self.meanv1Dv2 = self.meanv1Dv2/(float(self.Nokvalues12 - self.NDvalNone)) self.mean2v1Dv2 = self.mean2v1Dv2/(float(self.Nokvalues12 - self.NDvalNone)) self.stdv1Dv2 = np.sqrt(self.mean2v1Dv2-self.meanv1Dv2*self.meanv1Dv2) self.meanv1Pv2 = self.meanv1Pv2/float(self.Nokvalues12) self.mean2v1Pv2 = self.mean2v1Pv2/float(self.Nokvalues12) self.stdv1Pv2 = np.sqrt(self.mean2v1Pv2-self.meanv1Pv2*self.meanv1Pv2) self.mae = self.mae/self.Nokvalues12 self.rmse = np.sqrt(self.rmse/self.Nokvalues12) self.corr = sts.pearsonr(values1, values2) self.linRegress[0], self.linRegress[1], self.linRegress[2], self.linRegress[3], self.linRegress[4] = sts.linregress(values1, values2) polyfitvals=np.polyfit(values1, values2, power, full = True) self.polRegress = polyfitvals[0] self.polRegressRes = polyfitvals[1] self.polRegressSingVal = polyfitvals[3] class stats2Val_missVal(object): """two variables Statistics taking into account a missing value class providing: vals1 = variable 1 vals2 = variable 2 missVal = missing value power = power of the polynomial fitting to apply between both variables self.min[var], self.max[var], self.mean[var], self.mean2[var], self.std[var] of [var] = var1+var2[v1Av2], var1-var2[v1Sv2], var1/var2[v1Dv2], var1*var2[v1Pv2] self.Nokvalues1: number of correct values of variable 1 self.Nokvalues2: number of correct values of variable 2 self.Nokvalues12: number of correct coincident values of variable 1 and variable 2 self.mae=mean(abs(var1-var2)) self.rmse=sqrt((var1-var2)**2) self.correlation (and p-value) self.linRegress: linear regression [trend, intercept, regression coefficient, p_value, standard error] self.polRegress: polinomial Regresion of degree [power] [coef**[power], coef**[power-1], ...., coef**0] """ def __init__(self, vals1, vals2, power, missVal): import numpy as np from scipy import stats as sts fname='stats2Val_missVal' if vals1 is None: self.Nv = None self.Nokvalues1 = None self.Nokvalues2 = None self.Nokvalues12 = None self.NDvalNone = None self.minv1Av2 = None self.maxv1Av2 = None self.meanv1Av2 = None self.mean2v1Av2 = None self.stdv1Av2 = None self.minv1Sv2 = None self.maxv1Sv2 = None self.meanv1Sv2 = None self.mean2v1Sv2 = None self.stdv1Sv2 = None self.minv1Dv2 = None self.maxv1Dv2 = None self.meanv1Dv2 = None self.mean2v1Dv2 = None self.stdv1Dv2 = None self.minv1Pv2 = None self.maxv1Pv2 = None self.meanv1Pv2 = None self.mean2v1Pv2 = None self.stdv1Pv2 = None self.mae = None self.rmse = None self.corr = None self.linRegress = None self.polRegress = None self.polRegressResidual = None self.polRegressRes = None self.polRegressSingVal = None else: Npt1 = 1 for idim in range(len(vals1.shape)): Npt1 = Npt1 * vals1.shape[idim] if np.sum(vals1 >= missVal) == Npt1: print errormsg print ' ' + fname + ' all values 1 get missing!!' print errormsg quit(-1) Npt2 = 1 for idim in range(len(vals2.shape)): Npt2 = Npt2 * vals2.shape[idim] if np.sum(vals2 >= missVal) == Npt2: print errormsg print ' ' + fname + ' all values 2 get missing!!' print errormsg quit(-1) vals10 = np.where(abs(vals1) >= missVal, None, vals1) vals20 = np.where(abs(vals2) >= missVal, None, vals2) values1 = vals10.flatten() values2 = vals20.flatten() if not len(values1) == len(values2): print errormsg print ' stats2Val: lengths of variables differ!! Lvar1: ', len(values1), ' Lvar2: ',len(values2),' statistics between them can not be computed!' quit(-1) self.Nv=Npt1 self.minv1Av2=10000000000. self.maxv1Av2=-self.minv1Av2 self.meanv1Av2=0. self.mean2v1Av2=0. self.stdv1Av2=0. self.minv1Sv2=self.minv1Av2 self.maxv1Sv2=-self.minv1Av2 self.meanv1Sv2=0. self.mean2v1Sv2=0. self.stdv1Sv2=0. self.minv1Dv2=self.minv1Av2 self.maxv1Dv2=-self.minv1Av2 self.meanv1Dv2=0. self.mean2v1Dv2=0. self.stdv1Dv2=0. self.minv1Pv2=self.minv1Av2 self.maxv1Pv2=-self.minv1Av2 self.meanv1Pv2=0. self.mean2v1Pv2=0. self.stdv1Pv2=0. self.mae = 0. self.rmse = 0. self.corr = np.array([0., 0.]) self.linRegress = np.zeros(5, float) self.polRegress = np.zeros(power+1, float) self.polRegressResidual = 0. self.polRegressSingVal = np.zeros(power+1, float) # v1 [+ / - / / / *] v2 ## self.Nokvalues1 = 0 self.Nokvalues2 = 0 self.Nokvalues12 = 0 self.NDvalNone = 0 for inum in range(self.Nv): if not values1[inum] is None: self.Nokvalues1 = self.Nokvalues1 + 1 if not values2[inum] is None: self.Nokvalues2 = self.Nokvalues2 + 1 if not values1[inum] is None and not values2[inum] is None: self.Nokvalues12 = self.Nokvalues12 + 1 Aval = values1[inum] + values2[inum] Sval = values1[inum] - values2[inum] Pval = values1[inum] * values2[inum] if np.isinf(values1[inum] / values2[inum]) or np.isnan(values1[inum] / values2[inum]): if self.NDvalNone < 1: print warnmsg print ' stats2Val: val1/val2 inf or Nan!!!!' Dval = None self.NDvalNone = self.NDvalNone + 1 else: Dval = values1[inum] / values2[inum] self.mae = self.mae + abs(Sval) self.rmse = self.rmse + Sval**2 if Aval < self.minv1Av2: self.minv1Av2 = Aval if Aval > self.maxv1Av2: self.maxv1Av2 = Aval if Sval < self.minv1Sv2: self.minv1Sv2 = Sval if Sval > self.maxv1Sv2: self.maxv1Sv2 = Sval if not Dval is None and Dval < self.minv1Dv2: self.minv1Dv2 = Dval if not Dval is None and Dval > self.maxv1Dv2: self.maxv1Dv2 = Dval if Pval < self.minv1Pv2: self.minv1Pv2 = Pval if Pval > self.maxv1Pv2: self.maxv1Pv2 = Pval self.meanv1Av2 = self.meanv1Av2+Aval self.mean2v1Av2 = self.mean2v1Av2+Aval*Aval self.meanv1Sv2 = self.meanv1Sv2+Sval self.mean2v1Sv2 = self.mean2v1Sv2+Sval*Sval if not Dval is None: self.meanv1Dv2 = self.meanv1Dv2+Dval self.mean2v1Dv2 = self.mean2v1Dv2+Dval*Dval self.meanv1Pv2 = self.meanv1Pv2+Pval self.mean2v1Pv2 = self.mean2v1Pv2+Pval*Pval ## print 'Nokvalues1: ', self.Nokvalues1, 'Nokvalues2: ', self.Nokvalues2, 'Nokvalues12: ', float(self.Nokvalues12), 'NDvalNone: ',self.NDvalNone self.meanv1Av2 = self.meanv1Av2/float(self.Nokvalues12) self.mean2v1Av2 = self.mean2v1Av2/float(self.Nokvalues12) self.stdv1Av2 = np.sqrt(self.mean2v1Av2-self.meanv1Av2*self.meanv1Av2) self.meanv1Sv2 = self.meanv1Sv2/float(self.Nokvalues12) self.mean2v1Sv2 = self.mean2v1Sv2/float(self.Nokvalues12) self.stdv1Sv2 = np.sqrt(self.mean2v1Sv2-self.meanv1Sv2*self.meanv1Sv2) if self.Nokvalues12 - self.NDvalNone == 0: self.meanv1Dv2 = None self.mean2v1Dv2 = None self.stdv1Dv2 = None print warnmsg print ' stats2Val: all values of val1/val2 are None!' else: self.meanv1Dv2 = self.meanv1Dv2/(float(self.Nokvalues12 - self.NDvalNone)) self.mean2v1Dv2 = self.mean2v1Dv2/(float(self.Nokvalues12 - self.NDvalNone)) self.stdv1Dv2 = np.sqrt(self.mean2v1Dv2-self.meanv1Dv2*self.meanv1Dv2) self.meanv1Pv2 = self.meanv1Pv2/float(self.Nokvalues12) self.mean2v1Pv2 = self.mean2v1Pv2/float(self.Nokvalues12) self.stdv1Pv2 = np.sqrt(self.mean2v1Pv2-self.meanv1Pv2*self.meanv1Pv2) self.mae = self.mae/self.Nokvalues12 self.rmse = np.sqrt(self.rmse/self.Nokvalues12) vals1Nomiss = np.ones(len(values1), dtype=bool) vals2Nomiss = np.ones(len(values1), dtype=bool) for i in range(len(values1)): if values1[i] is None: vals1Nomiss[i] = False for i in range(len(values2)): if values2[i] is None: vals2Nomiss[i] = False v1 = np.array(values1[vals1Nomiss], dtype=float) v2 = np.array(values2[vals2Nomiss], dtype=float) if not v1.shape == v2.shape: print errormsg print ' ' + fname + ': variables without missing values v1: ',v1.shape , ' and v2: ',v2.shape ,' do not have the same shape! ' print errormsg quit(-1) self.corr = sts.pearsonr(v1, v2) self.linRegress[0], self.linRegress[1], self.linRegress[2], self.linRegress[3], self.linRegress[4] = sts.linregress(v1, v2) polyfitvals=np.polyfit(v1, v2, power, full = True) self.polRegress = polyfitvals[0] self.polRegressRes = polyfitvals[1] self.polRegressSingVal = polyfitvals[3] def mask_2masked(vals1, vals2): """ Function to provide the boolean matrix (in opposite way as it is in the mask) as combination of mask from to masked matrices """ import numpy.ma as ma fname = 'mask_2masked' # if len(vals1.shape) != len(vals2.shape): # print errormsg # print ' ' + fname + ' matrix 1 :', len(vals1.shape), ' and matrix 2 ', len(vals2.shape), ' have different size!' # print errormsg # quit(-1) # for idim in range(len(vals1.shape)): # if vals1.shape[idim] != vals2.shape[idim]: # print errormsg # print ' ' + fname + ' dimension ', idim,' from matrix 1 :', vals1.shape[idim], ' and matrix 2 ', \ # vals2.shape[idim], ' have different size!' # print errormsg # quit(-1) if type(vals1) == type(ma.array(1)): mask1array=np.where(ma.getmaskarray(vals1) == False, True, False) else: mask1array=np.ones(vals1.shape, dtype=bool) if type(vals2) == type(ma.array(1)): mask2array=np.where(ma.getmaskarray(vals2) == False, True, False) else: mask2array=np.ones(vals2.shape, dtype=bool) mask12 = mask1array*mask2array return mask12 def mask_pearsonr(xvals, yvals): """ Function to compute a pearson correlation from mask matrices """ from scipy import stats as sts fillVal = 1.e20 maskxy = mask_2masked(xvals, yvals) if np.sum(maskxy) > 1: pearsonr = sts.pearsonr(xvals[maskxy], yvals[maskxy]) if np.isnan(pearsonr[0]) or np.isnan(pearsonr[1]): pearsonr = ( fillVal, fillVal) else: pearsonr = (fillVal, fillVal) return pearsonr def mask_quantiles(maskmat, Nquants): """ Function to provide the quantiles of a masked array 20 for %5 bins (21 in total) """ import numpy.ma as ma fillValue = 1.e20 sortmat = maskmat.flatten().copy() sortmat.sort() quants = np.zeros(Nquants+1, dtype=type(maskmat[0])) Nv = ma.size(maskmat) NoMask=maskmat.count() if NoMask < Nquants: quants[:] = fillValue else: for iq in range(Nquants): quants[iq] = sortmat[int((NoMask-1)*iq/(Nquants))] quants[Nquants] = sortmat[NoMask-1] return quants def percendone(nvals,tot,percen,msg): """ Function to provide the percentage of an action across the matrix nvals=number of values tot=total number of values percen=percentage frequency for which the message is wanted msg= message """ from sys import stdout num = int(tot * percen/100) if (nvals%num == 0): print '\r ' + msg + '{0:8.3g}'.format(nvals*100./tot) + ' %', stdout.flush() return '' def mask_linregres(vals1, vals2): """ Function to compute a linear regression from masked data vals1: x-values for the regresion vals2: y-values for the regresion """ import numpy.ma as ma fname = 'mask_linregres' missval1 = vals1.get_fill_value() missval2 = vals2.get_fill_value() vals10 = np.where(abs(vals1) >= abs(missval1*0.9), None, vals1) vals20 = np.where(abs(vals2) >= abs(missval2*0.9), None, vals2) values1 = vals10.flatten() values2 = vals20.flatten() vals1Nomiss = np.ones(len(values1), dtype=bool) vals2Nomiss = np.ones(len(values2), dtype=bool) for i in range(len(values1)): if values1[i] is None: vals1Nomiss[i] = False for i in range(len(values2)): if values2[i] is None: vals2Nomiss[i] = False v1 = np.array(values1[vals1Nomiss], dtype=float) v2 = np.array(values2[vals2Nomiss], dtype=float) if len(v1) != len(v2): print errormsg print fname + ': length of masked matrices mat1:',len(v1),'and mat2:',len(v2),'does not match!' print errormsg quit(-1) linregres = np.array(sts.linregress(v1, v2), dtype= np.float64) return linregres def mask_space_stats(maskmat,statsvals,dim): """ Function to give back the multi-dimensional statisitcs of a given masked array maskmat=multidimensional masked array statsvals=[statn]:[values] [statn]: statistics to do: 'quant', quantiles [values]: value for the statistics: Nquantiles dim= dimension to run the statistics """ from sys import stdout fname = 'mask_space_stats' statn=statsvals.split(':')[0] if len(statsvals.split(':')) > 1: values=statsvals.split(':')[1] maskshape = maskmat.shape Ndims = len(maskshape) if statn == 'quant': if len(statsvals.split(':')) == 1: print errormsg print fname + ': statistics "' + statn + '" requires a value!!!' print errormsg quit(-1) Nquants=int(values) if Ndims == 2: if dim == 0: statval = np.ones((21, maskshape[1]), dtype=np.float64)*fillValue for i in range(maskshape[1]): percendone(i, maskshape[1], 5, 'quantiles') statval[:,i] = mask_quantiles(maskmat[:,i],Nquants) if dim == 1: statval = np.ones((21, maskshape[0]), dtype=np.float64)*fillValue for i in range(maskshape[0]): percendone(i, maskshape[0], 5, 'quantiles') statval[:,i] = mask_quantiles(statval[i,:],Nquants) elif Ndims == 3: if dim == 0: statval = np.ones((21, maskshape[1], maskshape[2]), dtype=np.float64)*fillValue for i in range(maskshape[1]): for j in range(maskshape[2]): percendone(i*maskshape[2] + j, maskshape[1]*maskshape[2], 5, 'quantiles') statval[:,i,j] = mask_quantiles(maskmat[:,i,j],Nquants) if dim == 1: statval = np.ones((21, maskshape[0], maskshape[2]), dtype=np.float64)*fillValue for i in range(maskshape[0]): for j in range(maskshape[2]): percendone(i*maskshape[2] + j, maskshape[0]*maskshape[2], 5, 'quantiles') statval[:,i,j] = mask_quantiles(maskmat[i,:,j],Nquants) if dim == 2: statval = np.ones((21, maskshape[0], maskshape[1]), dtype=np.float64)*fillValue for i in range(maskshape[0]): for j in range(maskshape[1]): percendone(i*maskshape[1] + j, maskshape[0]*maskshape[1], 5, 'quantiles') statval[:,i,j] = mask_quantiles(maskmat[i,j,:],Nquants) elif Ndims == 4: if dim == 0: statval = np.ones((21, maskshape[1], maskshape[2], maskshape[3]), dtype=np.float64)*fillValue for i in range(maskshape[1]): for j in range(maskshape[2]): for k in range(maskshape[3]): percendone(i*maskshape[1]*maskshape[2] + j*maskshape[2] + k, maskshape[1]*maskshape[2]*maskshape[3], 5, 'quantiles') statval[:,i,j,k] = mask_quantiles(maskmat[:,i,j,k],Nquants) if dim == 1: statval = np.ones((21, maskshape[0], maskshape[2], maskshape[3]), dtype=np.float64)*fillValue for i in range(maskshape[0]): for j in range(maskshape[2]): for k in range(maskshape[3]): percendone(i*maskshape[0]*maskshape[2] + j*maskshape[2] + k, maskshape[0]*maskshape[2]*maskshape[3], 5, 'quantiles') statval[:,i,j,k] = mask_quantiles(maskmat[i,:,j,k],Nquants) if dim == 2: statval = np.ones((21, maskshape[0], maskshape[1], maskshape[3]), dtype=np.float64)*fillValue for i in range(maskshape[0]): for j in range(maskshape[1]): for k in range(maskshape[3]): percendone(i*maskshape[0]*maskshape[1] + j*maskshape[1] + k, maskshape[0]*maskshape[1]*maskshape[3], 5, 'quantiles') statval[:,i,j,k] = mask_quantiles(maskmat[i,j,:,k],Nquants) if dim == 3: statval = np.ones((21, maskshape[0], maskshape[1], maskshape[2]), dtype=np.float64)*fillValue for i in range(maskshape[0]): for j in range(maskshape[1]): for k in range(maskshape[2]): percendone(i*maskshape[0]*maskshape[1] + j*maskshape[1] + k, maskshape[0]*maskshape[1]*maskshape[2], 5, 'quantiles') statval[:,i,j,k] = mask_quantiles(maskmat[i,j,k,:],Nquants) else: print errormsg print fname + ': size of matrix ', Ndims,'not ready!!!' print errormsg quit(-1) else: print errormsg print fname + ': statistics "' + statn + '" not ready!!!!' print errormsg quit(-1) print stdout.write("\n") return statval class statsVal(object): """Statistics class providing vals = variable self.Nv = number of values self.minv = minimum value self.maxv = maximum value self.meanv = mean value self.mean2v = cuadratic mean value self.stdv = standard deviation value self.Nokvalues = number of correct values of variable self.quantilesv = quantiles (%5 bins) of the variable """ def __init__(self, vals): if vals is None: self.Nv = None self.minv = None self.maxv = None self.meanv = None self.mean2v = None self.stdv = None self.Nokvalues = None self.quantilesv = None else: values = vals.flatten() self.Nv=len(values) self.minv=10000000000. self.maxv=-100000000. self.meanv=0. self.mean2v=0. self.stdv=0. sortedvalues = sorted(values) self.Nokvalues = 0 for inum in range(self.Nv): if not values[inum] is None: self.Nokvalues = self.Nokvalues + 1 if values[inum] < self.minv: self.minv = values[inum] if values[inum] > self.maxv: self.maxv = values[inum] self.meanv = self.meanv+values[inum] self.mean2v = self.mean2v+values[inum]*values[inum] self.meanv = self.meanv/float(self.Nokvalues) self.mean2v = self.mean2v/float(self.Nokvalues) self.stdv = np.sqrt(self.mean2v-self.meanv*self.meanv) self.quantilesv = [] for iq in range(20): self.quantilesv.append(sortedvalues[int((self.Nv-1)*iq/20)]) self.quantilesv.append(sortedvalues[self.Nv-1]) self.medianv = self.quantilesv[10] class Quantiles(object): """ Class to provide quantiles from a given arrayof values """ def __init__(self, values, Nquants): import numpy.ma as ma if values is None: self.quantilesv = None else: self.quantilesv = [] vals0 = values.flatten() Nvalues = len(vals0) vals = ma.masked_equal(vals0, None) Nvals=len(vals.compressed()) sortedvals = sorted(vals.compressed()) for iq in range(Nquants): self.quantilesv.append(sortedvals[int((Nvals-1)*iq/Nquants)]) self.quantilesv.append(sortedvals[Nvals-1]) class statsVal_missVal(object): """Statistics class tacking into account a missing value providing vals = variable missval = missing value self.Nv = number of values self.minv = minimum value self.maxv = maximum value self.meanv = mean value self.mean2v = cuadratic mean value self.stdv = standard deviation value self.Nokvalues = number of correct values of variable self.quantilesv = quantiles (%5 bins) of the variable """ def __init__(self, vals, missVal): fname='statsVal_missVal' if vals is None: self.Nv = None self.minv = None self.maxv = None self.meanv = None self.mean2v = None self.stdv = None self.Nokvalues = None self.quantilesv = None else: Npt = 1 for idim in range(len(vals.shape)): Npt = Npt * vals.shape[idim] if np.sum(vals >= missVal) == Npt: print errormsg print ' ' + fname + ' all values get missing!!' print errormsg quit(-1) vals1 = np.where(abs(vals) >= missVal, None, vals) vals0 = np.where(np.isnan(vals1), None, vals1) values = vals0.flatten() self.Nv=Npt self.minv=10000000000. self.maxv=-100000000. self.meanv=0. self.mean2v=0. self.stdv=0. sortedvalues = sorted(values) self.Nokvalues = 0 for inum in range(self.Nv): if not values[inum] is None: self.Nokvalues = self.Nokvalues + 1 if values[inum] < self.minv: self.minv = values[inum] if values[inum] > self.maxv: self.maxv = values[inum] self.meanv = self.meanv+values[inum] self.mean2v = self.mean2v+values[inum]*values[inum] self.meanv = self.meanv/float(self.Nokvalues) self.mean2v = self.mean2v/float(self.Nokvalues) self.stdv = np.sqrt(self.mean2v-self.meanv*self.meanv) self.quantilesv = [] for iq in range(20): self.quantilesv.append(sortedvalues[int((self.Nv-1)*iq/20)]) self.quantilesv.append(sortedvalues[self.Nv-1]) self.medianv = self.quantilesv[10] def spacemean(ncfile, varn): """ Function to retrieve a space mean series from a multidimensional variable of a file ncfile = netCDF file name varn = variable name """ import datetime as dt import calendar as cal ofile = 'spacemean_' + varn + '.nc' varfil=1.e20 statsn = ['min', 'max', 'mean', 'mean2', 'stdv', 'meanwgt', 'mean2wgt', 'stdvwgt', 'quant'] statslongn = ['minimum', 'maximum', 'mean', 'quadratic mean', 'standard deviation', 'weigthed mean', 'weigthed quadratic mean', 'weigthed standard deviation', 'quantiles'] ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' spacemean: File "' + ncfile + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) times = ncf.variables['time'] timevals = times[:] attvar = times.ncattrs() if not searchInlist(attvar, 'units'): print errormsg print ' spacemean: "time" does not have attribute: "units"' ncf.close() quit(-1) else: units = times.getncattr('units') txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: print ' spacemean: refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) dimt = len(timevals) realdates = np.zeros((dimt, 6), dtype=int) print realdates.shape ## Not in timedelta # if tunits == 'years': # for it in range(dimt): # realdate = refdate + dt.timedelta(years=float(times[it])) # realdates[it] = int(realdate.year) # elif tunits == 'months': # for it in range(dimt): # realdate = refdate + dt.timedelta(months=float(times[it])) # realdates[it] = int(realdate.year) if tunits == 'weeks': for it in range(dimt): realdate = refdate + dt.timedelta(weeks=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'days': for it in range(dimt): realdate = refdate + dt.timedelta(days=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'hours': for it in range(dimt): realdate = refdate + dt.timedelta(hours=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'minutes': for it in range(dimt): realdate = refdate + dt.timedelta(minutes=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'seconds': for it in range(dimt): realdate = refdate + dt.timedelta(seconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'milliseconds': for it in range(dimt): realdate = refdate + dt.timedelta(milliseconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) else: print errormsg print ' spacemean: time units "' + tunits + '" not ready!!!!' ncf.close() quit(-1) # Variable values (assuming time as first dimension) ## var = ncf.variables[varn] varinf = variable_inf(var) if searchInlist(varinf.attributes, 'scale_factor'): scalefact = var.getncattr('scale_factor') print ' spacemean: data has a scale factor of ', scalefact varinf.dtype=type(np.float(1.)) else: scalefact = 1. if searchInlist(varinf.attributes, 'add_offset'): offset = var.getncattr('add_offset') print ' spacemean: data has an offset of ', offset else: offset = 0. # varvals = var[:] vardims = var.shape varshape = len(vardims) vardimns = var.dimensions ## print ' spacemean: Shape of data: ',varshape, ':' , vardims dimt=vardims[0] vardnames = [] # Spatial average spatial weigthed ## lonvar = ncf.variables['lon'] latvar = ncf.variables['lat'] if not len(lonvar.shape) == 2: lonv = lonvar[:] latv = latvar[:] dx=lonvar.shape[0] dy=latvar.shape[0] lonval = np.zeros((dy, dx), dtype=np.float64) latval = np.zeros((dy, dx), dtype=np.float64) for iy in range(dy): lonval[iy,:] = lonv for ix in range(dx): latval[:,ix] = latv else: lonval = lonvar[:] latval = latvar[:] weightsv = abs(np.cos(latval*np.pi/180.)) if varinf.Ndims == 1: print errormsg print ' spacemean: You can not compute a space mean for a ', varinf.Ndims, 'D var!!!' ncf.close() quit(-1) elif varinf.Ndims== 2: print errormsg print ' spacemean: You can not compute a space mean for a ', varinf.Ndims, 'D var!!!' ncf.close() quit(-1) elif varinf.Ndims == 3: varstats = np.ones((dimt, 8), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((dimt, 21), dtype=np.float64) varquant = varquant*varfil vardnames.append(vardimns[0]) dy=vardims[1] dx=vardims[2] for it in range(dimt): if not varinf.FillValue is None: varvl = var[it,:,:] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[it,:,:] percendone(it,dimt,5,'computed space statistics') varst = statsVal(varval) varstwgt = statsValWeigthed(varval, weightsv) varstats[it,0] = varst.minv varstats[it,1] = varst.maxv varstats[it,2] = varst.meanv varstats[it,3] = varst.mean2v varstats[it,4] = varst.stdv varquant[it,:] = varst.quantilesv varstats[it,5] = varstwgt.meanv varstats[it,6] = varstwgt.mean2v varstats[it,7] = varstwgt.stdv elif varshape == 4: varstats = np.ones((dimt, vardims[1], 8), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((dimt, vardims[1], 21), dtype=np.float64) varquant = varquant*varfil vardimnames = (str(vardimns[0]), str(vardimns[1])) vardnames.append(vardimns[0]) vardnames.append(vardimns[1]) dy=vardims[2] dx=vardims[3] for it in range(dimt): for ik in range(vardims[1]): if not varinf.FillValue is None: varvl = var[it,ik,:,:] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[it,ik,:,:] percendone(it*vardims[1]+ik,dimt*vardims[1],5,'computed space statistics') varst = statsVal(varval) varstwgt = statsValWeigthed(varval, weightsv) varstats[it,ik,0] = varst.minv varstats[it,ik,1] = varst.maxv varstats[it,ik,2] = varst.meanv varstats[it,ik,3] = varst.mean2v varstats[it,ik,4] = varst.stdv varquant[it,ik,:] = varst.quantilesv varstats[it,ik,5] = varstwgt.meanv varstats[it,ik,6] = varstwgt.mean2v varstats[it,ik,7] = varstwgt.stdv elif varshape == 5: varstats = np.ones((dimt, vardims[1], vardims[2], 8), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((dimt, vardims[1], vardims[2], 21), dtype=np.float64) varquant = varquant*varfil vardnames.append(vardimns[0]) vardnames.append(vardimns[1]) vardnames.append(vardimns[2]) dy=vardims[3] dx=vardims[4] for it in range(dimt): for ik in range(vardims[1]): for il in range(vardims[2]): if not varinf.FillValue is None: varvl = var[it,ik,il,:,:] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[it,ik,il,:,:] percendone(it*vardims[1]*vardims[2]+ik*vardims[1]+il,dimt*vardims[1]*vardims[2],5,'computed space statistics') varst = statsVal(varval) varstwgt = statsValWeigthed(varval, weightsv) varstats[it,ik,il,0] = varst.minv varstats[it,ik,il,1] = varst.maxv varstats[it,ik,il,2] = varst.meanv varstats[it,ik,il,3] = varst.mean2v varstats[it,ik,il,4] = varst.stdv varquant[it,ik,il,:] = varst.quantilesv varstats[it,ik,il,5] = varstwgt.meanv varstats[it,ik,il,6] = varstwgt.mean2v varstats[it,ik,il,7] = varstwgt.stdv else: print errormsg print ' spacemean: ', varshape, ' shape of matrix not prepared !' ncf.close() quit(-1) vardimnames = tuple(vardnames) # print ' shape of desired values: ', vardesiredvalues.shape # Creation of file ## ncfo = NetCDFFile( ofile, 'w') vartype = var.dtype varattr = var.ncattrs() # Checking dimensions ## newdims = ncfo.dimensions Nvardnames = len(vardimnames) for idim in range(Nvardnames): rdim = vardimnames[idim] if not searchInlist(newdims, rdim): if not rdim == 'time': print ' spacemean: Adding dimension ' + rdim ncfo.sync() ncf.close() ncfo.close() fdimadd(ncfile + ',' + rdim, ofile) ncf = NetCDFFile(ncfile,'r') ncfo = NetCDFFile(ofile,'a') else: ncfo.createDimension('time', None) # Checking fill value ## if searchInlist(varattr, '_FillValue'): varfil = var._FillValue else: varfil = False Nstats = len(statsn) for ist in range(Nstats): if statsn[ist] == 'quant': print ist, statsn[ist]##, ': ', varquant[int(dimt/2),10] newdim = ncfo.createDimension('quant', 21) newvardnames = list(vardnames) newvardnames.append('quant') newvar = ncfo.createVariable(varn + statsn[ist], varinf.dtype, tuple(newvardnames), fill_value=varfil) newvar[:] = varquant else: print ist, statsn[ist]##, ': ', varstats[int(dimt/2),ist] newvar = ncfo.createVariable(varn + statsn[ist], varinf.dtype, vardimnames, fill_value=varfil) if varshape == 3: newvar[:] = varstats[:,ist]*1. elif varshape == 4: newvar[:] = varstats[:,:,ist]*1. elif varshape == 5: newvar[:] = varstats[:,:,:,ist]*1. newvar = ncfo.variables[varn + statsn[ist]] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = var.getncattr(attr) if not searchInlist(newvarattrs, attr): if attr != 'scale_factor' and attr != 'add_offset' and attr != 'valid_range' \ and attr != 'unpacked_valid_range' and attr != 'actual_range' : # if not statsn[ist] == 'stdv' and not statsn[ist] == 'stdvwgt': print 'attr:', attr newvar.setncattr(attr, attrv) # else: # newvar.setncattr(attr, attrv) newvar.setncattr('cell_methods', 'space ' + statslongn[ist] + ' all domain ' + str(dy) + 'x' + str(dx)) ## print ' Adding time variable' vartdims = times.dimensions vartype = times.dtype varattr = times.ncattrs() newvar = ncfo.createVariable('time', vartype, vartdims, fill_value=varfil) newvar = ncfo.variables['time'] newvar[:] = times ncf.close() ncfo.sync() ncfo.close() fattradd('time', ncfile + ':time', ofile) fvaradd(ncfile + ':lon', ofile) fvaradd(ncfile + ':lat', ofile) ncfo = NetCDFFile(ofile,'a') newvar = ncfo.variables['time'] newvarattrs = newvar.ncattrs() if searchInlist(newvarattrs, 'bounds'): if newvar.getncattr('bounds') == 'time_bnds': ncf = NetCDFFile(ncfile,'r') tbnds = ncf.variables['time_bnds'] vardims = tbnds.dimensions vartype = tbnds.dtype varattr = tbnds.ncattrs() ncfo.createDimension('bnds', 2) newvar = ncfo.createVariable('time_bnds', vartype, vardims, fill_value=varfil) newvar[:] = tbnds[:] ncf.close() ncfo.sync() ncfo.close() fattradd('time_bnds', ncfile + ':time_bnds', ofile) else: ncfo.close() else: ncfo.close() fgaddattr(ncfile, ofile) print ' spacemean: File "' + ofile + '" as space mean of "' + varn + '" has been created' def timemean(values, ncfile, varn): """ Function to retrieve a time mean series from a multidimensional variable of a file values = power of the polynomial fitting with time to be applied ncfile = netCDF file name varn = variable name """ import datetime as dt import calendar as cal powerv=int(values) ofile = 'timemean_' + varn + '.nc' varfil=1.e20 statsn = ['min', 'max', 'mean', 'mean2', 'stdv', 'quant','linregress','polregress'] statslongn = ['minimum', 'maximum', 'mean', 'quadratic mean', 'standard deviation', 'quantiles', \ 'linear regression', 'polynomial regression'] ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' timemean: File "' + ncfile + '" does not have variable ' + varn + ' !!!!' print errormsg ncf.close() quit(-1) times = ncf.variables['time'] timevals = times[:] attvar = times.ncattrs() if not searchInlist(attvar, 'units'): print errormsg print ' timemean: "time" does not have attribute: "units"' ncf.close() quit(-1) else: units = times.getncattr('units') txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: print ' timemean: refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) dimt = len(timevals) realdates = np.zeros((dimt, 6), dtype=int) print realdates.shape ## Not in timedelta # if tunits == 'years': # for it in range(dimt): # realdate = refdate + dt.timedelta(years=float(times[it])) # realdates[it] = int(realdate.year) # elif tunits == 'months': # for it in range(dimt): # realdate = refdate + dt.timedelta(months=float(times[it])) # realdates[it] = int(realdate.year) if tunits == 'weeks': for it in range(dimt): realdate = refdate + dt.timedelta(weeks=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'days': for it in range(dimt): realdate = refdate + dt.timedelta(days=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'hours': for it in range(dimt): realdate = refdate + dt.timedelta(hours=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'minutes': for it in range(dimt): realdate = refdate + dt.timedelta(minutes=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'seconds': for it in range(dimt): realdate = refdate + dt.timedelta(seconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'milliseconds': for it in range(dimt): realdate = refdate + dt.timedelta(milliseconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) else: print errormsg print ' timemean: time units "' + tunits + '" not ready!!!!' ncf.close() quit(-1) timesv = (realdates[:,0] - realdates[0,0])*12 + realdates[:,1] # Variable values (assuming time as first dimension) ## var = ncf.variables[varn] varinf = variable_inf(var) # varvals = var[:] vardims = varinf.dims varshape = varinf.Ndims vardimns = var.dimensions ## print ' timemean: Shape of data: ',varshape, ':' , vardims dimt=vardims[0] vardnames = [] if varshape == 1: print errormsg print ' timemean: You can not compute a time mean for a ', varshape, 'D var!!!' ncf.close() quit(-1) elif varshape == 2: dx=vardims[1] varstats = np.ones((5, dx), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((21, dx), dtype=np.float64) varquant = varquant*varfil varlregress = np.ones((5, dx), dtype=np.float64) varlregress = varlregress*varfil varpregress = np.ones((powerv+1, dx), dtype=np.float64) varpregress = varpregress*varfil varpregressres = np.ones((dx), dtype=np.float64) varpregressres = varpregressres*varfil varpregresssingval = varpregress.copy() varpregresssingval = varpregresssingval*varfil vardnames.append(vardimns[1]) for ix in range(dx): if not varinf.FillValue is None: varvl = var[:,ix] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[:,ix] percendone(ix,dx,5,'computed time statistics') varst = statsVal(varval) var2st = stats2Val(timesv, varval, powerv) varstats[0,ix] = varst.minv varstats[1,ix] = varst.maxv varstats[2,ix] = varst.meanv varstats[3,ix] = varst.mean2v varstats[4,ix] = varst.stdv varquant[:,ix] = varst.quantilesv varlregress[:,ix] = var2st.linRegress varpregress[:,ix] = var2st.polRegress varpregressres[ix] = var2st.polRegressRes varpregresssingval[:,ix] = var2st.polRegressSingVal elif varshape == 3: dy=vardims[1] dx=vardims[2] varstats = np.ones((5,dy, dx), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((21,dy, dx), dtype=np.float64) varquant = varquant*varfil varlregress = np.ones((5,dy, dx), dtype=np.float64) varlregress = varlregress*varfil varpregress = np.ones((powerv+1,dy, dx), dtype=np.float64) varpregress = varpregress*varfil varpregressres = np.ones((dy,dx), dtype=np.float64) varpregressres = varpregressres*varfil varpregresssingval = varpregress.copy() varpregresssingval = varpregresssingval*varfil vardnames.append(vardimns[1]) vardnames.append(vardimns[2]) for iy in range(dy): for ix in range(dx): if not varinf.FillValue is None: varvl = var[:,iy,ix] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[:,iy,ix] percendone(iy*dx+ix,dy*dx,5,'computed time statistics') varst = statsVal(varval) var2st = stats2Val(timesv, varval, powerv) varstats[0,iy,ix] = varst.minv varstats[1,iy,ix] = varst.maxv varstats[2,iy,ix] = varst.meanv varstats[3,iy,ix] = varst.mean2v varstats[4,iy,ix] = varst.stdv varquant[:,iy,ix] = varst.quantilesv varlregress[:,iy,ix] = var2st.linRegress varpregress[:,iy,ix] = var2st.polRegress varpregressres[iy,ix] = var2st.polRegressRes varpregresssingval[:,iy,ix] = var2st.polRegressSingVal elif varshape == 4: dz=vardims[1] dy=vardims[2] dx=vardims[3] varstats = np.ones((5, dz, dy, dx), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((21, dz, dy, dx), dtype=np.float64) varquant = varquant*varfil varlregress = np.ones((5, dz, dy, dx), dtype=np.float64) varlregress = varlregress*varfil varpregress = np.ones((powerv+1, dz, dy, dx), dtype=np.float64) varpregress = varpregress*varfil varpregressres = np.ones((dz,dy,dx), dtype=np.float64) varpregressres = varpregressres*varfil varpregresssingval = varpregress.copy() varpregresssingval = varpregresssingval*varfil vardnames.append(vardimns[1]) vardnames.append(vardimns[2]) vardnames.append(vardimns[3]) for iz in range(dz): for iy in range(dy): for ix in range(dx): if not varinf.FillValue is None: varvl = var[:,iz,iy,ix] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[:,iz,iy,ix] percendone(iz*dy*dx+iy*dx+ix,dz*dx*dy,5,'computed time statistics') varst = statsVal(varval) var2st = stats2Val(timesv, varval, powerv) varstats[0,iz,iy,ix] = varst.minv varstats[1,iz,iy,ix] = varst.maxv varstats[2,iz,iy,ix] = varst.meanv varstats[3,iz,iy,ix] = varst.mean2v varstats[4,iz,iy,ix] = varst.stdv varquant[:,iz,iy,ix] = varst.quantilesv varlregress[:,iz,iy,ix] = var2st.linRegress varpregress[:,iz,iy,ix] = var2st.polRegress varpregressres[iz,iy,ix] = var2st.polRegressRes varpregresssingval[:,iz,iy,ix] = var2st.polRegressSingVal elif varshape == 5: dn=vardims[1] dz=vardims[2] dy=vardims[3] dx=vardims[4] varstats = np.ones((5, dn, dz, dy, dx), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((21, dn, dz, dy, dx), dtype=np.float64) varquant = varquant*varfil varlregress = np.ones((5, dn, dz, dy, dx), dtype=np.float64) varlregress = varlregress*varfil varpregress = np.ones((powerv+1, dn, dz, dy, dx), dtype=np.float64) varpregress = varpregress*varfil varpregressres = np.ones((dn,dz,dy,dx), dtype=np.float64) varpregressres = varpregressres*varfil varpregresssingval = varpregress.copy() varpregresssingval = varpregresssingval*varfil vardnames.append(vardimns[1]) vardnames.append(vardimns[2]) vardnames.append(vardimns[3]) vardnames.append(vardimns[4]) for iN in range(dn): for iz in range(dz): for iy in range(dy): for ix in range(dx): if not varinf.FillValue is None: varvl = var[:,iN,iz,iy,ix] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[:,iN,iz,iy,ix] percendone(iN*dy*dx*dz+iz*dy*dx+iy*dx+ix,dn*dz*dx*dy,5,'computed time statistics') varst = statsVal(varval) var2st = stats2Val(timesv, varval, powerv) varstats[0,iN,iz,iy,ix] = varst.minv varstats[1,iN,iz,iy,ix] = varst.maxv varstats[2,iN,iz,iy,ix] = varst.meanv varstats[3,iN,iz,iy,ix] = varst.mean2v varstats[4,iN,iz,iy,ix] = varst.stdv varquant[:,iN,iz,iy,iiN,x] = varst.quantilesv varlregress[:,iN,iz,iy,ix] = var2st.linRegress varpregress[:,iN,iz,iy,ix] = var2st.polRegress varpregressres[iN,iz,iy,ix] = var2st.polRegressRes varpregresssingval[:,iN,iz,iy,ix] = var2st.polRegressSingVal else: print errormsg print ' timemean: ', varshape, ' shape of matrix not prepared !' ncf.close() quit(-1) vardimnames = tuple(vardnames) # print ' shape of desired values: ', vardesiredvalues.shape # Creation of file ## ncfo = NetCDFFile( ofile, 'w') vartype = var.dtype varattr = var.ncattrs() # Checking dimensions ## newdims = ncfo.dimensions Nvardnames = len(vardimnames) for idim in range(Nvardnames): rdim = vardimnames[idim] if not searchInlist(newdims, rdim): if not rdim == 'time': ## print ' timemean: Adding dimension ' + rdim ncfo.sync() ncf.close() ncfo.close() fdimadd(ncfile + ',' + rdim, ofile) ncf = NetCDFFile(ncfile,'r') ncfo = NetCDFFile(ofile,'a') else: print ' timemean: No time dimension!' # Checking fill value ## if searchInlist(varattr, '_FillValue'): varfil = var._FillValue else: varfil = False Nstats = len(statsn) for ist in range(Nstats): newvardnames = [] if statsn[ist] == 'quant': print ist, statsn[ist]##, ': ', varquant[int(dimt/2),10] newdim = ncfo.createDimension('quant', 21) newvardnames.append('quant') newvardnames = newvardnames + list(vardnames) newvar = ncfo.createVariable(varn + statsn[ist], vartype, tuple(newvardnames), fill_value=varfil) newvar[:] = varquant elif statsn[ist] == 'linregress': print ist, statsn[ist]##, ': ', varquant[int(dimt/2),10] newdim = ncfo.createDimension('lregress', 5) newvar = ncfo.createVariable('lregressn', str, ('lregress',)) newvar[0] = 'slope' newvar[1] = 'intercept' newvar[2] = 'r_value' newvar[3] = 'p_value' newvar[4] = 'std_err' newvardnames.append('lregress') newvardnames = newvardnames + list(vardnames) newvar = ncfo.createVariable(varn + statsn[ist], 'f4', tuple(newvardnames), fill_value=varfil) newvar[:] = varlregress*1. elif statsn[ist] == 'polregress': print ist, statsn[ist]##, ': ', varquant[int(dimt/2),10] newdim = ncfo.createDimension('pregress', powerv+1) newvar = ncfo.createVariable('pregressn', str, ('pregress',)) for ip in range(powerv+1): newvar[ip]='coefficient**' + str(powerv-ip) newvardnames.append('pregress') newvardnames = newvardnames + list(vardnames) newvar = ncfo.createVariable(varn + statsn[ist], 'f4', tuple(newvardnames), fill_value=varfil) newvar[:] = varpregress*1. newvar.setncattr('power',powerv) newvar.setncattr('values','Polynomial coefficients, highest power first') newvar = ncfo.createVariable(varn + statsn[ist] + '_Residual', vartype, tuple(vardnames), fill_value=varfil) newvar[:] = varpregressres newvar.setncattr('power',powerv) newvar.setncattr('values','Polynomial residuals') newvar = ncfo.createVariable(varn + statsn[ist] + '_VandermondeSingularVector', vartype, tuple(newvardnames), fill_value=varfil) newvar[:] = varpregresssingval newvar.setncattr('power',powerv) newvar.setncattr('values','Polynomial coefficients, highest power first') else: print ist, statsn[ist]##, ': ', varstats[int(dimt/2),ist] if statsn[ist] == 'mean' or statsn[ist] == 'stdv' or statsn[ist] == 'mean2' or statsn[ist] == 'polregress_Residual' \ or statsn[ist] == 'polregress_VandermondeSingularVector' and searchInlist(varattr, 'scale_factor'): newvar = ncfo.createVariable(varn + statsn[ist], 'f4', vardimnames, fill_value=varfil) if varshape == 2: newvar[:] = varstats[ist,:]*1. elif varshape == 3: newvar[:] = varstats[ist,:,:]*1. elif varshape == 4: newvar[:] = varstats[ist,:,:,:]*1. elif varshape == 5: newvar[:] = varstats[ist,:,:,:,:]*1. else: newvar = ncfo.createVariable(varn + statsn[ist], vartype, vardimnames, fill_value=varfil) if varshape == 2: newvar[:] = varstats[ist,:] elif varshape == 3: newvar[:] = varstats[ist,:,:] elif varshape == 4: newvar[:] = varstats[ist,:,:,:] elif varshape == 5: newvar[:] = varstats[ist,:,:,:,:] newvar = ncfo.variables[varn + statsn[ist]] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = var.getncattr(attr) if not searchInlist(newvarattrs, attr): if attr == 'scale_factor' or attr == 'add_offset' or attr == 'valid_range' \ or attr == 'unpacked_valid_range' or attr == 'actual_range' : if not statsn[ist] == 'mean' and not statsn[ist] == 'stdv' and not statsn[ist] == 'mean2' and not \ statsn[ist] == 'linregress' and not statsn[ist] == 'polregress' \ and not statsn[ist] == 'polregress_Residual' and not \ statsn[ist] == 'polregress_VandermondeSingularVector': newvar.setncattr(attr, attrv) else: newvar.setncattr(attr, attrv) newvar.setncattr('cell_methods', 'time ' + statslongn[ist] + ' all period in file ' + str(dimt) + ' time steps') ncfo.sync() ncfo.close() fvaradd(ncfile + ':lon', ofile) fvaradd(ncfile + ':lat', ofile) fgaddattr(ncfile, ofile) print ' timemean: File "' + ofile + '" as time mean of "' + varn + '" has been created' def printing_class(classobj): """ Function to print all the values of a given class """ valscls = vars(classobj) for attrcls in valscls: print attrcls, ':', valscls[attrcls] def fmtprinting_class(classobj): """ Function to print all the values of a given class """ valscls = vars(classobj) for attrcls in valscls: print '@' + attrcls + '@', ':', valscls[attrcls] def flipdim(values, filename, varn): """ flips the value following one dimension [Nflipdim] values=[Nflipdim]:[flipdim] [Nflipdim]: number of the dimension to flip [flipdim]: has also the corresondant variable dimensinon be flipped? (yes/no) filename= netCDF file name varn= variable name """ ncf = NetCDFFile(filename,'a') Nflipdim=int(values.split(':')[0]) flipdim=values.split(':')[1] if not ncf.variables.has_key(varn): print errormsg print ' flipdim: File "' + filename + '" does not have variable ' + varn + ' !!!!' print errormsg ncf.close() quit(-1) varnc = ncf.variables[varn] varinf = variable_inf(varnc) if varinf.Ndims < Nflipdim: print errormsg print ' flipdim: variable "' + varn + '" has less dimensions ', varinf.Ndims, 'than the required to flip "', Nflipdim, '!!!!' print errormsg quit(-1) if flipdim == 'yes': flipdimname = varinf.dimns[Nflipdim] print ' flipdim: Flipping also associated dimension variable "' + flipdimname + '"' flipdim = ncf.variables[flipdimname] if len(flipdim.shape) == 1: newdim = flipdim[:] flipdim[:] = newdim[::-1] elif len(flipdim.shape) == 2: if Nflipdim == varinf.Ndims: for i in range(flipdim.shape[0]): newdim = flipdim[i,:] flipdim[i,:] = newdim[::-1] if Nflipdim == varinf.Ndims - 1: for i in range(flipdim.shape[1]): newdim = flipdim[:,i] flipdim[:,:] = newdim[::-1] else: print errormsg print ' flipdim: dimension to flip has ', len(flipdim.shape), ' dimensions, not ready to be flipped !!!' print errormsg quit(-1) newvar = np.ones(varinf.dims, dtype=varinf.dtype) if varinf.Ndims == 1: newvar = varnc[:] varnc[:] = newvar[::-1] elif varinf.Ndims == 2: if Nflipdim == 1: for i in range(varinf.dims[0]): newvar[i,:] = varnc[i,:] varnc[i,:] = newvar[i,::-1] elif Nflipdim == 0: for i in range(varinf.dims[1]): newvar[:,i] = varnc[:,i] varnc[:,i] = newvar[::-1,i] elif varinf.Ndims == 3: if Nflipdim == 2: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): newvar[i,j,:] = varnc[i,j,:] varnc[i,j,:] = newvar[i,j,::-1] elif Nflipdim == 1: for i in range(varinf.dims[0]): for j in range(varinf.dims[2]): newvar[i,:,j] = varnc[i,:,j] varnc[i,:,j] = newvar[i,::-1,j] elif Nflipdim == 0: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): newvar[:,i,j] = varnc[:,i,j] varnc[:,i,j] = newvar[::-1,i,j] elif varinf.Ndims == 4: if Nflipdim == 3: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[2]): newvar[i,j,k,:] = varnc[i,j,k,:] varnc[i,j,k,:] = newvar[i,j,k,::-1] if Nflipdim == 2: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[3]): newvar[i,j,:,k] = varnc[i,j,:,k] varnc[i,j,:,k] = newvar[i,j,::-1,k] if Nflipdim == 1: for i in range(varinf.dims[0]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): newvar[i,:,j,k] = varnc[i,:,j,k] varnc[i,:,j,k] = newvar[i,::-1,j,k] if Nflipdim == 0: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): newvar[:,i,j,k] = varnc[:,i,j,k] varnc[:,i,j,k] = newvar[::-1,i,j,k] elif varinf.Ndims == 5: if Nflipdim == 4: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[2]): for l in range(varinf.dims[3]): newvar[i,j,k,l,:] = varnc[i,j,k,l,:] varnc[i,j,k,l,:] = newvar[i,j,k,l,::-1] if Nflipdim == 3: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[2]): for l in range(varinf.dims[4]): newvar[i,j,k,:,l] = varnc[i,j,k,:,l] var[i,j,k,:,l] = newvar[i,j,k,::-1,l] if Nflipdim == 2: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): newvar[i,j,:,k,l] = varnc[i,j,:,k,l] varnc[i,j,:,k,l] = newvar[i,j,::-1,k,l] if Nflipdim == 1: for i in range(varinf.dims[0]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): newvar[i,:,j,k,l] = varnc[i,:,j,k,l] varnc[i,:,j,k,l] = newvar[i,::-1,j,k,l] if Nflipdim == 0: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): newvar[:,i,j,k,l] = varnc[:,i,j,k,l] varnc[:,i,j,k,l] = newvar[::-1,i,j,k,l] else: print errormsg print ' flipdim: Number of dimensions ', varinf.Ndims, ' not ready!!!!' print errormsg quit(-1) ncf.sync() ncf.close() def cycl_incr(cyclval,cycle,ninc): """ Function to increment a cyclic value [cyclval] with a cycle [cycle] a given number [ninc] of times >>> cycl_incr(1,4,1) 2 >>> cycl_incr(3,4,1) 0 >>> cycl_incr(1,4,10) 3 """ if ninc >= cycle: print 'cycl_incr: WARNING -- warning -- WARNING -- warning' print ' given increment: ', ninc,' is larger than the cycle !!' ninc = ninc - cycle*int(ninc/cycle) print ' reducing it to: ', ninc val=cyclval + ninc if val >= cycle: val=cyclval + ninc - cycle return val def times_4seasons(tvals, integrity): """ Function to split a time series in matricial date format ([:,year,mon,day,hour,minute,second]) in the four stations DJF,MAM,JJA,SON tvals= matrix withe times as [:,year,mon,day,hour,minute,second] integrity= only give values for entire seasons [True, 3 months for the season], or just split the values by seasons [False] """ fillVal=1.e20 # print tvals dt=tvals.shape[0] seasons=np.ones((dt,4),dtype=bool) seasons=seasons*False monseas=[12,3,6,9] firstseas=False for it in range(dt): if not firstseas: if integrity: for iseas in range(4): if tvals[it,1] == monseas[iseas]: nseas=iseas seasons[it,nseas]=True firstseas=True begseas=it else: for iseas in range(4): for imon in range(3): if tvals[it,1] == cycl_incr(monseas[iseas],12,imon): nseas=iseas seasons[it,nseas]=True firstseas=True else: newseas=cycl_incr(nseas,4,1) if tvals[it,1] == monseas[newseas]: seasons[it,newseas] = True nseas=newseas begseas=it else: seasons[it,nseas] = True endseas = it ## print 'Last season: ',nseas,' beginnig: ',begseas,' ending: ',endseas # Checking integrity of last season (has to have 3 months) ## if integrity: fullseas=True Nmon=np.unique(tvals[begseas:endseas+1,1]) for it in range(begseas,endseas): fullseas=fullseas*seasons[it,nseas] if len(Nmon) < 3 or not fullseas: seasons[begseas:endseas+1,nseas] = False return seasons def realdatetime_CFcompilant(times, Srefdate, tunits): """ Function to transform a matrix with real time values ([year, month, day, hour, minute, second]) to a netCDF one times= matrix with times Srefdate= reference date ([YYYY][MM][DD][HH][MI][SS] format) tunits= units of time respect to Srefdate >>> realdatetime_CFcompilant(np.array([ [1976, 2, 17, 8, 20, 0], [1976, 2, 18, 8, 20, 0]], dtype=int), '19491201000000', 'hours') [ 229784.33333333 229808.33333333] """ import datetime as dt yrref=int(Srefdate[0:4]) monref=int(Srefdate[4:6]) dayref=int(Srefdate[6:8]) horref=int(Srefdate[8:10]) minref=int(Srefdate[10:12]) secref=int(Srefdate[12:14]) refdate=dt.datetime(yrref, monref, dayref, horref, minref, secref) dimt=times.shape[0] cfdates = np.zeros((dimt), dtype=np.float64) if tunits == 'weeks': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = (cfdate.days + cfdate.seconds/(3600.*24.))/7. elif tunits == 'days': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days + cfdate.seconds/(3600.*24.) elif tunits == 'hours': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*24. + cfdate.seconds/3600. elif tunits == 'minutes': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*24.*60. + cfdate.seconds/60. elif tunits == 'seconds': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*24.*3600. + cfdate.seconds elif tunits == 'milliseconds': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*1000.*24.*3600. + cfdate.seconds*1000. elif tunits == 'microseconds': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*1000000.*24.*3600. + cfdate.seconds*1000000. else: print errormsg print ' ' + fname + ': time units "' + tunits + '" is not ready!!!' quit(-1) return cfdates def realdatetime1_CFcompilant(time, Srefdate, tunits): """ Function to transform a matrix with a real time value ([year, month, day, hour, minute, second]) to a netCDF one time= matrix with time Srefdate= reference date ([YYYY][MM][DD][HH][MI][SS] format) tunits= units of time respect to Srefdate >>> realdatetime1_CFcompilant([1976, 2, 17, 8, 20, 0], '19491201000000', 'hours') 229784.33333333 """ import datetime as dt yrref=int(Srefdate[0:4]) monref=int(Srefdate[4:6]) dayref=int(Srefdate[6:8]) horref=int(Srefdate[8:10]) minref=int(Srefdate[10:12]) secref=int(Srefdate[12:14]) refdate=dt.datetime(yrref, monref, dayref, horref, minref, secref) if tunits == 'weeks': cfdate = dt.datetime(time[0],time[1],time[2],time[3],time[4],time[5])-refdate cfdates = (cfdate.days + cfdate.seconds/(3600.*24.))/7. elif tunits == 'days': cfdate = dt.datetime(time[0],time[1],time[2],time[3],time[4],time[5]) - refdate cfdates = cfdate.days + cfdate.seconds/(3600.*24.) elif tunits == 'hours': cfdate = dt.datetime(time[0],time[1],time[2],time[3],time[4],time[5]) - refdate cfdates = cfdate.days*24. + cfdate.seconds/3600. elif tunits == 'minutes': cfdate = dt.datetime(time[0],time[1],time[2],time[3],time[4],time[5]) - refdate cfdates = cfdate.days*24.*60. + cfdate.seconds/60. elif tunits == 'seconds': cfdate = dt.datetime(time[0],time[1],time[2],time[3],time[4],time[5]) - refdate cfdates = cfdate.days*24.*3600. + cfdate.seconds elif tunits == 'milliseconds': cfdate = dt.datetime(time[0],time[1],time[2],time[3],time[4],time[5]) - refdate cfdates = cfdate.days*1000.*24.*3600. + cfdate.seconds*1000. elif tunits == 'microseconds': cfdate = dt.datetime(time[0],time[1],time[2],time[3],time[4],times[5]) - refdate cfdates = cfdate.days*1000000.*24.*3600. + cfdate.seconds*1000000. else: print errormsg print ' ' + fname + ': time units "' + tunits + '" is not ready!!!' quit(-1) return cfdates def netCDFdatetime_realdatetime(units, tcalendar, times): """ Function to transfrom from netCDF CF-compilant times to real time [units]= CF time units [tunits] since [YYYY]-[MM]-[HH] [[HH]:[MI]:[SS]] [tcalendar]= time calendar [times]= CF time values """ import datetime as dt txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Calendar type ## is360 = False if tcalendar is not None: print ' netCDFdatetime_realdatetime: There is a calendar attribute' if tcalendar == '365_day' or tcalendar == 'noleap': print ' netCDFdatetime_realdatetime: No leap years!' isleapcal = False elif tcalendar == 'proleptic_gregorian' or tcalendar == 'standard' or tcalendar == 'gregorian': isleapcal = True elif tcalendar == '360_day': is360 = True isleapcal = False else: print errormsg print ' netCDFdatetime_realdatetime: Calendar "' + tcalendar + '" not prepared!' quit(-1) # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: print ' netCDFdatetime_realdatetime: refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) dimt = len(times) # datetype = type(dt.datetime(1972,02,01)) # realdates = np.array(dimt, datetype) # print realdates ## Not in timedelta # if tunits == 'years': # for it in range(dimt): # realdate = refdate + dt.timedelta(years=float(times[it])) # realdates[it] = int(realdate.year) # elif tunits == 'months': # for it in range(dimt): # realdate = refdate + dt.timedelta(months=float(times[it])) # realdates[it] = int(realdate.year) # realdates = [] realdates = np.zeros((dimt, 6), dtype=int) if tunits == 'weeks': for it in range(dimt): realdate = refdate + dt.timedelta(weeks=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'days': for it in range(dimt): realdate = refdate + dt.timedelta(days=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'hours': for it in range(dimt): realdate = refdate + dt.timedelta(hours=float(times[it])) # if not isleapcal: # Nleapdays = cal.leapdays(int(refdate.year), int(realdate.year)) # realdate = realdate - dt.timedelta(days=Nleapdays) # if is360: # Nyears360 = int(realdate.year) - int(refdate.year) + 1 # realdate = realdate -dt.timedelta(days=Nyears360*5) # realdates[it] = realdate # realdates = refdate + dt.timedelta(hours=float(times)) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'minutes': for it in range(dimt): realdate = refdate + dt.timedelta(minutes=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'seconds': for it in range(dimt): realdate = refdate + dt.timedelta(seconds=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'milliseconds': for it in range(dimt): realdate = refdate + dt.timedelta(milliseconds=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'microseconds': for it in range(dimt): realdate = refdate + dt.timedelta(microseconds=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] else: print errormsg print ' netCDFdatetime_realdatetime: time units "' + tunits + '" is not ready!!!' quit(-1) return realdates class cls_yearly_means(object): """ Class to compute the yearly mean of a time series of values tvals=time values (year, month, day, hour, minute, second) format values=time-series of values dateref= reference date [YYYY][MM][DD][HH][MI][SS] format tunits= time units self.dimt = Number of values self.yrv = yearly mean values self.yrt = years of the mean values >>> timesv = netCDFdatetime_realdatetime('days since 1949-12-01 00:00:00', 'standard', np.arange(1129)) >>> valuesv = np.zeros((1129), dtype=np.float) >>> valuesv[0:31] = 0. >>> valuesv[31:394] = 1. >>> valuesv[395:761] = 2. >>> valuesv[761:1127] = 3. >>> valuesv[1127:1129] = 4. >>> yrlmeans = cls_yearly_means(timesv, valuesv, '19491201000000', 'days') >>> print yrlmeans.dimt,yrlmeans.yrv, yrlmeans.yrt 5 [ 0. 1. 2. 3. 4.] [ 1949. 1950. 1951. 1952. 1953.] """ def __init__(self, tvals, values, dateref, tunits): import numpy.ma as ma fillVal=1.e20 fname = 'cls_yearly_means' if tvals is None: self.dimt = None self.yrv = None self.yrt = None elif tvals == 'h': print fname + '_____________________________________________________________' print cls_yearly_means.__doc__ quit() else: dt=len(values) if dt > 0: years = np.unique(tvals[:,0]) Nyrs = len(years) yrmean = np.zeros((Nyrs), dtype=np.float) yrt = np.zeros((Nyrs), dtype=np.float) iiyr = tvals[0,0] yrmean[0] = values[0] yrt[0] = iiyr iit = 0 for iyr in range(Nyrs): nvals = 1 for it in range(iit+1,dt): # print iyr, iiyr, it, tvals[it,0],':',values[it],'->',yrmean[iyr] if tvals[it,0] == iiyr: yrmean[iyr] = yrmean[iyr] + values[it] nvals = nvals + 1 else: yrmean[iyr] = yrmean[iyr] / (nvals*1.) iiyr = tvals[it,0] yrmean[iyr + 1] = values[it] yrt[iyr+1] = iiyr iit = it break yrmean[Nyrs-1] = yrmean[Nyrs-1]/nvals self.dimt = Nyrs self.yrv = yrmean self.yrt = yrt else: print errormsg print ' ' + fname + ': No values passed!' print ' values:', values quit(-1) class cls_seasonal_means(object): """ Class to compute the seasonal mean of a time series of values tvals=time values (year, month, day, hour, minute, second) format values=time-series of values dateref= reference date [YYYY][MM][DD][HH][MI][SS] format tunits= time units self.dimt = Number of values self.seasv = seasonal mean values self.seast = seasonal time mean values """ def __init__(self, tvals, values, dateref, tunits): import numpy.ma as ma fillVal=1.e20 fname = 'cls_seasonal_means' if tvals is None: self.dimt = None self.seasv = None self.seast = None else: tseasons=times_4seasons(tvals, True) dt=len(values) seasvals=np.ones((dt/12,4), dtype=np.float64)*fillVal seastimes=np.zeros((dt/12,4), dtype=np.float64) dates = realdatetime_CFcompilant(tvals, dateref, tunits) for iseas in range(4): for it in range(dt): if tseasons[it,iseas]: tfirstseas=int(it/11) firstseas=True ## print ' first time-step of season ', iseas, ' is: ',it,' position: ',tfirstseas break for itt in range(it, dt-1,12): seasvals[int(itt/12),iseas]=(values[itt] + values[itt+1] + values[itt+2])/3. seastimes[int(itt/12),iseas]=dates[itt] + (dates[itt+2] - dates[itt])/2. ## print itt, values[itt], values[itt+1], values[itt+2], '--->', seasvals[int(itt/12),iseas] self.dimt = dt/12 self.seasv = ma.masked_equal(seasvals, fillVal) self.seast = ma.masked_equal(seastimes, fillVal) class cls_seasonal_accums(object): """ Class to compute the seasonal accumulations of a time series of values tvals=time values (year, month, day, hour, minute, second) format values=time-series of values dateref= reference date [YYYY][MM][DD][HH][MI][SS] format tunits= time units self.dimt = Number of values self.seasv = seasonal mean values self.seast = seasonal time mean values """ def __init__(self, tvals, values, dateref, tunits): import numpy.ma as ma fillVal=1.e20 if tvals is None: self.dimt = None self.seasv = None self.seast = None else: tseasons=times_4seasons(tvals, True) dt=len(values) seasvals=np.ones((dt/12,4), dtype=np.float64)*fillVal seastimes=np.zeros((dt/12,4), dtype=np.float64) dates = realdatetime_CFcompilant(tvals, dateref, tunits) for iseas in range(4): for it in range(dt): if tseasons[it,iseas]: tfirstseas=int(it/11) firstseas=True # print ' first time-step of season ', iseas, ' is: ',it,' position: ',tfirstseas break for itt in range(it, dt-1,12): seasvals[int(itt/12),iseas]=values[itt] + values[itt+1] + values[itt+2] seastimes[int(itt/12),iseas]=dates[itt] + (dates[itt+2] - dates[itt])/2. self.dimt = dt/12 self.seasv = ma.masked_equal(seasvals, fillVal) self.seast = ma.masked_equal(seastimes, fillVal) def seasonal_means(tvals, values): """ Function to compute the seasonal mean of a time series of values tvals=time values (year, month, day, hour, minute, second) format values=time-series of values """ fillVal=1.e20 tseasons=times_4seasons(tvals, True) dt=len(values) seasvals=np.ones((dt/4,4), dtype=np.float64) seasvals=seasvals*fillVal for iseas in range(4): for it in range(dt): if tseasons[it,iseas]: tfirstseas=int(it/11) firstseas=True # print ' first time-step of season ', iseas, ' is: ',it,' position: ',tfirstseas break for itt in range(it, dt-1,12): seasvals[int(itt/12),iseas]=(values[itt] + values[itt+1] + values[itt+2])/3. return seasvals def seasonal_accum(tvals, values): """ Function to compute the seasonal accumulation of a time series of values tvals=time values (year, month, day, hour, minute, second) format values=time-series of values """ fillVal=1.e20 tseasons=times_4seasons(tvals, True) dt=len(values) seasvals=np.ones((dt/4,4), dtype=np.float64) seasvals=seasvals*fillVal for iseas in range(4): for it in range(dt): if tseasons[it,iseas]: tfirstseas=int(it/11) firstseas=True # print ' first time-step of season ', iseas, ' is: ',it,' position: ',tfirstseas break for itt in range(it, dt-1,12): seasvals[int(itt/11),iseas]=values[itt] + values[itt+1] + values[itt+2] return seasvals def seasmean(timename, filename, varn): """ Function to compute the seasonal mean of a variable timename= name of the variable time in the file filename= netCDF file name varn= name of the variable """ fillValue=1.e20 ncf = NetCDFFile(filename,'a') ofile = 'seasmean_' + varn + '.nc' if not ncf.variables.has_key(varn): print errormsg print ' seasmean: File "' + filename + '" does not have variable ' + varn + ' !!!!' print errormsg ncf.close() quit(-1) varobj = ncf.variables[varn] varinf = variable_inf(varobj) timeobj = ncf.variables[timename] timeinf = cls_time_information(ncf, timename) timevals=timeobj[:] netcdftimes = netCDFdatetime_realdatetime(timeinf.units + ' since ' + timeinf.Urefdate, timeinf.calendar, timevals) timeseasons=times_4seasons(netcdftimes, True) timeseasvals=seasonal_means(netcdftimes, timevals) ncfnew = NetCDFFile(ofile,'w') ncfnew.createDimension('seastime', timeinf.dimt/4) ncfnew.createDimension('seas', 4) newvarobj = ncfnew.createVariable('seastime', 'f4', ('seastime', 'seas',), fill_value=fillValue) newvarobj[:]=timeseasvals attr = newvarobj.setncattr('calendar', timeinf.calendar) attr = newvarobj.setncattr('units', timeinf.units) newvarobj = ncfnew.createVariable('seasn', str, ('seas')) attr = newvarobj.setncattr('standard_name', 'seasn') attr = newvarobj.setncattr('long_name', 'name of the season') attr = newvarobj.setncattr('units', '3-months') newvarobj[0]='DJF' newvarobj[1]='MAM' newvarobj[2]='JJA' newvarobj[3]='SON' newdims=[] idim = 0 for ndim in varinf.dimns: if ndim == timename: newdims.append(unicode('seastime')) if not idim == 0: print errormsg print ' seasmean: File "' + filename + '" does not have time dimension "' + timename + '" as the first one. It has it as # ', idim, ' !!!!' print errormsg ncf.close() quit(-1) else: newdims.append(ndim) if not ncfnew.dimensions.has_key(ndim): ncfnew.sync() ncfnew.close() ncf.close() fdimadd(filename + ',' + ndim, ofile) ncf = NetCDFFile(filename,'r') ncfnew = NetCDFFile(ofile,'a') print ncfnew.dimensions namedims=tuple(newdims) print namedims varobj = ncf.variables[varn] varinf = variable_inf(varobj) newvarobj=ncfnew.createVariable(varn + '_seasmean', 'f4', namedims, fill_value=fillValue) attr = newvarobj.setncattr('standard_name', varn + '_seasmean') attr = newvarobj.setncattr('long_name', 'seasonal mean of ' + varn) attr = newvarobj.setncattr('units', varinf.units) newvar = np.ones(varinf.dims[0], dtype=varinf.dtype) if varinf.Ndims == 1: newvar = newvarobj[:] newvarobj[:] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 2: for i in range(varinf.dims[1]): newvar = newvarobj[:,i] newvarobj[:,i] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 3: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): newvar = newvarobj[:,i,j] newvarobj[:,i,j] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 4: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): newvar = newvarobj[:,i,j,k] newvarobj[:,i,j,k] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 5: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): newvar = newvarobj[:,i,j,k,l] newvarobj[:,i,j,k,l] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 6: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): for m in range(varinf.dims[5]): newvar = newvarobj[:,i,j,k,l,m] newvarobj[:,i,j,k,l,m] = seasonal_means(netcdftimes, newvar) else: print errormsg print ' seasmean: number of dimensions ', varinf.Ndims, ' not ready !!!!' print errormsg ncf.close() quit(-1) ncfnew.sync() ncfnew.close() print 'seasmean: Seasonal mean file "' + ofile + '" written!' def load_variable_lastdims(var, prevdims, Nlastdims): """ Function to load the last [Nlastdims] dimensions of a variable [var] at the other dimensions at [prevdims] >>> load_variable_lastdims(np.array(range(5*5*5)).reshape(5,5,5), [1,2], 1) [35 36 37 38 39] """ fname='load_variable_lastdims' dims=var.shape Ndims=len(dims) Nprevdims = len(prevdims) if not Nprevdims + Nlastdims == Ndims: print erromsg print ' ' + fname + ': number of dimensions previous (',Nprevdim,') and last (',Nlastdims, ') does not match with variable size: ',Ndims,' !!!!' print errormsg quit(-1) if Nlastdims > Ndims-1: print errormsg print ' ' + fname + ': number of last dimensions ', Nlastdims,' >= than the number of dimensions of the variable ', Ndims,' !!!!' print errormsg quit(-1) if Ndims == 1: loadvar = var[:] elif Ndims == 2: loadvar = var[prevdims[0], :] elif Ndims == 3: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], :] else: loadvar = var[prevdims[0], :, :] elif Ndims == 4: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], :, :] else: loadvar = var[prevdims[0], :, :, :] elif Ndims == 5: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :, :] elif Nlastdims == 3: loadvar = var[prevdims[0], prevdims[1], :, :, :] else: loadvar = var[prevdims[0], :, :, :, :] elif Ndims == 6: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], prevdims[4], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :, :] elif Nlastdims == 3: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :, :, :] elif Nlastdims == 4: loadvar = var[prevdims[0], prevdims[1], :, :, :, :] else: loadvar = var[prevdims[0], :, :, :, :] else: print errormsg print ' ' + fname + ' variable size ', Ndims, ' not ready!!!' print errormsg quit(-1) return loadvar def load_ncvariable_lastdims(ncf, varn, prevdims, Nlastdims): """ Function to load the last [Nlastdims] dimensions of a variable [varn] from a netCDF object at the other dimensions at [prevdims] ncf= netCDF object varn= variable name prevdims= values at the previous dimensions Nlastims= number of last dimensions """ fname='load_ncvariable_lastdims' var=ncf.variables[varn] dims=var.shape Ndims=len(dims) Nprevdims = len(prevdims) if not Nprevdims + Nlastdims == Ndims: print erromsg print ' ' + fname + ': number of dimensions previous (',Nprevdim,') and last (',Nlastdims, ') does not match with variable size: ',Ndims,' !!!!' print errormsg quit(-1) if Nlastdims > Ndims-1: print errormsg print ' ' + fname + ': number of last dimensions ', Nlastdims,' >= than the number of dimensions of the variable ', Ndims,' !!!!' print errormsg quit(-1) if Ndims == 1: loadvar = var[:] elif Ndims == 2: loadvar = var[prevdims[0], :] elif Ndims == 3: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], :] else: loadvar = var[prevdims[0], :, :] elif Ndims == 4: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], :, :] else: loadvar = var[prevdims[0], :, :, :] elif Ndims == 5: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :, :] elif Nlastdims == 3: loadvar = var[prevdims[0], prevdims[1], :, :, :] else: loadvar = var[prevdims[0], :, :, :, :] elif Ndims == 6: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], prevdims[4], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :, :] elif Nlastdims == 3: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :, :, :] elif Nlastdims == 4: loadvar = var[prevdims[0], prevdims[1], :, :, :, :] else: loadvar = var[prevdims[0], :, :, :, :] else: print errormsg print ' ' + fname + ' variable size ', Ndims, ' not ready!!!' print errormsg quit(-1) return loadvar def fill_ncvariable_lastdims(ncf, varn, prevdims, Nlastdims, fillvals): """ Function to fill the last [Nlastdims] dimensions of a variable [varn] from a netCDF object at the other dimensions at [prevdims] with as given set of [fillvals] values ncf= netCDF object varn= variable name prevdims= values at the previous dimensions Nlastims= number of last dimensions fillvals= values to use to fill """ fname='fill_ncvariable_lastdims' var=ncf.variables[varn] dims=var.shape Ndims=len(dims) Nprevdims = len(prevdims) if not Nprevdims + Nlastdims == Ndims: print erromsg print ' ' + fname + ': number of dimensions previous (',Nprevdim,') and last (',Nlastdims, ') does not match with variable size: ',Ndims,' !!!!' print errormsg quit(-1) if Nlastdims > Ndims-1: print errormsg print ' ' + fname + ': number of last dimensions ', Nlastdims,' >= than the number of dimensions of the variable ', Ndims,' !!!!' print errormsg quit(-1) if Ndims == 1: var[:] = fillvals elif Ndims == 2: var[prevdims[0], :] = fillvals elif Ndims == 3: if Nlastdims == 1: var[prevdims[0], prevdims[1], :] = fillvals else: var[prevdims[0], :, :] = fillvals elif Ndims == 4: if Nlastdims == 1: var[prevdims[0], prevdims[1], prevdims[2], :] = fillvals elif Nlastdims == 2: var[prevdims[0], prevdims[1], :, :] = fillvals else: var[prevdims[0], :, :, :] = fillvals elif Ndims == 5: if Nlastdims == 1: var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :] = fillvals elif Nlastdims == 2: var[prevdims[0], prevdims[1], prevdims[2], :, :] = fillvals elif Nlastdims == 3: var[prevdims[0], prevdims[1], :, :, :] = fillvals else: var[prevdims[0], :, :, :, :] = fillvals elif Ndims == 6: if Nlastdims == 1: var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], prevdims[4], :] = fillvals elif Nlastdims == 2: var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :, :] = fillvals elif Nlastdims == 3: var[prevdims[0], prevdims[1], prevdims[2], :, :, :] = fillvals elif Nlastdims == 4: var[prevdims[0], prevdims[1], :, :, :, :] = fillvals else: var[prevdims[0], :, :, :, :] = fillvals else: print errormsg print ' ' + fname + ' variable size ', Ndims, ' not ready!!!' print errormsg quit(-1) def maskvar(values, filename, varn): """ Function to mask a variable using a mask. It is assumed that mask[...,dimM,dimJ,dimK] and var[...,dimM,dimJ,dimK] share the last dimensions values= [maskfilename]:[maskvar]:[dimsmask]:[timename] [maskfilename]= file with the mask [maskvar]= name of the variable with the mask [dimsmask]= dimensions d1,d2,d3,... to use from variable [mask] (ahead of the sharing ones with [var]) [timename]= name of the variable time in [filename] filename= netCF file name varn= variable name """ fname='maskvar' fillValue=1.e20 ofile = 'mask_' + varn + '.nc' maskfilename = values.split(':')[0] maskvar = values.split(':')[1] maskdims = np.array(values.split(':')[2].split(','), dtype=int) timename = values.split(':')[3] ncf = NetCDFFile(filename,'r') if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ': File "' + filename + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) varobj = ncf.variables[varn] varinf = variable_inf(varobj) timeobj = ncf.variables[timename] timeinf = cls_time_information(ncf, timename) ncfmask = NetCDFFile(maskfilename,'r') if not ncfmask.variables.has_key(maskvar): print errormsg print ' ' + fname + ': File "' + maskfilename + '" does not have variable mask "' + maskvar + '" !!!!' print errormsg ncf.close() ncfmask.close() quit(-1) varmaskobj = ncfmask.variables[maskvar] varmaskinf = variable_inf(varmaskobj) sdims=[] for ivdim in range(varinf.Ndims): vardim = varinf.Ndims - ivdim - 1 maskdim = varmaskinf.Ndims - ivdim - 1 if varinf.dims[vardim] == varmaskinf.dims[vardim]: sdims.append(varinf.dims[vardim]) else: break samedims = sdims[::-1] print ' ' + fname + ': variable: ', varinf.dims, ' and mask: ', varmaskinf.dims,' share: ', samedims # masking ## Nsamedims=len(samedims) if not Nsamedims + len(maskdims) == varmaskinf.Ndims: print errormsg print ' ' + fname + ': desired dimensions of mask ', maskdims, ' and coincident dims with variable ',Nsamedims, ' does not match mask dims ',varmaskinf.Ndims,' !!!!' print errormsg ncf.close() ncfmask.close() quit(-1) if varmaskinf.Ndims > Nsamedims: maskvals = load_ncvariable_lastdims(ncfmask, maskvar, maskdims, Nsamedims) else: maskvals = maskvarobj[:] ncfmask.close() # Creation of the new file ## ncfnew = NetCDFFile(ofile,'w') for idim in range(varinf.Ndims): newdimv = ncf.dimensions[varinf.dimns[idim]] if newdimv.isunlimited(): ncfnew.createDimension(varinf.dimns[idim], None) else: ncfnew.createDimension(varinf.dimns[idim], varinf.dims[idim]) newvar = ncfnew.createVariable(varn, varinf.dtype, varinf.dimns, fill_value=fillValue) for attrn in varinf.attributes: if not attrn == '_FillValue': attr = newvar.setncattr(attrn, varobj.getncattr(attrn)) attr = newvar.setncattr('mask', 'variable masked using ' + values.split(':')[2] + ' of variable ' + maskvar + ' from file ' + maskfilename) ncfnew.sync() ncfnew.close() for varns in ncf.variables: if not varns == varn: fvaradd(filename + ':' + varns, ofile) ncfnew = NetCDFFile(ofile,'a') Nptmask = 1 for isdim in range(Nsamedims): Nptmask = Nptmask * samedims[isdim] Nvarloop = 1 varsecdims = [] for insdim in range(varinf.Ndims - Nsamedims): Nvarloop = Nvarloop * varinf.dims[insdim] varsecdims.append(0) Nsecdims = len(varsecdims) for iloop in range(Nvarloop): varval = load_ncvariable_lastdims(ncf, varn, varsecdims, Nsamedims) newvarval = np.where(maskvals == 1, varval, fillValue) if np.sum(newvarval == fillValue) == Nptmask: print errormsg print ' ' + fname + ': all values got masked!!!' print erromsg quit(-1) fill_ncvariable_lastdims(ncfnew,varn,varsecdims,Nsamedims,newvarval) varsecdims[Nsecdims -1] = varsecdims[Nsecdims - 1] + 1 if varsecdims[Nsecdims - 1] > varinf.dims[Nsecdims - 1] - 1: varsecdims[Nsecdims - 1] = 0 for jdim in range(Nsecdims - 1): varsecdims[Nsecdims - 2 - jdim] = varsecdims[Nsecdims - 2 - jdim] + 1 if varsecdims[Nsecdims - 2 - jdim] <= varinf.dims[Nsecdims - 2 - jdim] - 1: break else: varsecdims[Nsecdims - 2 - jdim] = 0 print 'variable: ', varinf.dims, ' and mask: ', varmaskinf.dims,' share: ', samedims, Nvarloop ncfnew.sync() ncfnew.close() ncf.close() print 'masked file "' + ofile + '" generated !!!' def lonlat2D(ncfobj, lonn, latn): """ Function to create 2D matricies of longitudes and latitudes from the file ncfobj= netCDF object file lonn= longitude name latn= latitude name """ fname='lonlat2D' lonvar = ncfobj.variables[lonn] latvar = ncfobj.variables[latn] loninf = variable_inf(lonvar) latinf = variable_inf(latvar) if loninf.Ndims == 1: lonv = lonvar[:] latv = latvar[:] dx=loninf.dims[0] dy=latinf.dims[0] lonval = np.zeros((dy, dx), dtype=float) latval = np.zeros((dy, dx), dtype=float) for iy in range(dy): lonval[iy,:] = lonv for ix in range(dx): latval[:,ix] = latv elif loninf.Ndims == 2: lonval = lonvar[:] latval = latvar[:] elif loninf.Ndims == 3: lonval = lonvar[0,:,:] latval = latvar[0,:,:] else: print errormsg print ' ' + fname + ' dimension ', lonvar.Ndims, ' of the lon/lat matrices not ready !!!!' return lonval, latval def indices_mask(mask): """ Function to retrieve the indices from a 2D mask mask= 2D mask matrix mat=np.array(range(100), dtype=float).reshape((10,10)) >>> mask1= mat >= 45. >>> mask2= mat <= 55. >>> mask = mask1 * mask2 indices_mask(mask) [[4 5] [4 6] [4 7] [4 8] [4 9] [5 0] [5 1] [5 2] [5 3] [5 4] [5 5]] """ dy=mask.shape[0] dx=mask.shape[1] Nptmask = np.sum(mask) maskindices = np.zeros((Nptmask,2), dtype=int) ij=0 for j in range(dy): for i in range(dx): if mask[j,i] == True: maskindices[ij,0] = j maskindices[ij,1] = i ij = ij + 1 return maskindices def chgtimestep(values, origfile, varN): """ Function to change the values of a given time-step of a variable inside a netCDF for values from a nother file [values]=[origtime_step]:[newfile]:[newtime_step] [origtime_step]: desired original time step to change: 'first', 'last', 'middle', '[num]' [newfile]: new netCDF file name [newtime_step]: desired new time step to change: 'first', 'last', 'middle', '[num]' 'zeros', 'fillVal' 'first': first time-step 'last': last time-step 'middle': middle time-step '[num]': [num] time-step 'zeros': fill time-step with zeros 'fillVal': fill time-step with fill_Values [FillValue] [varN]= variable name [origfile]= original netCDF file name """ fname = 'chgtimestep' ermsg = fname + ': ERROR -- error -- ERROR -- error' vals = values.split(':') origtime_step = vals[0] newfile = vals[1] newtime_step = vals[2] if len(vals) == 4: FillValue = np.float(vals[3]) filexist(origfile, ermsg, 'original') filexist(newfile, ermsg, 'new') print 'origfile: ',origfile ncofile = NetCDFFile(origfile,'a') varinfile(ncofile, origfile, ermsg, 'change', varN) ncnfile = NetCDFFile(newfile,'r') varinfile(ncofile, newfile, ermsg, 'new', varN) ncnfile.close() if newtime_step == 'zeros': newvals = uploading_timestep(newfile, varN, 'first') newvals = 0. elif newtime_step == 'fillVal': newvals = uploading_timestep(newfile, varN, 'first') newvals = FillValue else: newvals = uploading_timestep(newfile, varN, newtime_step) varobj = ncofile.variables[varN] Ndims = len(varobj.shape) dimt = varobj.shape[0] if origtime_step == 'first': oval = 0 elif origtime_step == 'middle': oval = int(dimt/2) elif origtime_step == 'last': oval = dimt-1 else: oval = int(origtime_step) if Ndims == 1: varobj[oval] = newvals elif Ndims == 2: varobj[oval,:] = newvals elif Ndims == 3: varobj[oval,:,:] = newvals elif Ndims == 4: varobj[oval,:,:,:] = newvals elif Ndims == 5: varobj[oval,:,:,:,:] = newvals elif Ndims == 6: varobj[oval,:,:,:,:,:] = newvals else: print ermsg print ' variable size', Ndims, 'not ready !' print ermsg quit(-1) ncofile.sync() ncofile.close() def uploading_timestep(ncfile, varN, time_step): """ Function to upload a given time-step of a variable inside a netCDF [ncfile]= netCDF file name [varN]= variable name [time_step]= desired time step: 'first', 'last', 'middle', '[num]' 'first': first time-step 'last': last time-step 'middle': middle time-step '[num]': [num] time-step """ fname = 'uploading_timestep' ermsg = fname + ': ERROR -- error -- ERROR -- error' filexist(ncfile, ermsg, 'uploading') ncufile = NetCDFFile(ncfile,'r') varinfile(ncufile, ncfile, ermsg, 'uploading', varN) varobj = ncufile.variables[varN] Ndims = len(varobj.shape) dimt = varobj.shape[0] if time_step == 'first': upval = 0 elif time_step == 'middle': upval = int(dimt/2) elif time_step == 'last': upval = dimt-1 else: upval = int(time_step) if Ndims == 1: uploadvar = varobj[upval] elif Ndims == 2: uploadvar = varobj[upval,:] elif Ndims == 3: uploadvar = varobj[upval,:,:] elif Ndims == 4: uploadvar = varobj[upval,:,:,:] elif Ndims == 5: uploadvar = varobj[upval,:,:,:,:] elif Ndims == 6: uploadvar = varobj[upval,:,:,:,:,:] else: print ermsg print ' variable size', Ndims, 'not ready !' print ermsg quit(-1) return uploadvar def timeshiftvar(values, fileN, varN, FillVal=1.e20): """ Function to temporaly shift a number of time-steps a given variable inside a netCDF file [values]=[nsteps]:[[FillValue]] [nsteps]: desired number of time step to move [FillValue]: value to fill the empty time-steps [fileobj]= original netCDF object [varN]= variable name """ from sys import stdout fname = 'timesfhitvar' ermsg = fname + ': ERROR -- error -- ERROR -- error' nsteps = int(values.split(':')[0]) if len(values.split(':')) == 2: FillVal = np.float(values.split(':')[1]) fileobj = NetCDFFile(fileN,'a') varobj = fileobj.variables[varN] vardims = varobj.shape vartype = varobj.dtype Ndims = len(vardims) dimt = vardims[0] if nsteps > 0: inimov = nsteps endmov = dimt iniplace = 0 endplace = dimt - nsteps else: inimov = 0 endmov = dimt + nsteps iniplace = -nsteps endplace = dimt print ' ' + fname + ': moving', nsteps, '___ __ _' print ' from (', inimov, ',', endmov, ') to (',iniplace,',',endplace,')' if Ndims == 1: vals = np.ones(vardims, dtype=vartype)*FillVal vals[iniplace:endplace] = varobj[inimov:endmov] varobj[:] = vals elif Ndims == 2: vals = np.ones(vardims, dtype=vartype)*FillVal vals[iniplace:endplace,:] = varobj[inimov:endmov,:] varobj[:] = vals elif Ndims == 3: vals = np.ones((vardims[0], vardims[1]), dtype=vartype)*FillVal for i in range(vardims[2]): if (i%10 == 0): print '\r shifted ' + '{0:5.2g}'.format(i*100./vardims[2]) + ' %', stdout.flush() vals[iniplace:endplace,:] = varobj[inimov:endmov,:,i] varobj[:,:,i] = vals elif Ndims == 4: vals = np.ones((vardims[0], vardims[1]), dtype=vartype)*FillVal for i in range(vardims[2]): for j in range(vardims[3]): nvals=i*vardims[3] + j if (nvals%100 == 0): print '\r shifted ' + '{0:5.2g}'.format(nvals*100./(vardims[2]*vardims[3])) + ' %', stdout.flush() vals[iniplace:endplace,:] = varobj[inimov:endmov,:,i,j] varobj[:,:,i,j] = vals elif Ndims == 5: vals = np.ones((vardims[0], vardims[1]), dtype=vartype)*FillVal for i in range(vardims[2]): for j in range(vardims[3]): for k in range(vardims[4]): nvals=i*vardims[3]*vardims[4] + j*vardims[4] + k if (nvals%1000 == 0): print '\r shifted ' + '{0:5.2g}'.format(nvals*100./(vardims[2]*vardims[3]*vardims[4])) + ' %', stdout.flush() vals[iniplace:endplace,:] = varobj[inimov:endmov,:,i,j,k] varobj[:,:,i,j,k] = vals elif Ndims == 6: vals = np.ones((vardims[0], vardims[1]), dtype=vartype)*FillVal for i in range(vardims[2]): for j in range(vardims[3]): for k in range(vardims[4]): for l in range(vardims[5]): nvals=i*vardims[3]*vardims[4]*vardims[5] + j*vardims[4]*vardims[5] + k*vardims[5] + l if (nvals%10000 == 0): print '\r shifted ' + \ '{0:5.2g}'.format(nvals*100./(vardims[2]*vardims[3]*vardims[4]*vardims[5])) + ' %', stdout.flush() vals[iniplace:endplace,:] = varobj[inimov:endmov,:,i,j,k,l] varobj[:,:,i,j,k,l] = vals else: print ermsg print ' variable size', Ndims, 'not ready !' print ermsg quit(-1) print stdout.write("\n") fileobj.sync() fileobj.close() def typemod(value, typeval): """ Function to give back a value in a given dtype >>> print(typemod(8.2223, 'np.float64')) >>> print(typemod(8.2223, 'tuple')) """ fname='typemod' if typeval == 'int': return int(value) elif typeval == 'long': return long(value) elif typeval == 'float': return float(value) elif typeval == 'complex': return complex(value) elif typeval == 'str': return str(value) elif typeval == 'bool': return bool(value) elif typeval == 'list': newval = [] newval.append(value) return newval elif typeval == 'dic': newval = {} newval[value] = value return newval elif typeval == 'tuple': newv = [] newv.append(value) newval = tuple(newv) return newval elif typeval == 'np.int8': return np.int8(value) elif typeval == 'np.int16': return np.int16(value) elif typeval == 'np.int32': return np.int32(value) elif typeval == 'np.int64': return np.int64(value) elif typeval == 'np.uint8': return np.uint8(value) elif typeval == 'np.uint16': return np.uint16(value) elif typeval == 'np.np.uint32': return np.uint32(value) elif typeval == 'np.uint64': return np.uint64(value) elif typeval == 'np.float': return np.float(value) elif typeval == 'np.float16': return np.float16(value) elif typeval == 'np.float32': return np.float32(value) elif typeval == 'float32': return np.float32(value) elif typeval == 'np.float64': return np.float64(value) elif typeval == 'float64': return np.float64(value) elif typeval == 'np.complex': return np.complex(value) elif typeval == 'np.complex64': return np.complex64(value) elif typeval == 'np.complex128': return np.complex128(value) else: print errormsg print fname + ': data type "' + typeval + '" is not ready !' print errormsg quit(-1) def addvals(values, filename, varN): """ Function to add values to a given variable at a given dimension values = [dimension]:[position]:[Nvals]:[addval] [dimension]: dimension at which the variable wants to be increased [position]: position within the dimension at which the variable wants to be increased 'first': first position of the dimension 'middle': middle position of the dimension 'last': last position of the dimension 'num': before given value [Nvals]: number of values [addval]: value to give to the added values 'val': a given value 'missing': as a missing value filename = name of the netCDF file varN = name of the variable """ import subprocess as sub fname = 'addvals' if values == 'h': print fname + '_____________________________________________________________' print addvals.__doc__ quit() newf = 'addvals.nc' dimension=int(values.split(':')[0]) position=values.split(':')[1] Nvals=int(values.split(':')[2]) addval=values.split(':')[3] filexist(filename, errormsg, 'add values') ncfa = NetCDFFile(filename,'r') varinfile(ncfa, filename, errormsg, 'adding', varN) varobj=ncfa.variables[varN] varinf=variable_inf(varobj) newdims=list(varinf.dims) newdims[dimension] = newdims[dimension] + Nvals newdims=tuple(newdims) dmodname = varinf.dimns[dimension] print ' ' + fname + ' increassed dimension "' + dmodname + '"' if position == 'first': pos=0 elif position == 'middle': pos=(varinf.dims[dimension])/2 elif position == 'last': pos=varinf.dims[dimension] else: pos=int(position) if pos > varinf.dims[dimension]: print errormsg print fname + ': position given ', pos,' is too big for the size of the dimension ',varinf.dims[dimension], '!!!!' print errormsg quit(-1) ## print 'starting position: ', pos if addval == 'missing': addvaln = 1.e20 else: addvaln = addval addvalv=typemod(addvaln, str(varinf.dtype)) newnc = NetCDFFile(newf,'w') # Dimensions ## for dim in ncfa.dimensions.values(): dname = dim._name dsize = dim.__len__() if dim.isunlimited(): dsize=None else: if dname == varinf.dimns[dimension]: dsize=newdims[dimension] newnc.createDimension(dname, size=dsize) # Variables ## for var in ncfa.variables.values(): vname = var._name vkind = var.dtype vdims = var.dimensions varattrs = var.ncattrs() if searchInlist(varattrs, '_FillValue'): fillval = var.getncattr('_FillValue') newvarobj = newnc.createVariable(vname, vkind, vdims, fill_value=fillval) elif vname == varN and addval == 'missing': newvarobj = newnc.createVariable(vname, vkind, vdims, fill_value=typemod(addvaln, str(varinf.dtype))) else: newvarobj = newnc.createVariable(vname, vkind, vdims) varobj=ncfa.variables[vname] vals = varobj[:] if vname == varN: for ival in range(Nvals): ## print ' ' + fname + ': adding',ival+1,' values' newvar = np.insert(vals, pos, addvalv, axis=dimension) vals=newvar.copy() newvarobj[:] = newvar else: if vname == dmodname: # Increassing dimension variable values ## dvals=ncfa.variables[dmodname][:] if position == 'first': valB = -(dvals[1] - dvals[0]) valA=dvals[pos] + valB elif position == 'last': valB=-(dvals[1] - dvals[0]) valA=dvals[pos-1] - valB*Nvals else: valB=-(dvals[pos] - dvals[pos-1])/(Nvals+1) valA=dvals[pos-1] - valB*Nvals for ival in range(Nvals): addvalvar=valA + valB*ival ## print ' ' + fname + ' adding to dimension variable "', dmodname ,'" value ',addvalvar newvar = np.insert(vals, pos, addvalvar, axis=0) vals=newvar.copy() newvarobj[:] = newvar else: vardims=list(var.dimensions) if searchInlist(vardims,varinf.dimns[dimension]): ## print ' ' + fname + ' variable ' + vname + ' has also the increased dimension!' vardim=vardims.index(varinf.dimns[dimension]) for ival in range(Nvals): addvalvar = addval newvar = np.insert(vals, pos, addvalvar, axis=vardim) vals=newvar.copy() newvarobj[:] = newvar else: newvarobj[:] = vals for vattr in varobj.ncattrs(): if not vattr == '_FillValue': vattrval = var.getncattr(vattr) newvarobj = set_attribute(newvarobj, vattr, vattrval) # Global attributes ## for gattr in ncfa.ncattrs(): gattrval = ncfa.getncattr(gattr) newnc = set_attribute(newnc, gattr, gattrval) newnc.sync() newnc.close() ncfa.close() try: ins = 'cp ' + newf + ' ' + filename syscode = sub.call(ins, shell=True) if not syscode == 0: print errmsg print " Copy to the new file failed!" print ins print syscode quit() else: sub.call('rm ' + newf, shell=True) except OSError, e: print errmsg print " Copy failed!" print e quit(-1) def TimeInf(filename, tname): """ Function to print all the information from the variable time filename= netCDF file name tname= name of the variable time """ ncfobj = NetCDFFile(filename,'r') timeinf = cls_time_information(ncfobj, tname) fmtprinting_class(timeinf) ncfobj.close() def sorttimesmat(filename, tname): """ Function to sort the time values of a given file tname, name of the variable time filename= netCDF file name """ fname = 'sorttimesmat' filexist(filename, errormsg, 'to sort time') ncft = NetCDFFile(filename,'a') varinfile(ncft, filename, errormsg, 'time', tname) timeobj = ncft.variables[tname] timeinf = cls_time_information(ncft, tname) tvals = timeobj[:] Ltvals = list(tvals) sortedtvals = sorted(Ltvals) tinds = np.zeros((timeinf.dimt), dtype=int) for it in range(timeinf.dimt): tinds[it] = Ltvals.index(sortedtvals[it]) for var in ncft.variables: varobj = ncft.variables[var] if searchInlist(varobj.dimensions, tname): print fname + ': time-sorting variable: "' + var + '" ' varvals = varobj[:] newvarvals = varvals.copy() for it in range(timeinf.dimt): if var == 'time': print it, tinds[it], tvals[tinds[it]] newvarvals[it,] = varvals[tinds[it],] varobj[:] = newvarvals ncft.sync() ncft.close() def diffdate_units(diffdate, units): """ Function to transform a difference of dates to a certain units diffdate = difference of dates (as timedelta) units = units to transform: 'weeks', 'days', 'hours', 'minutes', 'seconds', 'miliseconds', 'nanoseconds' >>> diffdate_units(dt.datetime(1976, 2, 17, 8, 32, 5) - dt.datetime(1949, 12, 1, 0, 0, 0), 'seconds') 827224325.0 """ fname = 'diffdate_units' if units == 'weeks': diffunits = diffdate.days/7. + diffdate.seconds/(3600.*24.*7.) elif units == 'days': diffunits = diffdate.days + diffdate.seconds/(3600.*24.) elif units == 'hours': diffunits = diffdate.days*24. + diffdate.seconds/(3600.) elif units == 'minutes': diffunits = diffdate.days*24.*60. + diffdate.seconds/(60.) elif units == 'seconds': diffunits = diffdate.days*24.*3600. + diffdate.seconds elif units == 'miliseconds': diffunits = diffdate.days*24.*3600.*1000. + diffdate.seconds*1000. elif units == 'nanoseconds': diffunits = diffdate.days*24.*3600.*1000000. + diffdate.seconds*1000000. else: print errormsg print fname + ': time units "' + units + '" not ready!!!' print errormsg quit(-1) return diffunits def days_month(year,month): """ Function to give the number of days of a month of a given year >>> days_month(1976,2) 29 """ import datetime as dt date1=dt.date(year,month,1) date2=dt.date(year,month+1,1) diffdate = date2-date1 return diffdate.days def mid_period(yr1,mon1,day1,hour1,min1,sec1,yr2,mon2,day2,hour2,min2,sec2): """ Function to give the mid poiint of a period >>> mid_period(1976,2,1,0,0,0,1976,3,1,0,0,0) [1976 2 15 12 0 0] """ import datetime as dt date1=dt.datetime(yr1,mon1,day1,hour1,min1,sec1) date2=dt.datetime(yr2,mon2,day2,hour2,min2,sec2) diffdate = date2-date1 diffseconds = diffdate.days*24*3600 + diffdate.seconds diff2 = dt.timedelta(seconds=diffseconds/2) datenew = date1 + diff2 datenewvals = np.array([datenew.year, datenew.month, datenew.day, datenew.hour, \ datenew.minute, datenew.second]) return datenewvals def days_year(year): """ Function to give the number of days of a year >>> days_year(1976) 366 """ import datetime as dt date1=dt.date(year,1,1) date2=dt.date(year+1,1,1) diffdate = date2-date1 return diffdate.days def days_period(yeari,yearf): """ Function to give the number of days for a period >>> days_period(1976,1980) 1827 """ ndays=0 for iyr in range(yearf-yeari+1): ndays=ndays+days_year(yeari+iyr) return ndays def check_times_file(values, filename, timen): """ Function to check time-steps of a given file values=[FirstDate]:[LastDate][freq]:[auxmonth] [FirstDate] = first date within file (in [YYYY][MM][DD][HH][MI][SS] format) [LastDate] = last date within file (in [YYYY][MM][DD][HH][MI][SS] format) [freq] = frequency of the data within the file (in time units) or 'month' (monthly values) [auxmonth] = in case of 'month' midmon: dates are given according to the mid time value of the month real: dates are given increassing a month from the previous date filename: name of the file timen: name of the variable time """ import datetime as dt fname = 'check_times_file' FirstDate = values.split(':')[0] LastDate = values.split(':')[1] freq = values.split(':')[2] ncft = NetCDFFile(filename, 'r') varinfile(ncft, filename, errormsg, '', timen) timeobj=ncft.variables[timen] timeinf = cls_time_information(ncft, timen) timevals = timeobj[:] ncft.close() yrf=int(FirstDate[0:4]) monf=int(FirstDate[4:6]) dayf=int(FirstDate[6:8]) horf=int(FirstDate[8:10]) minf=int(FirstDate[10:12]) secf=int(FirstDate[12:14]) firstdate = dt.datetime(yrf, monf, dayf, horf, minf, secf) yrl=int(LastDate[0:4]) monl=int(LastDate[4:6]) dayl=int(LastDate[6:8]) horl=int(LastDate[8:10]) minl=int(LastDate[10:12]) secl=int(LastDate[12:14]) lastdate = dt.datetime(yrl, monl, dayl, horl, minl, secl) firstT = firstdate - timeinf.refdate t0 = diffdate_units(firstT, timeinf.units) period = lastdate - firstdate per = diffdate_units(period, timeinf.units) if freq != 'month': theordimt = np.int(per / np.float64(freq) + 1) else: theordimt = np.int((yrl - yrf + 1) * 12) theordates = np.zeros((theordimt), dtype=np.float64) theordates[0] = t0 if freq != 'month': theordates = t0 + np.arange(1,theordimt) * np.float64(freq) else: auxmonth = values.split(':')[3] if auxmonth == 'midmon': t01 = np.array([yrf, monf, 1, 0, 0, 0]) else: t01 = np.array([yrf, monf, dayf, horf, minf, secf]) for it in range(1,theordimt): t02 = t01.copy() t02[1] = t02[1] + 1 if t02[1] > 12: t02[1] = 1 t02[0] = t02[0] + 1 date01 = dt.datetime(t01[0], t01[1], t01[2], t01[3], t01[4], t01[5]) if auxmonth == 'midmon': date02 = dt.datetime(t02[0], t02[1], t02[2], t02[3], t02[4], t02[5]) diff012 = date02 - date01 t01 = diffdate_units(date01 - timeinf.refdate, timeinf.units) theordates[it] = t01 + diffdate_units(diff012, timeinf.units) / 2. elif auxmonth == 'real': date02 = dt.datetime(t02[0], t02[1], t01[2], t01[3], t01[4], t01[5]) diff012 = date02 - date01 theordates[it] = theordates[it-1] + diffdate_units(diff012, timeinf.units) else: print errormsg print fname + ': frequency "month" required third argument "' + auxmonth + '" not ready!!!!' print errormsg quit(-1) t01 = t02.copy() equaldates = sorted(list(set(theordates) & set(timevals))) missdates = sorted(list(set(theordates) - set(timevals))) Nequal = len(equaldates) Ndiff = len(missdates) if theordimt != timeinf.dimt: print warnmsg print fname + ': there are not the correspondant times (', theordimt ,') in file "' + filename + '" with ', timeinf.dimt,' time-steps !!!!!!!' print ' ' + fname + ': Number equal times-steps: ', Nequal, 'Number different time-steps: ', Ndiff if Nequal >= Ndiff or Nequal == 0: matdates = netCDFdatetime_realdatetime(timeinf.unitsval, timeinf.calendar, missdates) print ' file does not have dates: ' for it in range(Ndiff): print matdates[it,:] else: matdates = netCDFdatetime_realdatetime(timeinf.unitsval, timeinf.calendar, equaldates) print ' file only has: ' for it in range(Nequal): print matdates[it,:] def writing_str_nc(varo, values, Lchar): """ Function to write string values in a netCDF variable as a chain of 1char values varo= netCDF variable object values = list of values to introduce Lchar = length of the string in the netCDF file """ Nvals = len(values) for iv in range(Nvals): stringv=values[iv] charvals = np.chararray(Lchar) Lstr = len(stringv) charvals[Lstr:Lchar] = '' for ich in range(Lstr): charvals[ich] = stringv[ich:ich+1] varo[iv,:] = charvals return def writing_1str_nc(varo, values, Lchar): """ Function to write 1 string value in a netCDF variable as a chain of 1char values varo= netCDF variable object values = list of values to introduce Lchar = length of the string in the netCDF file """ stringv=values charvals = np.chararray(Lchar) Lstr = len(stringv) charvals[Lstr:Lchar] = '' for ich in range(Lstr): charvals[ich] = stringv[ich:ich+1] varo[:] = charvals return def get_1str_nc(varo, Lchar): """ Function to get 1 string value in a netCDF variable as a chain of 1char values varo= netCDF variable object Lchar = length of the string in the netCDF file """ string = '' for ich in range(Lchar): charval = str(stringv[ich:ich+1]).replace('[','').replace(']','') if charval == '': break else: if ich == 0: string = charval else: string = string + charval return string def get_str_nc(varo, Lchar): """ Function to get string values in a netCDF variable as a chains of 1char values varo= netCDF variable object Lchar = length of the string in the netCDF file """ Nvalues = varo.shape[0] strings = [] for ival in range(Nvalues): stringv = varo[ival] string = str('') for ich in range(Lchar): charval = str(stringv[ich:ich+1]).replace('[','').replace(']','') if charval == '--': break else: if ich == 0: string = charval else: string = string + charval strings.append(string) return strings class ysubsetstats(object): """ Class to compute multi-year statistics of a given subset of values values= values to use dates=dates of the values in matrix format ([year], [month], [day], [hour], [minute], [second]) sec=section to use for the period='year', 'month', 'day', 'hour', 'minute', 'second' vali=initial value of the period valf=final value of the period Nq= number of quantiles (20 for 5% bins, it is going to produce 21 values) missval= missing value self.Nvalues = Number of non masked values self.min = minimum non masked value self.max = maximum non masked value self.mean = mean non masked value self.mean2 = quandratic mean non masked value self.stdev = standard deviation non masked value self.quantiles = quantiles non masked value """ def __init__(self, values, dates, sec, vali, valf, Nq, missVal): import numpy.ma as ma fname = 'ysubsetstats' if values is None: self.Nvalues = None self.min = None self.max = None self.mean = None self.mean2 = None self.stdev = None self.quantiles = None else: Nvalues = len(values) Ndates = len(dates) if Nvalues != Ndates: print errormsg print fname + ': number of values ', Nvalues,' does not coincide with the number of dates ',Ndates, '!!!!!!' print errormsg quit(-1) initialsub = np.zeros(Nvalues, dtype=bool) endsub = initialsub.copy() if missVal > 0.: valuesmask = ma.masked_greater_equal(values, missVal*0.9) else: valuesmask = ma.masked_less_equal(values, missVal*0.9) if sec == 'year': isec = 0 elif sec == 'month': isec = 1 elif sec == 'day': isec = 2 elif sec == 'hour': isec = 3 elif sec == 'minute': isec = 4 elif sec == 'second': isec = 5 else: print errormsg print fname + ': sction "' + sec + '" not ready!!!!' print errormsg quit(-1) if vali < valf: timesmaskperi = ma.masked_less(dates[:,isec], vali) timesmaskperf = ma.masked_greater(dates[:,isec], valf) else: timesmaskperi = ma.masked_less(dates[:,isec], valf) timesmaskperf = ma.masked_greater(dates[:,isec], vali) finalmask = valuesmask.mask+timesmaskperi.mask+timesmaskperf.mask finalvalues = ma.array(values, mask=finalmask) finalvalues2 = finalvalues*finalvalues self.Nvalues = finalvalues.count() self.min = finalvalues.min() self.max = finalvalues.max() self.mean = finalvalues.mean() self.mean2 = finalvalues2.mean() self.stdev = finalvalues.std() self.quantiles = mask_quantiles(finalvalues, Nq) def remapnn(values, filename, varn, ddegL, ddegl): """ Function to remap to the nearest neightbor a variable using projection from another file values=[newprojectionfile]:[newlonname,newlatname]:[oldlonname,oldlatname] [newprojectionfile]: name of the file with the new projection [newlonname,newlatname]: name of the longitude and the latitude variables in the new file [oldlonname,oldlatname]: name of the longitude and the latitude variables in the old file filename= netCDF file name varn= variable name ddegL= Longitude range around the old grid point to speed up the search ddegl= latitude range around the old grid point to speed up the search """ fname = 'remapnn' ofile = 'remapnn_' + varn + '.nc' fillValue = 1.e20 newprojectionfile = values.split(':')[0] newlonlatname = values.split(':')[1] oldlonlatname = values.split(':')[2] filexist(filename, errormsg, 'old file') filexist(newprojectionfile, errormsg, 'new file') oldlon=oldlonlatname.split(',')[0] oldlat=oldlonlatname.split(',')[1] oldncfile = NetCDFFile(filename,'r') varinfile(oldncfile, filename, errormsg, 'old', oldlon) varinfile(oldncfile, filename, errormsg, 'old', oldlat) newlon=newlonlatname.split(',')[0] newlat=newlonlatname.split(',')[1] newncfile = NetCDFFile(newprojectionfile,'r') varinfile(newncfile, newprojectionfile, errormsg, 'old', newlon) varinfile(newncfile, newprojectionfile, errormsg, 'old', newlat) oldlon2d, oldlat2d = lonlat2D(oldncfile, oldlon, oldlat) newlon2d, newlat2d = lonlat2D(newncfile, newlon, newlat) newdimx = newlon2d.shape[1] newdimy = newlon2d.shape[0] olddimx = oldlon2d.shape[1] olddimy = oldlon2d.shape[0] # pairs old-->new ## mappairs = np.ones((newdimy, newdimx, 2), dtype=int) mappairs = mappairs*fillValue print ' ' + fname + ' searching nearest neighbor...' for jnew in range(newdimy): for inew in range(newdimx): if ((jnew*newdimx + inew)%100 == 0): print ' ' + fname + ' found: ', '{0:5.2f}'.format(float(jnew*newdimx + inew)*100/(newdimx*newdimy*1)), '%' masklonoldpt1 = oldlon2d >= newlon2d[jnew, inew] - float(ddegL) masklonoldpt2 = oldlon2d <= newlon2d[jnew, inew] + float(ddegL) masklonoldpt = masklonoldpt1 * masklonoldpt2 masklatoldpt1 = oldlat2d >= newlat2d[jnew, inew] - float(ddegl) masklatoldpt2 = oldlat2d <= newlat2d[jnew, inew] + float(ddegl) masklatoldpt = masklatoldpt1 * masklatoldpt2 Nmasklon = np.sum(masklonoldpt) Nmasklat = np.sum(masklatoldpt) if Nmasklon > 0 and Nmasklat > 0: ## print newlon2d[jnew, inew], newlat2d[jnew, inew], ' ### ', np.min(oldlon2d), np.max(oldlon2d), ':', np.min(oldlat2d), np.max(oldlat2d) ## print masklonoldpt.shape, masklatoldpt.shape, Nmasklon, Nmasklat maskindlon = indices_mask(masklonoldpt) maskindlat = indices_mask(masklatoldpt) mindist=1000000. # for jold in range(olddimy): # for iold in range(olddimx): # dist=np.sqrt((oldlon2d[jold, iold] - newlon2d[jnew, inew])**2. + \ # (oldlat2d[jold, iold] - newlat2d[jnew, inew])**2.) for jold in range(Nmasklat): for iold in range(Nmasklon): if maskindlon[iold,0] == maskindlat[jold,0] and maskindlon[iold,1] == maskindlat[jold,1]: dist=np.sqrt((newlon2d[jnew, inew] - oldlon2d[maskindlon[iold,0], maskindlon[iold,1]])**2. + \ (newlat2d[jnew, inew] - oldlat2d[maskindlat[jold,0], maskindlat[jold,1]])**2.) ## print jold, iold, ':' , maskindlon[iold,:], '-->', maskindlat[jold,:], '**** L:', \ ## oldlon2d[maskindlon[iold,0], maskindlon[iold,1]], 'l', oldlat2d[maskindlat[jold,0], maskindlat[jold,1]], \ ## 'dist:',dist, 'mindist: ', mindist if dist < mindist: mindist = dist ## print jold, iold, 'dist: ', dist, 'L', maskindlon[iold,:], 'l', maskindlat[jold,:] mappairs[jnew, inew, 0] = maskindlon[iold,1] mappairs[jnew, inew, 1] = maskindlat[jold,0] ## quit() varobj = oldncfile.variables[varn] varinf = variable_inf(varobj) # Creation of the new file ## ncfnew = NetCDFFile(ofile,'w') for idim in range(varinf.Ndims): newdimv = oldncfile.dimensions[varinf.dimns[idim]] if dimensions[varinf.dimns[idim]] == oldlon or dimensions[varinf.dimns[idim]] == oldlat: if dimensions[varinf.dimns[idim]] == oldlon: ncfnew.createDimension(varinf.dimns[idim], newdimx) else: ncfnew.createDimension(varinf.dimns[idim], newdimy) else: if newdimv.isunlimited(): ncfnew.createDimension(varinf.dimns[idim], None) else: ncfnew.createDimension(varinf.dimns[idim], varinf.dims[idim]) newvar = ncfnew.createVariable(varn, varinf.dtype, varinf.dimns, fill_value=fillValue) for attrn in varinf.attributes: if not attrn == '_FillValue': attr = newvar.setncattr(attrn, varobj.getncattr(attrn)) attr = newvar.setncattr('mask', 'variable remapped at nearest neighbors using ' + values.split(':')[0] + ' using ' + \ ' as longitude and latitude valules ') ncfnew.sync() ncfnew.close() for varns in ncf.variables: if not varns == varn: fvaradd(filename + ':' + varns, ofile) elif varns == oldlon: fvaradd(newprojectionfile + ':' + newlon, ofile) elif varns == oldlat: fvaradd(newprojectionfile + ':' + newlat, ofile) chvarname(oldlon, ofile, newlon) chvarname(oldlat, ofile, newlat) ncfnew = NetCDFFile(ofile,'a') # looping on all the variable ## varsecdims = [] Nvarloop=1 for insdim in range(varinf.Ndims - 2): Nvarloop = Nvarloop * varinf.dims[insdim] varsecdims.append(0) Nsecdims = len(varsecdims) for iloop in range(Nvarloop): varval = load_ncvariable_lastdims(oldncfile, varn, varsecdims, 2) for jnew in range(newdimy): for inew in range(newdimx): if ((jnew*newdimx + inew)%100 == 0): print ' ' + fname + ': ', '{0:5.2f}'.format(float((jnew*newdimx + inew)*100./(newdimx*newdimy))), '% done' newvarval[jnew, inew] = varval[mappairs[jnew, inew, 0], mappairs[jnew, inew, 1]] fill_ncvariable_lastdims(ncfnew,varn,varsecdims,2,newvarval) varsecdims[Nsecdims -1] = varsecdims[Nsecdims - 1] + 1 if varsecdims[Nsecdims - 1] > varinf.dims[Nsecdims - 1] - 1: varsecdims[Nsecdims - 1] = 0 for jdim in range(Nsecdims - 1): varsecdims[Nsecdims - 2 - jdim] = varsecdims[Nsecdims - 2 - jdim] + 1 if varsecdims[Nsecdims - 2 - jdim] <= varinf.dims[Nsecdims - 2 - jdim] - 1: break else: varsecdims[Nsecdims - 2 - jdim] = 0 ncfnew.sync() ncfnew.close() print 'remapnn: new file "' + ofile + '" with remapped variable written !!' #remapnn('data/R1_CCRC_NARCliM_MOM_1950-2009_pracc.nc:lon,lat:lon,lat', 'data/AWAP_pre_1950-2009_pre_mask.nc', 'pre', 1. , 1.) #quit() def checkallvars(values, filen): """ Function to check all variables of a file values=[dimn1],[[dimn2],...,[dimnN]]:[dim1],[[dim2],...,[dimN]] [dimn1/N]: name of dimension [dim1/N]: value for the given dimension [0-N]: dimension value -1: last value -2: half of the dimension size -3: all values filen=name of the netCDF file to check """ fname = "'checkallvars'" dimns = values.split(':')[0].split(',') dims0 = np.array(values.split(':')[1].split(',')) Ndims = len(dimns) dims = np.zeros( (Ndims), dtype=np.int) dimvals = np.zeros( (Ndims), dtype=np.int) ncobj = NetCDFFile(filen,'r') for idim in range(Ndims): dims[idim] = np.int(dims0[idim]) if not ncobj.dimensions.has_key(dimns[idim]): print errormsg print ' ' + fname + " file '" + filen + "' does not have dimension '" + dimns[idim] + "' !!!!" quit(-1) sizedim = len(ncobj.dimensions[dimns[idim]]) if dims[idim] == -3: dimvals[idim] = -1 elif dims[idim] == -2: dimvals[idim] = sizedim/2 elif dims[idim] == -1: dimvals[idim] = sizedim-1 else: dimvals[idim] = dims[idim] print " dimension '" + dimns[idim] + "' of size: ",sizedim,' shown at :',\ dimvals[idim] ivar = 1 for varn in ncobj.variables: varobj = ncobj.variables[varn] varinf = variable_inf(varobj) dimsvar = np.zeros( (varinf.Ndims), dtype=np.int) dimsvar[:] = -9 for ivardim in range(varinf.Ndims): vardimn = varinf.dimns[ivardim] for idim in range(Ndims): if vardimn == dimns[idim]: dimsvar[ivardim] = dimvals[idim] if dimsvar[ivardim] == -9: print errormsg print ' ' + fname + " No value given to the dimension '" + vardimn +\ "' !!" quit(-1) print ivar, ':', varn + ' with: ', varinf.dimns,' outputted as: ',dimsvar,' ___________________' if varinf.Ndims == 1: if dimsvar == -1: varvals = varobj[:] else: varvals = varobj[dimsvar] elif varinf.Ndims == 2: if dimsvar[0] == -1 and dimsvar[1] == -1: varvals = varobj[:] elif dimsvar[0] == -1 and dimsvar[1] != -1: varvals = varobj[:,dimsvar[1]] elif dimsvar[0] != -1 and dimsvar[1] == -1: varvals = varobj[dimsvar[0],:] else: varvals = varobj[dimsvar[0],dimsvar[1]] elif varinf.Ndims == 3: if dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] == -1: varvals = varobj[:] elif dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] != -1: varvals = varobj[:,:,dimsvar[2]] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] == -1: varvals = varobj[:,dimsvar[1],:] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] == -1: varvals = varobj[dimsvar[0],:,:] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] != -1: varvals = varobj[:,dimsvar[1],dimsvar[2]] elif dimsvar[0] != -1 and dimsvar[1] != -1 and dimsvar[2] == -1: varvals = varobj[dimsvar[0],dimsvar[1],:] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] != -1: varvals = varobj[dimsvar[0],:,dimsvar[2]] else: varvals = varobj[dimsvar[0],dimsvar[1],dimsvar[2]] elif varinf.Ndims == 4: if dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] == -1 and dimsvar[3] == -1: varvals = varobj[:] elif dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] == -1 and dimsvar[3] != -1: varvals = varobj[:,:,:,dimsvar[2]] elif dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] != -1 and dimsvar[3] == -1: varvals = varobj[:,:,dimsvar[2],:] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] == -1 and dimsvar[3] == -1: varvals = varobj[:,dimsvar[1],:,:] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] == -1 and dimsvar[3] == -1: varvals = varobj[dimsvar[0],:,:,:] elif dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] != -1 and dimsvar[3] != -1: varvals = varobj[:,:,dimsvar[2],dimsvar[3]] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] == -1 and dimsvar[3] != -1: varvals = varobj[:,dimsvar[1],:,dimsvar[3]] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] == -1 and dimsvar[3] != -1: varvals = varobj[dimsvar[1],:,:,dimsvar[3]] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] != -1 and dimsvar[3] == -1: varvals = varobj[:,dimsvar[1],dimsvar[2],:] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] != -1 and dimsvar[3] == -1: varvals = varobj[dimsvar[0],:,dimsvar[2],:] elif dimsvar[0] != -1 and dimsvar[1] != -1 and dimsvar[2] == -1 and dimsvar[3] == -1: varvals = varobj[dimsvar[0],dimsvar[1],:,:] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] != -1 and dimsvar[3] != -1: varvals = varobj[:,dimsvar[1],dimsvar[2],dimsvar[3]] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] != -1 and dimsvar[3] != -1: varvals = varobj[dimsvar[0],:,dimsvar[2],dimsvar[3]] elif dimsvar[0] != -1 and dimsvar[1] != -1 and dimsvar[2] == -1 and dimsvar[3] != -1: varvals = varobj[dimsvar[0],dimsvar[1],:,dimsvar[3]] elif dimsvar[0] != -1 and dimsvar[1] != -1 and dimsvar[2] != -1 and dimsvar[3] == -1: varvals = varobj[dimsvar[0],dimsvar[1],dimsvar[2],:] else: varvals = varobj[dimsvar[0],dimsvar[1],dimsvar[2],dimvals[3]] else: print errormsg print ' ' + fname + ' number of dimensions ',varinf.Ndims,' not ready!!' quit(-1) print varvals ivar = ivar + 1 ncobj.close() def give_fix1dim_values(mat,iddim,dimval): """ Function to return matrix values when one dimension has been removed at a given value mat = matrix iddim = dimesion to remove (starting from 0 tacking the right most) dimval = value of the dimension >>> matrix = np.array( (range(27)), dtype=np.float).reshape(3,3,3) give_fix1dim_values(matrix,2,2) [[[ 0. 1. 2.] [ 3. 4. 5.] [ 6. 7. 8.]] [[ 9. 10. 11.] [ 12. 13. 14.] [ 15. 16. 17.]]] """ fname='give_fix1dim_values' # Not working ma.squeeze ## import numpy.ma as ma ## print ma.squeeze(mat, axis = (idim,)) matshape = mat.shape mattype = mat.dtype matmask = np.ones( tuple(matshape), dtype=bool) zerodims=np.ones( (6), dtype=int) Ndims = len(matshape) if Ndims > 6: print errmsg print ' ' + fname + ' number of dimensions: ',Ndims,' not ready!!!!!' quit(-1) zerodims[5-Ndims+1:6] = matshape newdims = list(matshape) newdims[Ndims-1-iddim] = matshape[iddim]-1 mask = np.ones(tuple(zerodims), dtype=bool) vals = np.zeros(tuple(zerodims), dtype=mattype) vals[0:zerodims[0],0:zerodims[1],0:zerodims[2],0:zerodims[3],0:zerodims[4], 0:zerodims[5]] = mat for i5 in range(zerodims[5]): for i4 in range(zerodims[4]): for i3 in range(zerodims[3]): for i2 in range(zerodims[2]): for i1 in range(zerodims[1]): for i0 in range(zerodims[0]): # print 'iddim:',iddim,'dimval:',dimval,'-->',i0,i1,i2,i3,i4,i5 if iddim == 5 and dimval == i0: mask[i0,i1,i2,i3,i4,i5] = False elif iddim == 4 and dimval == i1: mask[i0,i1,i2,i3,i4,i5] = False elif iddim == 3 and dimval == i2: mask[i0,i1,i2,i3,i4,i5] = False elif iddim == 2 and dimval == i3: mask[i0,i1,i2,i3,i4,i5] = False elif iddim == 1 and dimval == i4: mask[i0,i1,i2,i3,i4,i5] = False elif iddim == 0 and dimval == i5: mask[i0,i1,i2,i3,i4,i5] = False newmat = vals[mask].reshape(tuple(newdims)) # print 'newmat dim:',iddim,'val:',dimval,':',newdims,'______' # print newmat return newmat def portion_fix1dim_values(mat,iddim,dimval): """ Function to return that portion of the matrix values when one dimension has been removed at a given value mat = matrix iddim = dimesion to remove (starting from 0 tacking the right most) dimval = value of the dimension >>> matrix = np.array( (range(27)), dtype=np.float).reshape(3,3,3) portion_fix1dim_values(matrix,2,2) [[ 18. 19. 20.] [ 21. 22. 23.] [ 24. 25. 26.]] """ fname='portion_fix1dim_values' # Not working ma.squeeze ## import numpy.ma as ma ## print ma.squeeze(mat, axis = (idim,)) matshape = mat.shape mattype = mat.dtype matmask = np.zeros( tuple(matshape), dtype=bool) zerodims=np.ones( (6), dtype=int) Ndims = len(matshape) if Ndims > 6: print errmsg print ' ' + fname + ' number of dimensions: ',Ndims,' not ready!!!!!' quit(-1) zerodims[5-Ndims+1:6] = matshape newdims = [] for idim in range(Ndims): if idim != iddim: newdims.append(matshape[idim]) newdims.reverse() mask = np.zeros(tuple(zerodims), dtype=bool) vals = np.zeros(tuple(zerodims), dtype=mattype) vals[0:zerodims[0],0:zerodims[1],0:zerodims[2],0:zerodims[3],0:zerodims[4], 0:zerodims[5]] = mat for i5 in range(zerodims[5]): for i4 in range(zerodims[4]): for i3 in range(zerodims[3]): for i2 in range(zerodims[2]): for i1 in range(zerodims[1]): for i0 in range(zerodims[0]): # print 'iddim:',iddim,'dimval:',dimval,'-->',i0,i1,i2,i3,i4,i5 if iddim == 5 and dimval == i0: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 4 and dimval == i1: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 3 and dimval == i2: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 2 and dimval == i3: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 1 and dimval == i4: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 0 and dimval == i5: mask[i0,i1,i2,i3,i4,i5] = True # print 'newmat dim:',iddim,'val:',dimval,':',newdims,'______' newmat = vals[mask].reshape(tuple(newdims)) # print newmat return newmat def valmod_dim(values, ncfile, varn): """ Function to modify the value of a variable at a given dimension and value values = dimn,dimval,modins,modval1,[modval2,...] dimn = name of the dimension dimval = value of the dimension at which change the values modins = instruction: 'sumc', add [modval1] 'subc', substraction [modval1] 'mulc', multiply by [modval1] 'divc', divide by [modval1] 'lowthres': modify all values below [modval1] to [modval2] 'upthres': modify all values above [modval1] to [modval2] ncfile = netCDF file name varn = name of the variable >>> valmod_dim('num_metgrid_levels,10,mulc,-1','aqua_met-em_control.nc','PRES') """ fname = 'valmod_dim' if values == '-h': print """ Function to modify the value of a variable at a given dimension and value values = dimn,dimval,modins,modval1,[modval2,...] dimn = name of the dimension dimval = value of the dimension at which change the values modins = instruction: 'sumc', add [modval1] 'subc', substraction [modval1] 'mulc', multiply by [modval1] 'divc', divide by [modval1] 'lowthres': modify all values below [modval1] to [modval2] 'upthres': modify all values above [modval1] to [modval2] ncfile = netCDF file name varn = name of the variable >>> valmod_dim('num_metgrid_levels,10,mulc,-1','aqua_met-em_control.nc','PRES') """ quit() ## mat = np.array( (range(27)), dtype=np.float).reshape(3,3,3) vals = values.split(',') Lvals = len(vals) dimn = vals[0] dimval = np.int(vals[1]) modins = vals[2] modval = np.float(vals[3]) newvalues = ','.join(vals[2:Lvals]) if not os.path.isfile(ncfile): print errormsg print ' valmod: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.dimensions.has_key(dimn): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not have dimension "' + \ dimn + '" !!!!' ncf.close() quit(-1) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not have variable "' + \ varn + '" !!!!' ncf.close() quit(-1) varobj = ncf.variables[varn] varinf = variable_inf(varobj) varvals = varobj[:] # Dimension id iddim=varinf.Ndims - 1 for idimn in varinf.dimns: # print iddim, idimn if dimn == idimn: break iddim = iddim - 1 matshape = varvals.shape mattype = varvals.dtype matmask = np.zeros( tuple(matshape), dtype=bool) zerodims=np.ones( (6), dtype=int) Ndims = len(matshape) if Ndims > 6: print errmsg print ' ' + fname + ' number of dimensions: ',Ndims,' not ready!!!!!' quit(-1) zerodims[5-Ndims+1:6] = matshape newdims = [] for idim in range(Ndims): # print idim,'matshape[idim]:',matshape[Ndims-1-idim],'iddim:',iddim if idim != iddim: newdims.append(matshape[Ndims-1-idim]) newdims.reverse() mask = np.zeros(tuple(zerodims), dtype=bool) vals = np.zeros(tuple(zerodims), dtype=mattype) newvals = np.zeros(tuple(zerodims), dtype=mattype) vals[0:zerodims[0],0:zerodims[1],0:zerodims[2],0:zerodims[3],0:zerodims[4], \ 0:zerodims[5]] = varvals # print 'vals shape:',vals.shape,':',newdims for i5 in range(zerodims[5]): for i4 in range(zerodims[4]): for i3 in range(zerodims[3]): for i2 in range(zerodims[2]): for i1 in range(zerodims[1]): for i0 in range(zerodims[0]): # print 'iddim:',iddim,'dimval:',dimval,'-->',i0,i1,i2,i3,i4,i5 if iddim == 5 and dimval == i0: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 4 and dimval == i1: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 3 and dimval == i2: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 2 and dimval == i3: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 1 and dimval == i4: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 0 and dimval == i5: mask[i0,i1,i2,i3,i4,i5] = True # print 'newmat dim:',iddim,'val:',dimval,':',newdims,'______' newmat = vals[mask].reshape(tuple(newdims)) varmod = np.zeros(tuple(newdims), dtype=mattype) varshape = newmat.shape vartype = newmat.dtype Ndims = len(varshape) if Ndims <= 2: varmod = valmodoper(newmat, newvalues) elif Ndims == 3: for i in range(varshape[0]): varmod[i,:,:] = valmodoper(newmat[i,:,:], newvalues) elif Ndims == 4: for i in range(varshape[0]): for j in range(varshape[1]): varmod[i,j,:,:] = valmodoper(newmat[i,j,:,:], newvalues) elif Ndims == 5: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): varmod[i,j,k,:,:] = valmodoper(newmat[i,j,k,:,:], newvalues) newvals = np.where( mask, varmod, vals) zerdims = np.where( zerodims == 1, 0, zerodims ) finalvals = newvals[0:zerodims[0],0:zerodims[1],0:zerodims[2],0:zerodims[3],\ 0:zerodims[4],0:zerodims[5]] finalvals = np.squeeze(finalvals) if len(varobj.shape) != len(finalvals.shape): print ' Modified values have a different shape:', finalvals.shape, \ 'than variable:', varobj.shape print ' Assuming that extra dimensions are at the beggining' newfinalvals = finalvals.reshape(varobj.shape) varobj[:] = newfinalvals[:] else: varobj[:] = finalvals[:] ncf.sync() ncf.close() def checkNaNs(values, filen): """ Function to check for NaN values over all variables in a file values = absolute lower threshold for NaN value filen=name of the netCDF file to check """ fname = "'checkNaNs'" valthres = np.float(values) ncobj = NetCDFFile(filen,'r') S1val = np.dtype('a1') Sval = np.dtype(str) Uval = np.dtype(unicode) ivar = 1 print fname + ' checking with: ',valthres,' in _______' for varn in ncobj.variables: varobj = ncobj.variables[varn] varinf = variable_inf(varobj) varvals = varobj[:] var1 = varvals.flatten()[0] if varinf.dtype != type('c') and varinf.dtype != type('Str') and \ varinf.dtype != Sval.type and varinf.dtype != Uval.type and \ varinf.dtype != S1val.type and type(var1) != S1val.type and \ type(var1) != Sval.type and type(var1) != Uval.type: print ' ' + varn + ':',varinf.dims found=0 if varinf.Ndims == 1: for i0 in range(varinf.dims[0]): if abs(varvals[i0]) > valthres or np.isnan(varvals[i0]): print ' ' + fname + ': NaN value found on variable "' + \ varn + '" at: ',i0,' value: ',varvals[i0] break elif varinf.Ndims == 2: for i1 in range(varinf.dims[0]): for i0 in range(varinf.dims[1]): if abs(varvals[i1,i0]) > valthres or np.isnan(varvals[i1,i0]): print ' ' + fname + ': NaN value found on variable "' + \ varn + '" at: ',i1,', ',i0,' value: ',varvals[i1,i0] found=1 break if found == 1: break elif varinf.Ndims == 3: for i2 in range(varinf.dims[0]): for i1 in range(varinf.dims[1]): for i0 in range(varinf.dims[2]): if abs(varvals[i2,i1,i0]) > valthres or \ np.isnan(varvals[i2,i1,i0]): print ' ' + fname + \ ': NaN value found on variable "' + varn + \ '" at: ', i2,', ',i1,', ',i0,' value: ', \ varvals[i2,i1,i0] found=1 break if found == 1: break if found == 1: break elif varinf.Ndims == 4: for i3 in range(varinf.dims[0]): for i2 in range(varinf.dims[1]): for i1 in range(varinf.dims[2]): for i0 in range(varinf.dims[3]): if abs(varvals[i3,i2,i1,i0]) > valthres or \ np.isnan(varvals[i3,i2,i1,i0]): print ' ' + fname + \ ': NaN value found on variable "' + varn + \ '" at: ', i3,', ', i2,', ',i1,', ',i0, \ ' value: ',varvals[i3,i2,i1,i0] found=1 break if found == 1: break if found == 1: break if found == 1: break elif varinf.Ndims == 5: for i4 in range(varinf.dims[0]): for i3 in range(varinf.dims[1]): for i2 in range(varinf.dims[2]): for i1 in range(varinf.dims[3]): for i0 in range(varinf.dims[4]): if abs(varvals[i4,i3,i2,i1,i0]) > valthres or \ np.isnan(varvals[i4,i3,i2,i1,i0]): print ' ' + fname + \ ': NaN value found on variable "' + varn + \ '" at: ', i4,', ', i3,', ', i2,', ',i1, \ ', ',i0,' value: ',varvals[i4,i3,i2,i1,i0] found=1 break if found == 1: break if found == 1: break if found == 1: break if found == 1: break else: print errormsg print ' ' + fname + ' number of dimensions ',varinf.Ndims, \ ' not ready!!' quit(-1) ivar = ivar + 1 ncobj.close() ##checkNaNs(10.e10, '/d4/lflmd/etudes/WRF_LMDZ/WaquaL/WRF/control/wrfout_d01_1979-12-01_00:00:00') ##checkNaNs(10.e10, '/d4/lflmd/etudes/FF/tests/em_quarter_ss/run/wrfout_d01_0001-01-01_00:00:00') def checkAllValues(values, filen): """ Function to check for variables with along all their dimensions with the same value in a file values = value to check filen=name of the netCDF file to check """ fname = "'checkAllValues'" valcheck = np.float(values) ncobj = NetCDFFile(filen,'r') S1val = np.dtype('a1') Sval = np.dtype(str) Uval = np.dtype(unicode) ivar = 1 print fname + ' checking with: ',valcheck,' in _______' for varn in ncobj.variables: varobj = ncobj.variables[varn] varinf = variable_inf(varobj) varvals = varobj[:] var1 = varvals.flatten()[0] if varinf.dtype != type('c') and varinf.dtype != type('Str') and \ varinf.dtype != Sval.type and varinf.dtype != Uval.type and \ varinf.dtype != S1val.type and type(var1) != S1val.type and \ type(var1) != Sval.type and type(var1) != Uval.type: print ' ' + varn + ':',varinf.dims if np.all(varvals == valcheck): print ' ' + fname + ': variable "' + varn + \ '" with all its values == ',valcheck ivar = ivar + 1 ncobj.close() ##checkAllValues(0., '/d4/lflmd/etudes/WRF_LMDZ/WaquaL/WRF/control/wrfout_d01_1979-12-01_00:00:00') ##checkAllValues(0., '/d4/lflmd/etudes/FF/tests/em_quarter_ss/run/wrfout_d01_0001-01-01_00:00:00') ##checkAllValues(0., '/home/lluis/etudes/WRF_LMDZ/tests/wrf_input/AR40.0/wrfout_d01_1979-01-01_00:00:00') def DataSetSection(values, filen): """ Function to get a section (values along a dimension) of a given data-set values= [dimn],[beg],[end],[int] [dimn]: name of the dimension [beg],[end],[int]: beginning, end and interval along the dimension-axis [end] = -1, values until the end filen= netCDF with the data-set DataSetSection('time,3,15,1','test.nc') """ fname = 'DataSetSection' if values == 'h': print fname + '_____________________________________________________________' print DataSetSection.__doc__ quit() dimn = values.split(',')[0] begv = np.int(values.split(',')[1]) endv = np.int(values.split(',')[2]) intv = np.int(values.split(',')[3]) ofile=filen.split('.')[0] + '_' + dimn + '_B' + str(begv) + '-E' + str(endv) + \ '-I' + str(intv) + '.nc' nciobj = NetCDFFile(filen,'r') ncoobj = NetCDFFile(ofile,'w') # Creating dimensions ## for dims in nciobj.dimensions: objdim = nciobj.dimensions[dims] if objdim.isunlimited(): dimsize = None else: if dims == dimn: if begv > len(objdim): print errormsg print ' ' + fname + ': beginning value' ,begv, \ 'is larger than dimension (', len(objdim), ')!!!!' quit(-1) if endv > len(objdim): print errormsg print ' ' + fname + ': endining value' ,endv, \ 'is larger than dimension (', len(objdim), ')!!!!' quit(-1) if endv == -1: endv = len(objdim) dimsize = (endv - begv + 1)/intv - 1 else: dimsize = len(objdim) print ' ' + fname + ': adding dimension: ' + dims + ' size:',dimsize dim = ncoobj.createDimension(dims, dimsize) ncoobj.sync() # Creating variables ## for varns in nciobj.variables: print ' ' + fname + ': adding variable "' + varns + '"...' varobj = nciobj.variables[varns] vardims = varobj.dimensions if not searchInlist(list(vardims),dimn): varvals = varobj[:] else: varslice = [] for dimname in varobj.dimensions: if dimname == dimn: varslice.append(slice(begv,endv,intv)) else: Ldim = len(nciobj.dimensions[dimname]) varslice.append(slice(0,Ldim)) varvals = varobj[tuple(varslice)] # Adding fill value attribute ## varattrs = varobj.ncattrs() if searchInlist(varattrs, '_FillValue'): varfil = varobj._FillValue else: varfil = False vartype = varobj.dtype newvar = ncoobj.createVariable(varns, vartype, vardims, fill_value=varfil) newvar[:] = varvals for attrs in varattrs: if not attrs == '_FillValue': attrv = varobj.getncattr(attrs) attr = set_attribute(newvar, attrs, attrv) # Global attributes ## for attrs in nciobj.ncattrs(): attrv = nciobj.getncattr(attrs) attr = set_attribute(ncoobj, attrs, attrv) nciobj.close() ncoobj.sync() ncoobj.close() print ' ' + fname + ' succesfull creation of file "' + ofile + '" !!!' return def DataSetSection_multidims(values, filen): """ Function to get a section (values along multiple dimensions) of a given data-set values= [dimn1],[beg1],[end1],[int1]@[...[[dimnM],[begM],[endM],[intM]]] [dimni]: name of the dimension [begi],[endi],[inti]: beginning, end and interval along the dimension-axis [endi] = -1, last value NOTE: dimensions without values are taken allover their size filen= netCDF with the data-set DataSetSection_multidims('Time,-1,-1,1@bottom_top,6,6,1','wrfout_d01_1979-12-01_00:00:00') """ fname = 'DataSetSection_multidims' if values == 'h': print fname + '_____________________________________________________________' print DataSetSection_multidims.__doc__ quit() Ndims = len(values.split('@')) dimns = [] ofiletile = '' begvs = np.zeros((Ndims), dtype=int) endvs = np.zeros((Ndims), dtype=int) intvs = np.zeros((Ndims), dtype=int) for idim in range(Ndims): val = values.split('@')[idim] dimns.append(val.split(',')[0]) begvs[idim] = int(val.split(',')[1]) endvs[idim] = int(val.split(',')[2]) intvs[idim] = int(val.split(',')[3]) ofiletile = ofiletile + '_' + dimns[idim] + '_B' + str(begvs[idim]) + '-E' + \ str(endvs[idim]) + '-I' + str(intvs[idim]) ofile = ofile=filen.split('.')[0] + ofiletile + '.nc' nciobj = NetCDFFile(filen,'r') ncoobj = NetCDFFile(ofile,'w') # Creating dimensions ## print ' ' + fname + ': adding dimensions...' newdimsizes = {} dimslices = {} varslice = [] for dims in nciobj.dimensions: objdim = nciobj.dimensions[dims] dimsize = -1 if not searchInlist(dimns, dims): if objdim.isunlimited(): dimsize = None else: dimsize = len(objdim) varslice.append(slice(0,len(objdim))) else: dimid = dimns.index(dims) if begvs[dimid] > len(objdim): print errormsg print ' ' + fname + ': beginning value' ,begvs[dimid], \ 'is larger than dimension (', len(objdim), ')!!!!' quit(-1) if endvs[dimid] > len(objdim): print errormsg print ' ' + fname + ': endining value' ,endvs[dimid], \ 'is larger than dimension (', len(objdim), ')!!!!' quit(-1) if begvs[dimid] != endvs[dimid]: dimsize = [(endvs[dimid]-begvs[dimid]+1)/intvs[dimid]-1] dimslices[dims] = [begvs[dimid],endvs[dimid]+1,intvs[dimid]] else: dimsize = 1 if begvs[dimid] == -1: dimslices[dims] = [len(objdim)-1,len(objdim),1] else: dimslices[dims] = [begvs[dimid],begvs[dimid]+1,1] newdimsizes[dims] = dimsize if dimsize != -1: if len(objdim) == 1 or dimsize != 1: print ' ' + fname + ': adding dimension: '+ dims +' size:',dimsize if objdim.isunlimited(): dimsize=None dim = ncoobj.createDimension(dims, dimsize) ncoobj.sync() print ' ' + fname + ': Slice according to dimension selection: ',dimslices # Creating variables ## print ' ' + fname + ': adding variables...' for varns in nciobj.variables: # print ' ' + fname + ': adding variable "' + varns + '"...' varobj = nciobj.variables[varns] varorigdims = varobj.dimensions vardims = [] # From: http://stackoverflow.com/questions/1388818/how-can-i-compare-two-lists-in-python-and-return-matches coinc = list(set(varorigdims) & set(dimns)) if len(coinc) == 0: varvals = varobj[:] vardims = varorigdims else: varslice = [] for dimname in varorigdims: if searchInlist(dimns, dimname): varslice.append(slice(dimslices[dimname][0], \ dimslices[dimname][1], dimslices[dimname][2])) if newdimsizes[dimname] != 1: vardims.append(dimname) else: Ldim = len(nciobj.dimensions[dimname]) varslice.append(slice(0,Ldim)) vardims.append(dimname) varvals = varobj[tuple(varslice)] # Adding fill value attribute ## # print ' final variable dimensions:',vardims varattrs = varobj.ncattrs() if searchInlist(varattrs, '_FillValue'): varfil = varobj._FillValue else: varfil = False vartype = varobj.dtype newvar = ncoobj.createVariable(varns, vartype, tuple(vardims), \ fill_value=varfil) # print 'newvar:',newvar.shape,'varvals:',varvals.shape newvar[:] = varvals for attrs in varattrs: if not attrs == '_FillValue': attrv = varobj.getncattr(attrs) attr = set_attribute(newvar, attrs, attrv) # Global attributes ## for attrs in nciobj.ncattrs(): attrv = nciobj.getncattr(attrs) attr = set_attribute(ncoobj, attrs, attrv) slicevalS = '' for dimid in range(Ndims): dn = dimns[dimid] if dimslices[dn][2] is not None: slicevalS = slicevalS + dn + ' (' + str(dimslices[dn][0]) + ',' + \ str(dimslices[dn][1]) + ',' + str(dimslices[dn][2]) + '); ' else: slicevalS = slicevalS + dn + ' (' + str(dimslices[dn][0]) + ',' + \ str(dimslices[dn][1]) + ',1); ' attr = set_attribute(ncoobj, 'sliced_dimensions', slicevalS) nciobj.close() ncoobj.sync() ncoobj.close() print ' ' + fname + ' succesfull creation of file "' + ofile + '" !!!' return #DataSetSection_multidims('Time,-1,-1,1@bottom_top,6,6,1','wrfout_d01_1979-12-01_00:00:00') #DataSetSection_multidims('bottom_top,6,6,1@south_north,3,3,1@west_east,26,26,1','wrfout_d01_1979-12-01_00:00:00') def sellonlatbox(values, ncfile, varn): """ Function to select a lotlan box from a data-set values= [lonName],[latName],[lonSW],[latSW],[lonNE],[latNE] [lonName]: name of the variable with the longitudes [latName]: name of the variable with the latitudes [lonSW],[latSW]: longitude and latitude of the SW vertex of the box [lonNE],[latNE]: longitude and latitude of the NE vertex of the box ncfile= netCDF file varn= name of the variable """ fname = 'sellonlatbox' ofile = 'sellonlatbox_' + varn + '.nc' if values == 'h': print fname + '_____________________________________________________________' print sellonlatbox.__doc__ quit() lonn = values.split(',')[0] latn = values.split(',')[1] lonSW = np.float(values.split(',')[2]) latSW = np.float(values.split(',')[3]) lonNE = np.float(values.split(',')[4]) latNE = np.float(values.split(',')[5]) objfile = NetCDFFile(ncfile, 'r') lonobj = objfile.variables[lonn] latobj = objfile.variables[latn] lonv = lonobj[:] latv = latobj[:] if not searchInlist(objfile.variables, varn): print errormsg print ' ' + fname + ': variable name "' + varn + '" is not in file "' + \ ncfile + '" !!!!!' quit(-1) varobj = objfile.variables[varn] Ndimslon = len(lonobj.shape) Ndimslat = len(latobj.shape) Ndimsvar = len(varobj.shape) # Looking for coincidence of dimensions samedim = [] for idv in range(Ndimsvar): for idl in range(Ndimslon): if varobj.dimensions[idv] == lonobj.dimensions[idl]: if not searchInlist(samedim,varobj.dimensions[idv]): samedim.append(varobj.dimensions[idv]) break for idl in range(Ndimslat): if varobj.dimensions[idv] == latobj.dimensions[idl]: if not searchInlist(samedim,varobj.dimensions[idv]): samedim.append(varobj.dimensions[idv]) break Ndimshare = len(samedim) print 'variable and lon/lat matrices share ', Ndimshare,' dimensions: ',samedim samedimslonlat = True for idL in range(Ndimslon): for idl in range(Ndimslat): if lonobj.dimensions[idl] != latobj.dimensions[idl]: samedimslonlat = False break if not samedimslonlat: break # Creation of the lon/lat matrices to search if Ndimshare == 1: lonmat = lonobj[:] latmat = latobj[:] elif Ndimshare == 2: if samedimslonlat: lonmat = lonobj[:] latmat = latobj[:] else: if Ndimslon != 1 and Ndimslat != 1: print errormsg print ' ' + fname + ': I do not know how to keep going!' print ' lon ', Ndimslon,' and lat ',Ndimslat, \ ' matrices do not share the same dimensions, and do not have shape 1' quit(-1) else: lonmat = np.zeros((latobj.shape[0], lonobj.shape[0]), dtype=np.float) latmat = np.zeros((latobj.shape[0], lonobj.shape[0]), dtype=np.float) for i in range(lonobj.shape[0]): latmat[:,i] = latobj[:] for j in range(latobj.shape[0]): lonmat[j,:] = lonobj[:] elif Ndimshare == 3: if samedimslonlat: lonmat = lonobj[0,:,:] latmat = latobj[0,:,:] else: print errormsg print ' ' + fname + ': dimension sharing of lon/lat not ready!' quit(-1) # Searching for inside points iind = 0 if Ndimshare == 1: inside = {} for iL in range(lonobj.shape[0]): if lonobj[iL] >= lonSW and lonobj[iL] <= lonNE and latobj[iL] >= latSW \ and latobj[iL] <= latNE: inside[iind] = iL iind = iind + 1 elif Ndimshare == 2: newidx = [] newidy = [] if (len(lonobj.shape) == 3): inside = np.zeros((lonobj.shape[1], lonobj.shape[2]), dtype=bool) else: inside = np.zeros((lonobj.shape), dtype=bool) for iL in range(lonobj.shape[1]): for il in range(lonobj.shape[0]): if lonobj[il,iL] >= lonSW and lonobj[il,iL] <= lonNE and \ latobj[il,iL] >= latSW and latobj[il,iL] <= latNE: if not searchInlist(newidx, iL): newidx.append(iL) if not searchInlist(newidy, il): newidy.append(il) inside[il, iL] = True iind = iind + 1 elif Ndimshare == 3: newidx = [] newidy = [] inside = np.zeros((lonobj.shape[1], lonobj.shape[2]), dtype=bool) for iL in range(lonobj.shape[2]): for il in range(lonobj.shape[1]): if lonobj[0,il,iL] >= lonSW and lonobj[0,il,iL] <= lonNE and \ latobj[0,il,iL] >= latSW and latobj[0,il,iL] <= latNE: if not searchInlist(newidx, iL): newidx.append(iL) if not searchInlist(newidy, il): newidy.append(il) inside[il, iL] = True iind = iind + 1 Ninpts = len(inside) newdx = len(newidx) newdy = len(newidy) print ' ' + fname +': ',Ninpts,' pts found in the box: ',lonSW,',',latSW,' x ', \ lonNE,',',latNE # Creation of cropped matrices inlon = np.zeros((Ninpts), dtype=np.float) inlat = np.zeros((Ninpts), dtype=np.float) invar = np.zeros((varobj.shape[0],Ninpts), dtype=np.float) # Creation of the netCDF file ## objofile = NetCDFFile(ofile, 'w') if Ndimshare == 1: # Dimensions newdim = objofile.createDimension('boxpt', Ninpts) newdim = objofile.createDimension('time', None) # var dimensions newvar = objofile.createVariable('lon', 'f8', ('boxpt')) newattr = basicvardef(newvar, 'lon', 'longitude', 'degrees west_east') newvar[:] = lonobj[inside[iin]] newvar = objofile.createVariable('lat', 'f8', ('boxpt')) newattr = basicvardef(newvar, 'lat', 'latitude', 'degrees north_south') newvar[:] = latobj[inside[iin]] newvar = objofile.createVariable('time', 'f8', ('time')) if objfile.variables.has_key('time'): otime = objfile.variables['time'] timevals = otime[:] if searchInlist(otime.ncattrs(),'units'): tunits = otime.getncattr['units'] else: tunits = 'unkown' else: timevals = np.arange(varobj.shape[0]) tunits = 'unkown' newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = timevals # variable newvar = objofile.createVariable(varn, 'f4', ('time', 'boxpt')) newvar[:] = varobj[invar] else: # Dimensions newdim = objofile.createDimension('x', newdx) newdim = objofile.createDimension('y', newdy) newdim = objofile.createDimension('time', None) # var dimensions newvar = objofile.createVariable('lon', 'f8', ('y', 'x')) if Ndimshare == 2: Vals = lonobj[:] else: Vals = lonobj[0,:,:] for it in range(varobj.shape[0]): newvar[:] = Vals[inside].reshape(newdy,newdx) newattr = basicvardef(newvar, 'lon', 'longitude', 'degrees west_east') newvar = objofile.createVariable('lat', 'f8', ('y', 'x')) newattr = basicvardef(newvar, 'lat', 'latitude', 'degrees north_south') if Ndimshare == 2: Vals = latobj[:] else: Vals = latobj[0,:,:] for it in range(varobj.shape[0]): newvar[:] = Vals[inside].reshape(newdy,newdx) newvar = objofile.createVariable('time', 'f8', ('time')) if objfile.variables.has_key('time'): otime = objfile.variables['time'] timevals = otime[:] if searchInlist(otime.ncattrs(),'units'): tunits = otime.getncattr['units'] else: tunits = 'unkown' else: timevals = np.arange(varobj.shape[0]) tunits = 'unkown' newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = timevals # variable newvar = objofile.createVariable(varn, 'f4', ('time', 'y', 'x')) for it in range(varobj.shape[0]): valsvar = varobj[it,:,:] newvar[it,:,:] = valsvar[inside] if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = varn if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = varn if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = 'unkown' newattr = basicvardef(newvar, vsname, vlname, vunits) # global attributes objofile.setncattr('author', 'L. Fita') objofile.setncattr('institution', 'Laboratire de Meteorologie Dynamique') objofile.setncattr('university', 'Pierre Marie Curie - Jussieu') objofile.setncattr('center', 'Centre National de Recherches Scientifiques') objofile.setncattr('city', 'Paris') objofile.setncattr('country', 'France') objofile.setncattr('script', 'nc_var_tools.py') objofile.setncattr('function', 'sellonlatbox') objofile.setncattr('version', '1.0') objfile.close() objofile.sync() objofile.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return #sellonlatbox('h', '/san0/lflmd/DATA/SAFRAN/ForcPRCP_france_SAFRAN_8Km_1hour_1996080100_1997073123_V1_01.nc', 'RR') #sellonlatbox('lon,lat,2.,42.5,5.,45.', '/home/lluis/DATA/SAFRAN/safran_RR_1996.nc', 'RR') #sellonlatbox('lon,lat,2.,42.5,5.,45.', '/san0/lflmd/DATA/SAFRAN/ForcPRCP_france_SAFRAN_8Km_1hour_1996080100_1997073123_V1_01.nc', 'product') #sellonlatbox('XLONG,XLAT,2.,42.5,5.,45.', '/san0/lflmd/etudes/WRF_LMDZ/WL_HyMeX/Cevennes96/control/wrfout/wrfout_d01_1996-09-17_00:00:00', 'RAINC') def compute_deaccum(values, ncfile, varname): """ Function to compute deaccum: deaccumulation of a variable (VAR[t+1]-VAR[t]) values= [timedimname] name of the time dimension ncfile= netCDF file to use varname= variable to deaccumulate """ fname='compute_deaccum' ofile = 'deaccumulated_' + varname + '.nc' if values == 'h': print fname + '_____________________________________________________________' print compute_deacccum.__doc__ quit() objnc = NetCDFFile(ncfile, 'r') if not objnc.variables.has_key(varname): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varname + '" !!!!' quit(-1) var1obj = objnc.variables[varname] vardims = var1obj.dimensions Ndimvar = len(vardims) vdim = [] deacdims = [] # Searcing for time dimension for idd in range(Ndimvar): vdim.append(slice(0,var1obj.shape[idd])) deacdims.append(var1obj.shape[idd]) if vardims[idd] == values: dimtid = idd dimt = var1obj.shape[dimtid] print 'Number of time-steps: ',dimt,' id dimt: ',dimtid deacdims[dimtid] = dimt varvals = np.zeros(tuple(deacdims), dtype=np.float) # Slicing for the deaccumulation for it in range(dimt): vals1 = list(vdim) vals2 = list(vdim) vals1[dimtid] = it if it > 0: vals2[dimtid] = it - 1 else: vals2[dimtid] = it var1 = var1obj[tuple(vals1)] var2 = var1obj[tuple(vals2)] varvals[tuple(vals1)] = var1-var2 # Creation of the new file objofile = NetCDFFile(ofile, 'w') # Dimensions fdims = objnc.dimensions for dimn in fdims: dimobj = objnc.dimensions[dimn] if dimobj.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(dimobj) newdim = objofile.createDimension(dimn, dimsize) # Variables fvars = objnc.variables for varn in fvars: varobj = objnc.variables[varn] if varn == varname: newvarn = 'deaccum' + varn else: newvarn = varn newvar = objofile.createVariable(newvarn, varobj.dtype, varobj.dimensions) for attrn in varobj.ncattrs(): attrval = varobj.getncattr(attrn) newattr = newvar.setncattr(attrn, attrval) if varn == varname: newvar[:] = varvals else: newvar[:] = varobj[:] # Global attributes for attrn in objnc.ncattrs(): attrval = objnc.getncattr(attrn) newattr = objofile.setncattr(attrn, attrval) objnc.close() objofile.sync() objofile.close() print ' ' + fname + ' success written of "' + ofile + '" !!!!!' return #compute_deaccum('time', 'control/sellonlatbox_PRECIP_dayaccum.nc', 'PRECIP') def compute_opersvarsfiles(values, varname): """ Function to compute opersvarfiles: operation of variables from different files (OPER1.FILE1_VAR1 OPER2.FILE2_VAR2), operations are going to be secuentially made values= [dimvons]@[operfilevars] [dimnvons]: [dimo1]|[dimo2]|[...|[dimoN]] '|' seaprated list of names with the variables which contain the values of the dimensions of the compute variable operfilevars: [oper1]|[file1]|[varN],[...,[operK]|[fileM]|[varN]] ',' separated list of triplets of [operation], [file], [variable name to use] * [oper]: operations: add,sub,mul,div,pot varname= name to the final variable """ # import numpy.ma as ma fname='compute_opersvarsfiles' if values == 'h': print fname + '_____________________________________________________________' print compute_opersvarsfiles.__doc__ quit() dimovars = values.split('@')[0].split('|') filevars = values.split('@')[1].split(',') ofile = 'opersvarsfiles_' + varname + '.nc' Nfilevars = len(filevars) operation = 'operation: ' newunits = '-' opervars = [] for ifv in range(Nfilevars): opern = filevars[ifv].split('|')[0] filen = filevars[ifv].split('|')[1] varn = filevars[ifv].split('|')[2] opervars.append(varn) if not os.path.isfile(filen): print errormsg print ' ' + fname + ' file: "' + filen + '" does not exist !!' quit(-1) objnc = NetCDFFile(filen, 'r') if not objnc.variables.has_key(varn): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varn + '" !!!!' quit(-1) varobj = objnc.variables[varn] varvals = varobj[:] # Creation of the operated variable if ifv == 0: vardims = varobj.dimensions vartype = varobj.dtype if opern == 'add' or opern == 'sub': newvarv = np.zeros(tuple(varobj.shape), dtype=np.float) elif opern == 'mul' or opern == 'div' or opern == 'pot': newvarv = np.ones(tuple(varobj.shape), dtype=np.float) else: print errormsg print ' ' + fname + ': operation "' + opern + '" is not ready !!!!' quit(-1) if searchInlist(varobj.ncattrs(), '_FillValue'): NOvalue = varobj.getncattr('_FillValue') print warnmsg print ' ' + fname + ': variable with a missing value:',NOvalue if opern == 'add': if searchInlist(varobj.ncattrs(), '_FillValue'): prevals = varvals.filled(0.) newvarv = newvarv + prevals else: newvarv = newvarv + varvals operation = operation + ' +' + varn newunits = variables_values(varname)[5] elif opern == 'sub': if searchInlist(varobj.ncattrs(), '_FillValue'): prevals = varvals.filled(0.) newvarv = newvarv - prevals else: newvarv = newvarv - varvals operation = operation + ' -' + varn newunits = variables_values(varname)[5] elif opern == 'mul': if searchInlist(varobj.ncattrs(), '_FillValue'): prevals = varvals.filled(1.) newvarv = newvarv * prevals else: newvarv = newvarv * varvals operation = operation + ' *' + varn newunits = variables_values(varname)[5] elif opern == 'div': if searchInlist(varobj.ncattrs(), '_FillValue'): prevals = varvals.filled(1.) newvarv = newvarv / prevals else: newvarv = newvarv / varvals operation = operation + ' /' + varn newunits = variables_values(varname)[5] elif opern == 'pot': if searchInlist(varobj.ncattrs(), '_FillValue'): prevals = varvals.filled(1.) newvarv = newvarv ** prevals else: newvarv = newvarv ** varvals operation = operation + ' **' + varn newunits = variables_values(varname)[5] # Creation of the new file objofile = NetCDFFile(ofile, 'w') # Dimensions for dimn in vardims: dimobj = objnc.dimensions[dimn] if dimobj.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(dimobj) newdim = objofile.createDimension(dimn, dimsize) # Variables for dimensions for varn in dimovars: varobj = objnc.variables[varn] newvar = objofile.createVariable(varn, varobj.dtype, varobj.dimensions) for attrn in varobj.ncattrs(): attrval = varobj.getncattr(attrn) newattr = newvar.setncattr(attrn, attrval) newvar[:] = varobj[:] newvar = objofile.createVariable(varname, vartype, vardims) newvar[:] = newvarv newattr = basicvardef(newvar, varname, operation, newunits) # Global attributes for attrn in objnc.ncattrs(): attrval = objnc.getncattr(attrn) newattr = objofile.setncattr(attrn, attrval) objnc.close() objofile.sync() objofile.close() print ' ' + fname + ' success written of "' + ofile + '" !!!!!' return #compute_opersvarsfiles('add|control/sellonlatbox_RAINC.nc|RAINC,add|control/sellonlatbox_RAINNC.nc|RAINNC', 'PRECIP') def compute_opervartimes(values, ncfile, varname): """ Function to compute opervartimes: operation of variable for a given sub-set of time-steps (OPER1.VAR[IT,ET]), operations are going to be secuentially made values= [oper]|[timen]|[intT] dimension [oper]: operations: add,sub,mul,div,pot [timen]: name of the dimension time [intT]: number time-steps to accumulate ncfile= netCDF file to use varname= variable to use """ fname='compute_opervartimes' ofile = 'opervartimes_' + varname + '.nc' if values == 'h': print fname + '_____________________________________________________________' print opervartimes.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ' file: "' + ncfile + '" does not exist !!' quit(-1) objnc = NetCDFFile(ncfile, 'r') if not objnc.variables.has_key(varname): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varname + '" !!!!' quit(-1) opern = values.split('|')[0] timen = values.split('|')[1] intT = int(values.split('|')[2]) varobj = objnc.variables[varname] vardims = varobj.dimensions vartype = varobj.dtype Ndims = len(vardims) newdimns = vardims newdims = np.zeros((Ndims), dtype=int) newslice = [] newunits = '-' idim = 0 for dimn in vardims: if dimn == timen: dimtid = idim newslice.append(0) origdimt = varobj.shape[dimtid] newdims[idim] = len(np.arange(origdimt)[intT/2::intT]) else: newdims[idim] = varobj.shape[idim] newslice.append(slice(0,varobj.shape[idim])) idim = idim + 1 print 'dimtid: ',dimtid if opern == 'add' or opern == 'sub': newvarv = np.zeros(tuple(newdims), dtype=np.float) elif opern == 'mul' or opern == 'div' or opern == 'pot': newvarv = np.ones(tuple(newdims), dtype=np.float) else: print errormsg print ' ' + fname + ': operation "' + opern + '" is not ready !!!!' quit(-1) # Creation of the accumulated operated variable idint = 0 itt = 0 nint = 0 for it in range(1,origdimt): tslice = list(newslice) tslice[dimtid] = it tnewvar = list(newslice) tnewvar[dimtid] = nint if opern == 'add': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] + varobj[tuple(tslice)] if searchInlist(varobj.ncattrs(), 'units') and newunits != '-': newunits = varobj.getncattr('units') elif opern == 'sub': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] - varobj[tuple(tslice)] if searchInlist(varobj.ncattrs(), 'units') and newunits != '-': newunits = varobj.getncattr('units') elif opern == 'mul': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] * varobj[tuple(tslice)] elif opern == 'div': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] / varobj[tuple(tslice)] elif opern == 'pot': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] **varobj[tuple(tslice)] it = it + 1 itt = itt + 1 if itt == intT: nint = nint + 1 itt = 0 # Creation of the new file objofile = NetCDFFile(ofile, 'w') # Dimensions for dimn in newdimns: dimobj = objnc.dimensions[dimn] if dimobj.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(dimobj) if dimn == timen: if np.mod(varobj.shape[dimtid],intT) != 0: dimsize = varobj.shape[dimtid]/intT + 1 newdim = objofile.createDimension(dimn, dimsize) newdim = objofile.createDimension('bnds', 2) # Variables fvars = objnc.variables for varn in fvars: if varn != varname: varobj = objnc.variables[varn] if varn != timen: if not searchInlist(varobj.dimensions,timen): newvar = objofile.createVariable(varn, varobj.dtype, \ varobj.dimensions) for attrn in varobj.ncattrs(): attrval = varobj.getncattr(attrn) newattr = newvar.setncattr(attrn, attrval) newvar[:] = varobj[:] else: newvar = objofile.createVariable(varn, varobj.dtype, ('time')) basicvardef(newvar, 'time', 'time steps','-') times = np.arange(origdimt)[intT/2::intT] newvar[:] = times newdimt = newvar.shape[0] newattr = newvar.setncattr('bounds','time_bnds') newvar = objofile.createVariable('time_bnds', 'i4', ('time', 'bnds')) basicvardef(newvar, 'time_bnds', 'time steps bnds of accumulation','-') for it in range(newdimt): newvar[it,0] = (it - 1)*intT newvar[it,1] = (it)*intT-1 newvar = objofile.createVariable('ac' + varname, vartype, vardims) newvar[:] = newvarv newattr = basicvardef(newvar, 'ac' + varname, 'accumulated ' + opern + \ ' values' , newunits) # Global attributes for attrn in objnc.ncattrs(): attrval = objnc.getncattr(attrn) newattr = objofile.setncattr(attrn, attrval) objnc.close() objofile.sync() objofile.close() print ' ' + fname + ' success written of "' + ofile + '" !!!!!' return #compute_opervartimes('add|time|24', 'obs/PRCP.nc', 'product') # def compute_opervaralltime(values, ncfile, varname): """ Function to compute opervaralltime: operation of variable successible allover the time-steps (OPER1.VAR), operations are going to be secuentially made values= [oper]|[timen] dimension [oper]: operations: add,sub,mul,div,pot [timen]: name of the dimension time ncfile= netCDF file to use varname= variable to use """ fname='compute_opervarlltime' ofile = 'opervaralltime_' + varname + '.nc' if values == 'h': print fname + '_____________________________________________________________' print compute_opervaralltime.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ' file: "' + ncfile + '" does not exist !!' quit(-1) objnc = NetCDFFile(ncfile, 'r') if not objnc.variables.has_key(varname): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varname + '" !!!!' quit(-1) opern = values.split('|')[0] timen = values.split('|')[1] varobj = objnc.variables[varname] vardims = varobj.dimensions vartype = varobj.dtype Ndims = len(vardims) newdimns = vardims newdims = np.zeros((Ndims), dtype=int) newslice = [] prevslice = [] newunits = '-' idim = 0 for dimn in vardims: if dimn == timen: dimtid = idim newslice.append(0) origdimt = varobj.shape[dimtid] newdims[idim] = origdimt else: newdims[idim] = varobj.shape[idim] newslice.append(slice(0,varobj.shape[idim])) idim = idim + 1 print 'dimtid: ',dimtid if opern == 'add' or opern == 'sub': newvarv = np.zeros(tuple(newdims), dtype=np.float) elif opern == 'mul' or opern == 'div' or opern == 'pot': newvarv = np.ones(tuple(newdims), dtype=np.float) else: print errormsg print ' ' + fname + ': operation "' + opern + '" is not ready !!!!' quit(-1) # Creation of the accumulated operated variable idint = 0 for it in range(1,origdimt): tslice = list(newslice) tslice[dimtid] = it prevslice = list(newslice) prevslice[dimtid] = it - 1 if opern == 'add': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] + varobj[tuple(tslice)] if searchInlist(varobj.ncattrs(), 'units') and newunits != '-': newunits = varobj.getncattr('units') elif opern == 'sub': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] - varobj[tuple(tslice)] if searchInlist(varobj.ncattrs(), 'units') and newunits != '-': newunits = varobj.getncattr('units') elif opern == 'mul': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] * varobj[tuple(tslice)] elif opern == 'div': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] / varobj[tuple(tslice)] elif opern == 'pot': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] **varobj[tuple(tslice)] it = it + 1 # Creation of the new file objofile = NetCDFFile(ofile, 'w') # Dimensions for dimn in newdimns: dimobj = objnc.dimensions[dimn] if dimobj.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(dimobj) newdim = objofile.createDimension(dimn, dimsize) newdim = objofile.createDimension('bnds', 2) # Variables fvars = objnc.variables for varn in fvars: if varn != varname: varobj = objnc.variables[varn] if varn != timen: if not searchInlist(varobj.dimensions,timen): newvar = objofile.createVariable(varn, varobj.dtype, \ varobj.dimensions) for attrn in varobj.ncattrs(): attrval = varobj.getncattr(attrn) newattr = newvar.setncattr(attrn, attrval) newvar[:] = varobj[:] else: newvar = objofile.createVariable(varn, varobj.dtype, ('time')) basicvardef(newvar, 'time', 'time steps','-') timevals = varobj[:] newvar[:] = timevals newdimt = newvar.shape[0] newattr = newvar.setncattr('bounds','time_bnds') newvar = objofile.createVariable('time_bnds', 'i4', ('bnds')) basicvardef(newvar, 'time_bnds', 'time steps bnds of accumulation','-') newvar[0] = timevals[0] newvar[1] = timevals[origdimt-1] newvar = objofile.createVariable('ac' + varname, vartype, vardims) newvar[:] = newvarv newattr = basicvardef(newvar, 'ac' + varname, 'accumulated ' + opern + \ ' values' , newunits) # Global attributes for attrn in objnc.ncattrs(): attrval = objnc.getncattr(attrn) newattr = objofile.setncattr(attrn, attrval) objnc.close() objofile.sync() objofile.close() print ' ' + fname + ' success written of "' + ofile + '" !!!!!' return #compute_opervaralltime('add|time', 'obs/PRCP.nc', 'product') def retype(val, vtype): """ Function to transform a value to a given type retype(val, vtype) [val]= value [vtype]= type to transform >>> retype(0, type(True)) False """ fname = 'retype' if val == 'h': print fname + '_____________________________________________________________' print retype.__doc__ quit() if vtype == type(int(1)): newval = int(val) elif vtype == type(float(1)): newval = float(val) # elif vtype == type(float32(1)): # newval = float32(val) elif vtype == type(np.int(1)): newval = np.int(val) elif vtype == type(np.int16(1)): newval = np.int16(val) elif vtype == type(np.int32(1)): newval = np.int32(val) elif vtype == type(np.int64(1)): newval = np.int64(val) elif vtype == type(np.float(1)): newval = np.float(val) elif vtype == type(np.float16(1)): newval = np.float16(val) elif vtype == type(np.float32(1)): newval = np.float32(val) elif vtype == type(np.float64(1)): newval = np.float64(val) elif vtype == type(True): if val == 0: newval = False else: newval = True elif vtype == '|S1': newval = char(val) else: print errormsg print ' ' + fname + ': variable type "', vtype, '" not ready!!!' quit(-1) return newval def setvar_asciivalues(values, ncfile, varname): """ Function to set de values of a variable with an ASCII file (common Fortran-like format) values= asciifile: ASCII file with a Fortran-like structure [dim1]_1 [[dim2]_1 ... [dim2]_M] ... [dim1]_N ncfile= netCDF file to use varname= variable to use """ fname='setvar_asciivalues' if values == 'h': print fname + '_____________________________________________________________' print setvar_asciivalues.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ' file: "' + ncfile + '" does not exist !!' quit(-1) if not os.path.isfile(values): print errormsg print ' ' + fname + ' ASCII file: "' + values + '" does not exist !!' quit(-1) objnc = NetCDFFile(ncfile, 'a') if not objnc.variables.has_key(varname): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varname + '" !!!!' quit(-1) objvar = objnc.variables[varname] vardims = objvar.shape vartype = objvar.dtype Ndims = len(vardims) objasciif = open(values, 'r') values = np.zeros((objvar.shape), dtype=vartype) it = 0 if Ndims == 1: valfinal = np.zeros((1), dtype=vartype) for line in objasciif: val = (line.replace('\n','').replace(' ','').replace('\t','')) valfinal[0] = retype(val, vartype) if it <= values.shape[0]-1: values[it] = retype(valfinal[0], vartype) it = it +1 elif Ndims == 2: iline=0 for line in objasciif: vals = line.replace('\n','').replace('\t','').split(' ') if len(vals) != values.shape[1]: print errormsg print ' ' + fname + ': given: ',len(vals),' but variable reaquires: ',values.shape[1],'!!!' exit(-1) for i1 in range(len(vals)): values[iline,i1] = retype(vals[i1], vartype) iline=iline+1 else: print errormsg print ' ' + fname + ': number of dimensions',Ndims,'not ready !!!!' quit(-1) objasciif.close() objvar[:] = values objnc.sync() objnc.close() #setvar_asciivalues('obstimes.inf', 'obs/re_project_accumulated-1h_PRCP.nc', 'time') class stats_space2D(object): """spatial statistics for a 2D file: vals= variable ro check (assuming var[t,dy,dx]) self.minv[t]: spatial minimum at each time-step self.maxv[t]: spatial maximum at each time-step self.meanv[t]: spatial mean at each time-step self.mean2v[t]: spatial quadratic mean at each time-step self.stdv[t]: spatial standard deviation at each time-step self.anomv[t]: spatial anomaly from the whole mean at each time-step """ def __init__(self, vals): from scipy import stats as sts fname = 'stats_space2D' if vals1 == 'h': print fname + '_____________________________________________________________' print stats_space2Dvars.__doc__ quit() if vals1 is None: self.minv = None self.maxv = None self.meanv = None self.mean2v = None self.stdv = None else: dimt = vals.shape[0] stats=np.zeros((dimt,5), dtype=np.float) absmean = np.mean(vals) for it in range(dimt): stats[it,0]=np.min(vals[it,:,:]) stats[it,1]=np.max(vals[it,:,:]) stats[it,2]=np.mean(vals[it,:,:]) stats[it,3]=np.mean(vals[it,:,:]*vals[it,:,:]) stats[it,4]=absmean - stats[it,2] stats = np.where(stats > 0.1*fillValue, fillValue, stats) stats = np.where(stats is nan, fillValue, stats) stats = np.where(stats is np.inf, fillValue, stats) stats = np.where(stats is None, fillValue, stats) self.minv=stats[:,0] self.maxv=stats[:,1] self.meanv=stats[:,2] self.mean2v=stats[:,3] self.stdv=np.sqrt(stats[:,3]-stats[:,2]*stats[:,2]) self.anomv=stats[:,4] class stats_space2Dvars(object): """spatial statistics beween 2 2D files: valsA = variable 1 (assuming var[t,dy,dx]) valsB = variable 2 (assuming var[t,dy,dx]) self.min[t,var], self.max[t,var], self.mean[t,var], self.mean2[t,var], self.std[t,var] [var] = var1+var2[v1Av2], var1-var2[v1Sv2], var1/var2[v1Dv2], var1*var2[v1Pv2] self.mae=mean[t,abs(var1-var2)] self.correlation=correlation[var1,var2] (and p-value) self.bias=[t,mean(var1)-mean(var2)] self.meancorrelation=correlation[mean(var1),mean(var2)] (and p-value) """ def __init__(self, vals1, vals2): from scipy import stats as sts fname = 'stats_space2Dvars' if vals1 == 'h': print fname + '_____________________________________________________________' print stats_space2Dvars.__doc__ quit() if vals1 is None: self.minv1Av2 = None self.maxv1Av2 = None self.meanv1Av2 = None self.mean2v1Av2 = None self.stdv1Av2 = None self.minv1Sv2 = None self.maxv1Sv2 = None self.meanv1Sv2 = None self.mean2v1Sv2 = None self.stdv1Sv2 = None self.minv1Dv2 = None self.maxv1Dv2 = None self.meanv1Dv2 = None self.mean2v1Dv2 = None self.stdv1Dv2 = None self.minv1Pv2 = None self.maxv1Pv2 = None self.meanv1Pv2 = None self.mean2v1Pv2 = None self.stdv1Pv2 = None self.mae = None self.corr = None else: dimt = vals1.shape[0] stats=np.zeros((dimt,26), dtype=np.float) meanvals1 = np.zeros((dimt), dtype=np.float) meanvals2 = np.zeros((dimt), dtype=np.float) for it in range(dimt): v1 = vals1[it,:,:].flatten() v2 = vals2[it,:,:].flatten() # add vs = v1 + v2 stats[it,0] = np.min(vs) stats[it,1] = np.max(vs) stats[it,2] = np.mean(vs) stats[it,3] = np.mean(vs*vs) stats[it,4] = np.sqrt(stats[it,3] - stats[it,2]*stats[it,2]) # sub stats[it,20] = np.mean(np.abs(v1-v2)) vs = v1 - v2 stats[it,5] = np.min(vs) stats[it,6] = np.max(vs) stats[it,7] = np.mean(vs) stats[it,8] = np.mean(vs*vs) stats[it,9] = np.sqrt(stats[it,8] - stats[it,7]*stats[it,7]) # mul vs = v1 * v2 stats[it,10] = np.min(vs) stats[it,11] = np.max(vs) stats[it,12] = np.mean(vs) stats[it,13] = np.mean(vs*vs) stats[it,14] = np.sqrt(stats[it,13] - stats[it,12]*stats[it,12]) # div vs = v1 / v2 stats[it,15] = np.min(vs) stats[it,16] = np.max(vs) stats[it,17] = np.mean(vs) stats[it,18] = np.mean(vs*vs) stats[it,19] = np.sqrt(stats[it,18] - stats[it,17]*stats[it,17]) # corr stats[it,21], stats[it,22] = mask_pearsonr(v1, v2) # Mean values meanvals1[it] = np.mean(v1) meanvals2[it] = np.mean(v2) stats[it,23] = meanvals1[it] - meanvals2[it] stats = np.where(stats > 0.1*fillValue, fillValue, stats) stats = np.where(stats is np.nan, fillValue, stats) stats = np.where(stats is np.inf, fillValue, stats) stats = np.where(stats is None, fillValue, stats) self.minv1Av2 = stats[:,0] self.maxv1Av2 = stats[:,1] self.meanv1Av2 = stats[:,2] self.meanv21Av2 = stats[:,3] self.stdv1Av2 = stats[:,4] self.minv1Sv2 = stats[:,5] self.maxv1Sv2 = stats[:,6] self.meanv1Sv2 = stats[:,7] self.meanv21Sv2 = stats[:,8] self.stdv1Sv2 = stats[:,9] self.minv1Pv2 = stats[:,10] self.maxv1Pv2 = stats[:,11] self.meanv1Pv2 = stats[:,12] self.meanv21Pv2 = stats[:,13] self.stdv1Pv2 = stats[:,14] self.minv1Dv2 = stats[:,15] self.maxv1Dv2 = stats[:,16] self.meanv1Dv2 = stats[:,17] self.meanv21Dv2 = stats[:,18] self.stdv1Dv2 = stats[:,19] self.mae = stats[:,20] self.corr = stats[:,21] self.p_value = stats[:,22] self.bias = stats[:,23] self.meancorr, self.meanp_value = sts.pearsonr(meanvals1, meanvals2) class stats_time2D(object): """temporal statistics for a 2D file: vals= variable ro check (assuming var[t,dy,dx]) self.minv[t]: temporal minimum at each grid point self.maxv[t]: temporal maximum at each grid point self.meanv[t]: temporal mean at each grid point self.mean2v[t]: temporal quadratic mean at each grid point self.stdv[t]: temporal standard deviation at each grid point self.anomv[t]: temporal anomaly from the whole mean at each grid point """ def __init__(self, vals): from scipy import stats as sts fname = 'stats_time2D' if vals1 == 'h': print fname + '_____________________________________________________________' print stats_time2Dvars.__doc__ quit() if vals1 is None: self.minv = None self.maxv = None self.meanv = None self.mean2v = None self.stdv = None else: dimx = vals.shape[2] dimy = vals.shape[1] stats=np.zeros((dimy,dimx,5), dtype=np.float) absmean = np.mean(vals,axis=0) stats[:,:,0]=np.min(vals, axis=0) stats[:,:,1]=np.max(vals, axis=0) stats[:,:,2]=np.mean(vals, axis=0) stats[:,:,3]=np.mean(vals*vals, axis=0) stats[:,:,4]=absmean - stats[:,:,2] stats = np.where(stats > 0.1*fillValue, fillValue, stats) stats = np.where(stats is np.nan, fillValue, stats) stats = np.where(stats is np.inf, fillValue, stats) stats = np.where(stats is None, fillValue, stats) self.minv=stats[:,:,0] self.maxv=stats[:,:,1] self.meanv=stats[:,:,2] self.mean2v=stats[:,:,3] self.stdv=np.sqrt(stats[:,:,3]-stats[:,:,2]*stats[:,:,2]) self.anomv=stats[:,:,4] class stats_time2Dvars(object): """temporal statistics beween 2 2D files: valsA = variable 1 (assuming var[t,dy,dx]) valsB = variable 2 (assuming var[t,dy,dx]) self.min[dy,dx,var], self.max[dy,dx,var], self.mean[dy,dx,var], self.mean2[dy,dx,var], self.std[dy,dx,var] [var] = var1+var2[v1Av2], var1-var2[v1Sv2], var1/var2[v1Dv2], var1*var2[v1Pv2] self.mae=mean[dy,dx,abs(var1-var2)] self.correlation=correlation[var1,var2] (and p-value) self.bias=[dy,dx,mean(var1)-mean(var2)] self.meancorrelation=correlation[mean(var1),mean(var2)] (and p-value) """ def __init__(self, vals1, vals2): from scipy import stats as sts fname = 'stats_time2Dvars' if vals1 == 'h': print fname + '_____________________________________________________________' print stats_time2Dvars.__doc__ quit() if vals1 is None: self.minv1Av2 = None self.maxv1Av2 = None self.meanv1Av2 = None self.mean2v1Av2 = None self.stdv1Av2 = None self.minv1Sv2 = None self.maxv1Sv2 = None self.meanv1Sv2 = None self.mean2v1Sv2 = None self.stdv1Sv2 = None self.minv1Dv2 = None self.maxv1Dv2 = None self.meanv1Dv2 = None self.mean2v1Dv2 = None self.stdv1Dv2 = None self.minv1Pv2 = None self.maxv1Pv2 = None self.meanv1Pv2 = None self.mean2v1Pv2 = None self.stdv1Pv2 = None self.mae = None self.corr = None else: dimx = vals1.shape[1] dimy = vals1.shape[1] stats=np.zeros((dimy,dimx,24), dtype=np.float) meanvals1 = np.zeros((dimy,dimx), dtype=np.float) meanvals2 = np.zeros((dimy,dimx), dtype=np.float) # add vs = vals1 + vals2 stats[:,:,0] = np.min(vs,axis=0) stats[:,:,1] = np.max(vs,axis=0) stats[:,:,2] = np.mean(vs,axis=0) stats[:,:,3] = np.mean(vs*vs,axis=0) stats[:,:,4] = np.sqrt(stats[:,:,3] - stats[:,:,2]*stats[:,:,2]) # sub stats[:,:,20] = np.mean(np.abs(vals1-vals2), axis=0) vs = vals1 - vals2 stats[:,:,5] = np.min(vs,axis=0) stats[:,:,6] = np.max(vs,axis=0) stats[:,:,7] = np.mean(vs,axis=0) stats[:,:,8] = np.mean(vs*vs,axis=0) stats[:,:,9] = np.sqrt(stats[:,:,8] - stats[:,:,7]*stats[:,:,7]) # mul vs = vals1 * vals2 stats[:,:,10] = np.min(vs,axis=0) stats[:,:,11] = np.max(vs,axis=0) stats[:,:,12] = np.mean(vs,axis=0) stats[:,:,13] = np.mean(vs*vs,axis=0) stats[:,:,14] = np.sqrt(stats[:,:,13] - stats[:,:,12]*stats[:,:,12]) # div vs = vals1 / vals2 stats[:,:,15] = np.min(vs,axis=0) stats[:,:,16] = np.max(vs,axis=0) stats[:,:,17] = np.mean(vs,axis=0) stats[:,:,18] = np.mean(vs*vs,axis=0) stats[:,:,19] = np.sqrt(stats[:,:,18] - stats[:,:,17]*stats[:,:,17]) # Mean values meanvals1[:,:] = np.mean(vals1,axis=0) meanvals2[:,:] = np.mean(vals2,axis=0) stats[:,:,23] = meanvals1[:,:] - meanvals2[:,:] # corr self.meancorr, self.meanp_value = sts.pearsonr(meanvals1.flatten(), meanvals2.flatten()) for j in range(dimy): for i in range(dimx): stats[j,i,21], stats[j,i,22] = sts.pearsonr(vals1[:,j,i], vals2[:,j,i]) stats = np.where(stats > 0.1*fillValue, fillValue, stats) stats = np.where(stats is np.nan, fillValue, stats) stats = np.where(stats is np.inf, fillValue, stats) stats = np.where(stats is None, fillValue, stats) self.minv1Av2 = stats[:,:,0] self.maxv1Av2 = stats[:,:,1] self.meanv1Av2 = stats[:,:,2] self.meanv21Av2 = stats[:,:,3] self.stdv1Av2 = stats[:,:,4] self.minv1Sv2 = stats[:,:,5] self.maxv1Sv2 = stats[:,:,6] self.meanv1Sv2 = stats[:,:,7] self.meanv21Sv2 = stats[:,:,8] self.stdv1Sv2 = stats[:,:,9] self.minv1Pv2 = stats[:,:,10] self.maxv1Pv2 = stats[:,:,11] self.meanv1Pv2 = stats[:,:,12] self.meanv21Pv2 = stats[:,:,13] self.stdv1Pv2 = stats[:,:,14] self.minv1Dv2 = stats[:,:,15] self.maxv1Dv2 = stats[:,:,16] self.meanv1Dv2 = stats[:,:,17] self.meanv21Dv2 = stats[:,:,18] self.stdv1Dv2 = stats[:,:,19] self.mae = stats[:,:,20] self.corr = stats[:,:,21] self.p_value = stats[:,:,22] self.bias = stats[:,:,23] #vals1 = np.arange(27).reshape(3,3,3)*1. #vals2 = np.arange(1,28).reshape(3,3,3)*1. #printing_class(stats_time2Dvars(vals1,vals2)) def statcompare_files(values): """ Python script to statistically compare two different files values= [file1]:[var1],[file2][var2] [file1/2]: files to compare any diferent would be [file1]-[file2] [var1/2]: variable in file1/2 to compare """ fname = 'statcompare_files' ofile='statcompare_files.nc' if values == 'h': print fname + '_____________________________________________________________' print statcompare_files.__doc__ quit() file1=values.split(',')[0].split(':')[0] var1=values.split(',')[0].split(':')[1] file2=values.split(',')[1].split(':')[0] var2=values.split(',')[1].split(':')[1] if not os.path.isfile(file1): print errormsg print ' ' + fname + ' file: "' + file1 + '" does not exist !!' quit(-1) objnc1 = NetCDFFile(file1, 'r') if not objnc1.variables.has_key(var1): print errormsg print ' ' + fname + ': netCDF file "' + file1 + \ '" does not have variable "' + var1 + '" !!!!' quit(-1) if not os.path.isfile(file2): print errormsg print ' ' + fname + ' file: "' + file2 + '" does not exist !!' quit(-1) objnc2 = NetCDFFile(file2, 'r') if not objnc2.variables.has_key(var2): print errormsg print ' ' + fname + ': netCDF file "' + file2 + \ '" does not have variable "' + var2 + '" !!!!' quit(-1) objvar1 = objnc1.variables[var1] objvar2 = objnc2.variables[var2] Ndim1 = len(objvar1.shape) Ndim2 = len(objvar2.shape) if Ndim1 != Ndim2: print errormsg print ' ' + fname + ' variable: "' + var1 + '" from file "' + file1 + \ '" does not have the same size:' , Ndim1, 'as variable: "' + var2 + \ '" from file "' + file2 + '": ',Ndim2 quit(-1) dims1 = objvar1.shape dims2 = objvar2.shape for iid in range(Ndim1): if dims1[iid] != dims2[iid]: print errormsg print ' ' + fname + ' shape:',iid,'is different between variables!!!' print ' dim_var1:', dims1[iid], 'dim_var2:', dims2[iid] print ' var1 shape:', dims1 print ' var2 shape:', dims2 quit(-1) varvals1 = objvar1[:] varvals2 = objvar2[:] if Ndim1 == 3: timecompare = stats_time2Dvars(varvals1,varvals2) spacecompare = stats_space2Dvars(varvals1,varvals2) dimt = varvals1.shape[0] dimy = varvals1.shape[1] dimx = varvals1.shape[2] # print 'bias std.dev. ___________________________________' # for it in range(dimt): # print it,':',spacecompare.bias[it],spacecompare.stdv1Sv2[it] # print 'global spatial correlation (% significance)______' # for it in range(dimt): # print it, ':', spacecompare.corr[it], '(', \ # (1.-spacecompare.p_value[it])*100.,')' # print 'spatial means temporal correlation:',spacecompare.meancorr,'(', \ # (1.-spacecompare.meanp_value)*100.,' % significance)' # print 'temporal means temporal correlation:',timecompare.meancorr,'(', \ # (1.-timecompare.meanp_value)*100.,' % significance)' # print 'temporal bias map________________________________' # print timecompare.meanv1Sv2 # print 'temporal correlation map_________________________' # print timeecompare.corr else: print errormsg print ' ' + fname + ' numbe of dimensions:' + Ndim1 + ' not ready!!!' quit(-1) # Creation of results file ## objofile = NetCDFFile(ofile, 'w') # Dimensions for dimn in objnc1.dimensions: dimobj = objnc1.dimensions[dimn] if dimobj.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(dimobj) newdim = objofile.createDimension(dimn, dimsize) # Variables fvars = objnc1.variables for varn in fvars: if varn != var1: varobj = objnc1.variables[varn] if varn != var1: newvar = objofile.createVariable(varn, varobj.dtype, \ varobj.dimensions) for attrn in varobj.ncattrs(): attrval = varobj.getncattr(attrn) newattr = newvar.setncattr(attrn, attrval) newvar[:] = varobj[:] # Statistical variables vartype = objvar1.dtype varunits = objvar2.getncattr('units') print 'vartype=',vartype # temporal bias newvar = objofile.createVariable('t_bias_'+var1+'_'+var2, vartype, ('y', 'x'), \ fill_value = fillValue) newvar[:] = timecompare.bias newattr = basicvardef(newvar, 't_bias_'+var1+'_'+var2, 'temporal bias between ' +\ var1 + '&' + var2, varunits) # temporal Standard deviation newvar = objofile.createVariable('t_stddev_'+var1+'_'+var2, vartype, ('y', 'x'), \ fill_value = fillValue) newvar[:] = timecompare.stdv1Sv2 newattr = basicvardef(newvar, 't_stddev_'+var1+'_'+var2, \ 'temporal standard deviation between '+ var1 + '&' + var2, varunits) # space bias newvar = objofile.createVariable('s_bias_'+var1+'_'+var2, vartype, ('time'), \ fill_value = fillValue) newvar[:] = spacecompare.bias newattr = basicvardef(newvar, 's_bias_'+var1+'_'+var2, 'spatial bias between ' + \ var1 + '&' + var2, varunits) # space Standard deviation newvar = objofile.createVariable('s_stddev_'+var1+'_'+var2, vartype, ('time'), \ fill_value = fillValue) newvar[:] = spacecompare.stdv1Sv2 newattr = basicvardef(newvar, 's_stddev_'+var1+'_'+var2, \ 'spatial standard deviation between '+ var1 + '&' + var2, varunits) # t Correlation map newvar = objofile.createVariable('t_corr_'+var1+'_'+var2, vartype, ('y','x'), \ fill_value = fillValue) newvar[:] = timecompare.corr newattr = basicvardef(newvar, 't_corr_'+var1+'_'+var2, \ 'temporal correlation between ' + var1 + '&' + var2, '-') newattr = newvar.setncattr('projection', 'lon lat') # t p-value map newvar = objofile.createVariable('t_p_value_'+var1+'_'+var2, vartype, ('y','x'), \ fill_value = fillValue) newvar[:] = timecompare.p_value newattr = basicvardef(newvar, 't_p_value_'+var1+'_'+var2, \ 'temporal p_value between ' + var1 + '&' + var2, '-') newattr = newvar.setncattr('projection', 'lon lat') # s Correlation map newvar = objofile.createVariable('s_corr_'+var1+'_'+var2, vartype, ('time'), \ fill_value = fillValue) newvar[:] = spacecompare.corr newattr = basicvardef(newvar, 's_corr_'+var1+'_'+var2, \ 'spatial correlation between ' + var1 + '&' + var2, '-') # s p-value map newvar = objofile.createVariable('s_p_value_'+var1+'_'+var2, vartype, ('time'), \ fill_value = fillValue) newvar[:] = spacecompare.p_value newattr = basicvardef(newvar, 's_p_value_'+var1+'_'+var2, \ 'spatial p_value between ' + var1 + '&' + var2, '-') # print 'spatial means temporal correlation:',spacecompare.meancorr,'(', \ # (1.-spacecompare.meanp_value)*100.,' % significance)' # t mean Correlation newvar = objofile.createVariable('t_mean_corr_'+var1+'_'+var2, 'f4') newvar[:] = timecompare.meancorr newattr = basicvardef(newvar, 't_mean_corr_'+var1+'_'+var2, \ 'time mean values correlation between ' + var1 + '&' + var2, '-') # t mean p-value newvar = objofile.createVariable('t_mean_p_value_'+var1+'_'+var2, 'f4') newvar[:] = timecompare.meanp_value newattr = basicvardef(newvar, 't_mean_p_value_'+var1+'_'+var2, \ 'time mean p_value between ' + var1 + '&' + var2, '-') # s mean Correlation newvar = objofile.createVariable('s_mean_corr_'+var1+'_'+var2, 'f4') newvar[:] = spacecompare.meancorr newattr = basicvardef(newvar, 's_mean_corr_'+var1+'_'+var2, \ 'space mean values correlation between ' + var1 + '&' + var2, '-') # s mean p-value newvar = objofile.createVariable('s_mean_p_value_'+var1+'_'+var2, 'f4') newvar[:] = spacecompare.meanp_value newattr = basicvardef(newvar, 's_mean_p_value_'+var1+'_'+var2, \ 'space mean p_value between ' + var1 + '&' + var2, '-') # Global attributes for attrn in objnc1.ncattrs(): attrval = objnc1.getncattr(attrn) newattr = objofile.setncattr(attrn, attrval) newattr = objofile.setncattr('statisitcs', 'variables retrieved from files ' + \ file1 + ' & ' + file2) objnc1.close() objnc2.close() objofile.sync() objofile.close() varsfillvalue = ['t_bias_', 't_stddev_', 's_bias_', 's_stddev_', 't_corr_', \ 't_p_value', 's_corr_', 's_p_value'] # for varn in varsfillvalue: # print varn # fill = varaddattrk('_FillValue|'+ str(fillValue) + '|R32', ofile, \ # varn+var1+'_'+var2) print ' ' + fname + ': successfull generation of file "' + ofile + '" !!!!!' return #statcompare_files('control/sellonlatbox_wss_17-20.nc:wss,obs/re_project_Vu_17-20.nc:wss') def sellonlatlevbox(values, ncfile, varn): """ Function to select a lotlan box and a given level from a data-set values= [lonName],[latName],[lonSW],[latSW],[lonNE],[latNE],[levi],[levf] [lonName]: name of the variable with the longitudes [latName]: name of the variable with the latitudes [levName]: name of the variable with the value of the levels [lonSW],[latSW]: longitude and latitude of the SW vertex of the box [lonNE],[latNE]: longitude and latitude of the NE vertex of the box [levI],[levF]: range of levels to retrieve ncfile= netCDF file varn= name of the variable """ fname = 'sellonlatlevbox' ofile = 'sellonlatlevbox_' + varn + '.nc' if values == 'h': print fname + '_____________________________________________________________' print sellonlatlevbox.__doc__ quit() lonn = values.split(',')[0] latn = values.split(',')[1] levn = values.split(',')[2] lonSW = np.float(values.split(',')[3]) latSW = np.float(values.split(',')[4]) lonNE = np.float(values.split(',')[5]) latNE = np.float(values.split(',')[6]) levI = int(values.split(',')[7]) levF = int(values.split(',')[8]) objfile = NetCDFFile(ncfile, 'r') lonobj = objfile.variables[lonn] latobj = objfile.variables[latn] levobj = objfile.variables[levn] lonv = lonobj[:] latv = latobj[:] levv = levobj[:] if not searchInlist(objfile.variables, varn): print errormsg print ' ' + fname + ': variable name "' + varn + '" is not in file "' + \ ncfile + '" !!!!!' quit(-1) varobj = objfile.variables[varn] Ndimslon = len(lonobj.shape) Ndimslat = len(latobj.shape) Ndimslev = len(levobj.shape) Ndimsvar = len(varobj.shape) # Looking for coincidence of dimensions samedim = [] for idv in range(Ndimsvar): for idl in range(Ndimslon): if varobj.dimensions[idv] == lonobj.dimensions[idl]: if not searchInlist(samedim,varobj.dimensions[idv]): samedim.append(varobj.dimensions[idv]) break for idl in range(Ndimslat): if varobj.dimensions[idv] == latobj.dimensions[idl]: if not searchInlist(samedim,varobj.dimensions[idv]): samedim.append(varobj.dimensions[idv]) break Ndimshare = len(samedim) print 'variable and lon/lat matrices share ', Ndimshare,' dimensions: ',samedim samedimslonlat = True for idL in range(Ndimslon): for idl in range(Ndimslat): if lonobj.dimensions[idl] != latobj.dimensions[idl]: samedimslonlat = False break if not samedimslonlat: break # Creation of the lon/lat matrices to search if Ndimshare == 1: lonmat = lonobj[:] latmat = latobj[:] elif Ndimshare == 2: if samedimslonlat: lonmat = lonobj[:] latmat = latobj[:] else: if Ndimslon != 1 and Ndimslat != 1: print errormsg print ' ' + fname + ': I do not know how to keep going!' print ' lon ', Ndimslon,' and lat ',Ndimslat, \ ' matrices do not share the same dimensions, and do not have shape 1' quit(-1) else: lonmat = np.zeros((latobj.shape[0], lonobj.shape[0]), dtype=np.float) latmat = np.zeros((latobj.shape[0], lonobj.shape[0]), dtype=np.float) for i in range(lonobj.shape[0]): latmat[:,i] = latobj[:] for j in range(latobj.shape[0]): lonmat[j,:] = lonobj[:] elif Ndimshare == 3: if samedimslonlat: lonmat = lonobj[0,:,:] latmat = latobj[0,:,:] else: print errormsg print ' ' + fname + ': dimension sharing of lon/lat not ready!' quit(-1) # Searching for inside points iind = 0 if Ndimshare == 1: inside = {} for iL in range(lonobj.shape[0]): if lonobj[iL] >= lonSW and lonobj[iL] <= lonNE and latobj[iL] >= latSW \ and latobj[iL] <= latNE: inside[iind] = iL iind = iind + 1 elif Ndimshare == 2: newidx = [] newidy = [] newidz = [] if (len(lonobj.shape) == 3): inside3D = np.zeros((varobj.shape[1], lonobj.shape[1], lonobj.shape[2]), dtype=bool) inside2D = np.zeros((lonobj.shape[1], lonobj.shape[2]), dtype=bool) insideZ = np.zeros((varobj.shape[1]), dtype=bool) else: inside3D = np.zeros((varobj.shape[1], lonobj.shape), dtype=bool) inside2D = np.zeros((lonobj.shape), dtype=bool) insideZ = np.zeros((varobj.shape[1]), dtype=bool) for iL in range(lonobj.shape[1]): for il in range(lonobj.shape[0]): if lonobj[il,iL] >= lonSW and lonobj[il,iL] <= lonNE and \ latobj[il,iL] >= latSW and latobj[il,iL] <= latNE: inside2D[il, iL] = True if not searchInlist(newidx, iL): newidx.append(iL) if not searchInlist(newidy, il): newidy.append(il) for iz in range(Ndimslev): if iz >= levI and iz <= levF: if not searchInlist(newidz, iz): newidz.append(iz) inside3D[iz,il, iL] = True insideZ[iz] = True iind = iind + 1 elif Ndimshare == 3: newidx = [] newidy = [] newidz = [] inside3D = np.zeros((varobj.shape[1], lonobj.shape[1], lonobj.shape[2]), dtype=bool) inside2D = np.zeros((lonobj.shape[1], lonobj.shape[2]), dtype=bool) insideZ = np.zeros((varobj.shape[1]), dtype=bool) for iL in range(lonobj.shape[2]): for il in range(lonobj.shape[1]): if lonobj[0,il,iL] >= lonSW and lonobj[0,il,iL] <= lonNE and \ latobj[0,il,iL] >= latSW and latobj[0,il,iL] <= latNE: inside2D[il, iL] = True if not searchInlist(newidx, iL): newidx.append(iL) if not searchInlist(newidy, il): newidy.append(il) for iz in range(Ndimslev): if iz >= levI and iz <= levF: if not searchInlist(newidz, iz): newidz.append(iz) inside3D[iz, il, iL] = True insideZ[iz] = True iind = iind + 1 Ninpts3D = len(inside3D) Ninpts2D = len(inside2D) NinptsZ = len(insideZ) newdx = len(newidx) newdy = len(newidy) newdz = len(newidz) print ' ' + fname + ': ',Ninpts3D,' pts found in the box: ',lonSW,',',latSW,',', \ levI,' x ', lonNE,',',latNE,',',levF print ' ' + fname + ': dimensions output matrix:', newdz, ',', newdy, ',', newdx # Creation of cropped matrices Ninlev = levF - levI + 1 inlon = np.zeros((Ninpts2D), dtype=np.float) inlat = np.zeros((Ninpts2D), dtype=np.float) inlev = np.zeros((NinptsZ), dtype=np.float) invar = np.zeros((varobj.shape[0],Ninpts3D), dtype=np.float) # Creation of the netCDF file ## objofile = NetCDFFile(ofile, 'w') if Ndimshare == 1: # Dimensions newdim = objofile.createDimension('boxpt', Ninpts3D) newdim = objofile.createDimension('time', None) # var dimensions newvar = objofile.createVariable('lon', 'f8', ('boxpt')) newattr = basicvardef(newvar, 'lon', 'longitude', 'degrees west_east') newvar[:] = lonobj[inside2D[iin]] newvar = objofile.createVariable('lat', 'f8', ('boxpt')) newattr = basicvardef(newvar, 'lat', 'latitude', 'degrees north_south') newvar[:] = latobj[inside2D[iin]] newvar = objofile.createVariable('lev', 'f8', ('NinptsZ')) newattr = basicvardef(newvar, 'lev', 'level', '-') newvar[:] = levobj[insideZ[iin]] newvar = objofile.createVariable('time', 'f8', ('time')) if objfile.variables.has_key('time'): otime = objfile.variables['time'] timevals = otime[:] if searchInlist(otime.ncattrs(),'units'): tunits = otime.getncattr['units'] else: tunits = 'unkown' else: timevals = np.arange(varobj.shape[0]) tunits = 'unkown' newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = timevals # variable newvar = objofile.createVariable(varn, 'f4', ('time', 'boxpt')) newvar[:] = varobj[invar] else: # Dimensions newdim = objofile.createDimension('x', newdx) newdim = objofile.createDimension('y', newdy) newdim = objofile.createDimension('z', newdz) newdim = objofile.createDimension('time', None) # var dimensions newvar = objofile.createVariable('lon', 'f8', ('y', 'x')) if Ndimshare == 2: Vals = lonobj[:] else: Vals = lonobj[0,:,:] for it in range(varobj.shape[0]): newvar[:] = Vals[inside2D].reshape(newdy,newdx) newattr = basicvardef(newvar, 'lon', 'longitude', 'degrees west_east') newvar = objofile.createVariable('lat', 'f8', ('y', 'x')) newattr = basicvardef(newvar, 'lat', 'latitude', 'degrees north_south') if Ndimshare == 2: Vals = latobj[:] else: Vals = latobj[0,:,:] for it in range(varobj.shape[0]): newvar[:] = Vals[inside2D].reshape(newdy,newdx) newvar = objofile.createVariable('lev', 'f8', ('z')) newattr = basicvardef(newvar, 'lev', 'level', '-') if len(levobj.shape) == 2: newvar[:] = levobj[0,insideZ] else: newvar[:] = levobj[insideZ] newvar = objofile.createVariable('time', 'f8', ('time')) if objfile.variables.has_key('time'): otime = objfile.variables['time'] timevals = otime[:] if searchInlist(otime.ncattrs(),'units'): tunits = otime.getncattr['units'] else: tunits = 'unkown' else: timevals = np.arange(varobj.shape[0]) tunits = 'unkown' newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = timevals # variable newvar = objofile.createVariable(varn, 'f4', ('time', 'z', 'y', 'x')) for it in range(varobj.shape[0]): valsvar = varobj[it,:,:,:] newvar[it,:,:,:] = valsvar[inside3D] if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = varn if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = varn if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = 'unkown' newattr = basicvardef(newvar, vsname, vlname, vunits) # global attributes objofile.setncattr('author', 'L. Fita') objofile.setncattr('institution', 'Laboratire de Meteorologie Dynamique') objofile.setncattr('university', 'Pierre Marie Curie - Jussieu') objofile.setncattr('center', 'Centre National de Recherches Scientifiques') objofile.setncattr('city', 'Paris') objofile.setncattr('country', 'France') objofile.setncattr('script', 'nc_var_tools.py') objofile.setncattr('function', 'sellonlatlevbox') objofile.setncattr('version', '1.0') objfile.close() objofile.sync() objofile.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return #sellonlatlevbox('XLONG,XLAT,ZNU,-12.4,25.35,32.4,52.65,0,0', 'control/wrfout_d01_2001-11-09_00:00:00', 'P') def file_nlines(filen,char): """ Function to provide the number of lines of a file filen= name of the file char= character as no line >>> file_nlines('trajectory.dat','#') 49 """ fname = 'file_nlines' if not os.path.isfile(filen): print errormsg print ' ' + fname + ' file: "' + filen + '" does not exist !!' quit(-1) fo = open(filen,'r') nlines=0 for line in fo: if line[0:1] != char: nlines = nlines + 1 fo.close() return nlines def datetimeStr_conversion(StringDT,typeSi,typeSo): """ Function to transform a string date to an another date object StringDT= string with the date and time typeSi= type of datetime string input typeSo= type of datetime string output [typeSi/o] 'cfTime': [time],[units]; ]time in CF-convention format [units] = [tunits] since [refdate] 'matYmdHMS': numerical vector with [[YYYY], [MM], [DD], [HH], [MI], [SS]] 'YmdHMS': [YYYY][MM][DD][HH][MI][SS] format 'Y-m-d_H:M:S': [YYYY]-[MM]-[DD]_[HH]:[MI]:[SS] format 'Y-m-d H:M:S': [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] format 'Y/m/d H-M-S': [YYYY]/[MM]/[DD] [HH]-[MI]-[SS] format 'WRFdatetime': [Y], [Y], [Y], [Y], '-', [M], [M], '-', [D], [D], '_', [H], [H], ':', [M], [M], ':', [S], [S] >>> datetimeStr_conversion('1976-02-17_08:32:05','Y-m-d_H:M:S','matYmdHMS') [1976 2 17 8 32 5] >>> datetimeStr_conversion(str(137880)+',minutes since 1979-12-01_00:00:00','cfTime','Y/m/d H-M-S') 1980/03/05 18-00-00 """ import datetime as dt fname = 'datetimeStr_conversion' if StringDT[0:1] == 'h': print fname + '_____________________________________________________________' print datetimeStr_conversion.__doc__ quit() if typeSi == 'cfTime': timeval = np.float(StringDT.split(',')[0]) tunits = StringDT.split(',')[1].split(' ')[0] Srefdate = StringDT.split(',')[1].split(' ')[2] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## yrref=Srefdate[0:4] monref=Srefdate[5:7] dayref=Srefdate[8:10] trefT = Srefdate.find(':') if not trefT == -1: # print ' ' + fname + ': refdate with time!' horref=Srefdate[11:13] minref=Srefdate[14:16] secref=Srefdate[17:19] refdate = datetimeStr_datetime( yrref + '-' + monref + '-' + dayref + \ '_' + horref + ':' + minref + ':' + secref) else: refdate = datetimeStr_datetime( yrref + '-' + monref + '-' + dayref + \ + '_00:00:00') if tunits == 'weeks': newdate = refdate + dt.timedelta(weeks=float(timeval)) elif tunits == 'days': newdate = refdate + dt.timedelta(days=float(timeval)) elif tunits == 'hours': newdate = refdate + dt.timedelta(hours=float(timeval)) elif tunits == 'minutes': newdate = refdate + dt.timedelta(minutes=float(timeval)) elif tunits == 'seconds': newdate = refdate + dt.timedelta(seconds=float(timeval)) elif tunits == 'milliseconds': newdate = refdate + dt.timedelta(milliseconds=float(timeval)) else: print errormsg print ' timeref_datetime: time units "' + tunits + '" not ready!!!!' quit(-1) yr = newdate.year mo = newdate.month da = newdate.day ho = newdate.hour mi = newdate.minute se = newdate.second elif typeSi == 'matYmdHMS': yr = StringDT[0] mo = StringDT[1] da = StringDT[2] ho = StringDT[3] mi = StringDT[4] se = StringDT[5] elif typeSi == 'YmdHMS': yr = int(StringDT[0:4]) mo = int(StringDT[4:6]) da = int(StringDT[6:8]) ho = int(StringDT[8:10]) mi = int(StringDT[10:12]) se = int(StringDT[12:14]) elif typeSi == 'Y-m-d_H:M:S': dateDT = StringDT.split('_') dateD = dateDT[0].split('-') timeT = dateDT[1].split(':') yr = int(dateD[0]) mo = int(dateD[1]) da = int(dateD[2]) ho = int(timeT[0]) mi = int(timeT[1]) se = int(timeT[2]) elif typeSi == 'Y-m-d H:M:S': dateDT = StringDT.split(' ') dateD = dateDT[0].split('-') timeT = dateDT[1].split(':') yr = int(dateD[0]) mo = int(dateD[1]) da = int(dateD[2]) ho = int(timeT[0]) mi = int(timeT[1]) se = int(timeT[2]) elif typeSi == 'Y/m/d H-M-S': dateDT = StringDT.split(' ') dateD = dateDT[0].split('/') timeT = dateDT[1].split('-') yr = int(dateD[0]) mo = int(dateD[1]) da = int(dateD[2]) ho = int(timeT[0]) mi = int(timeT[1]) se = int(timeT[2]) elif typeSi == 'WRFdatetime': yr = int(StringDT[0])*1000 + int(StringDT[1])*100 + int(StringDT[2])*10 + \ int(StringDT[3]) mo = int(StringDT[5])*10 + int(StringDT[6]) da = int(StringDT[8])*10 + int(StringDT[9]) ho = int(StringDT[11])*10 + int(StringDT[12]) mi = int(StringDT[14])*10 + int(StringDT[15]) se = int(StringDT[17])*10 + int(StringDT[18]) else: print errormsg print ' ' + fname + ': type of String input date "' + typeSi + \ '" not ready !!!!' quit(-1) if typeSo == 'matYmdHMS': dateYmdHMS = np.zeros((6), dtype=int) dateYmdHMS[0] = yr dateYmdHMS[1] = mo dateYmdHMS[2] = da dateYmdHMS[3] = ho dateYmdHMS[4] = mi dateYmdHMS[5] = se elif typeSo == 'YmdHMS': dateYmdHMS = str(yr).zfill(4) + str(mo).zfill(2) + str(da).zfill(2) + \ str(ho).zfill(2) + str(mi).zfill(2) + str(se).zfill(2) elif typeSo == 'Y-m-d_H:M:S': dateYmdHMS = str(yr).zfill(4) + '-' + str(mo).zfill(2) + '-' + \ str(da).zfill(2) + '_' + str(ho).zfill(2) + ':' + str(mi).zfill(2) + ':' + \ str(se).zfill(2) elif typeSo == 'Y-m-d H:M:S': dateYmdHMS = str(yr).zfill(4) + '-' + str(mo).zfill(2) + '-' + \ str(da).zfill(2) + ' ' + str(ho).zfill(2) + ':' + str(mi).zfill(2) + ':' + \ str(se).zfill(2) elif typeSo == 'Y/m/d H-M-S': dateYmdHMS = str(yr).zfill(4) + '/' + str(mo).zfill(2) + '/' + \ str(da).zfill(2) + ' ' + str(ho).zfill(2) + '-' + str(mi).zfill(2) + '-' + \ str(se).zfill(2) elif typeSo == 'WRFdatetime': dateYmdHMS = [] yM = yr/1000 yC = (yr-yM*1000)/100 yD = (yr-yM*1000-yC*100)/10 yU = yr-yM*1000-yC*100-yD*10 mD = mo/10 mU = mo-mD*10 dD = da/10 dU = da-dD*10 hD = ho/10 hU = ho-hD*10 miD = mi/10 miU = mi-miD*10 sD = se/10 sU = se-sD*10 dateYmdHMS.append(str(yM)) dateYmdHMS.append(str(yC)) dateYmdHMS.append(str(yD)) dateYmdHMS.append(str(yU)) dateYmdHMS.append('-') dateYmdHMS.append(str(mD)) dateYmdHMS.append(str(mU)) dateYmdHMS.append('-') dateYmdHMS.append(str(dD)) dateYmdHMS.append(str(dU)) dateYmdHMS.append('_') dateYmdHMS.append(str(hD)) dateYmdHMS.append(str(hU)) dateYmdHMS.append(':') dateYmdHMS.append(str(miD)) dateYmdHMS.append(str(miU)) dateYmdHMS.append(':') dateYmdHMS.append(str(sD)) dateYmdHMS.append(str(sU)) else: print errormsg print ' ' + fname + ': type of output date "' + typeSo + '" not ready !!!!' quit(-1) return dateYmdHMS def radius_dist(dx,dy,ptx,pty): """ Function to generate a matrix with the distance at a given point radius_dist(dx,dy,ptx,pty) [dx/y]: dimension of the matrix [ptx/y]: grid point coordinates of the point >>> radius_dist(3,5,2,2) [[ 1.41421356 1. 1.41421356 2.23606798 3.16227766] [ 1. 0. 1. 2. 3. ] [ 1.41421356 1. 1.41421356 2.23606798 3.16227766]] """ fname = 'radius_dist' if ptx < 0 or ptx > dx-1 or pty < 0 or pty > dy-1: print errormsg print ' ' + fname + ': wrong point coordinates:',dx,',',dy,'for matrix;',dx \ ,'x',dy quit(-1) xdist = np.zeros((dx,dy), dtype=np.float) ydist = np.zeros((dx,dy), dtype=np.float) dist = np.zeros((dx,dy), dtype=np.float) for ix in range(dx): xdist[ix,:] = np.float(ix-ptx) for iy in range(dy): ydist[:,iy] = np.float(iy-pty) dist = np.sqrt(xdist*xdist + ydist*ydist) return dist def compute_tevolboxtraj(values, ncfile, varn): """ Function to compute tevolboxtraj: temporal evolution at a given point along a box following a trajectory values= [trajfile]@[Tbeg],[lonname],[latname],[timename],[timekind],[boxsize],[circler] [trajfile]: ASCII file with the trajectory ('#' not readed) [time] [xpoint] [ypoint] [Tbeg]: equivalent first time-step of the trajectory within the netCDF file [lonname],[latname],[timename]: longitude, latitude and time variables names [timekind]: kind of time 'cf': cf-compilant 'wrf': WRF kind [boxsize]: size in grid points of the box (square centered, better even number!) [circler]: radius in grid points of a centerd circle ncfile= netCDF file to use varn= variable name EMERGENCY version, assuming 3D [time,lat,lon] variable ! """ import numpy.ma as ma fname='compute_tevolboxtraj' if values == 'h': print fname + '_____________________________________________________________' print compute_tevolboxtraj.__doc__ quit() ofile = 'tevolboxtraj_' + varn + '.nc' trajfile = values.split(',')[0].split('@')[0] Tbeg = int(values.split(',')[0].split('@')[1]) lonn = values.split(',')[1] latn = values.split(',')[2] timn = values.split(',')[3] timekind = values.split(',')[4] boxs = int(values.split(',')[5]) circler = int(values.split(',')[6]) if np.mod(boxs,2) != 1: print errormsg print ' ' + fname + ': box size:', boxs, 'is not even !!!' quit(-1) box2 = int(boxs/2) if not os.path.isfile(ncfile): print errormsg print ' ' + fname + " netCDF file: '" + ncfile + "' does not exist !!" quit(-1) if not os.path.isfile(trajfile): print errormsg print ' ' + fname + " trajectory file: '" + trajfile + "' does not exist !!" quit(-1) # circle values # radius = np.float(rboxs.split('#')[0]) # ddist = np.float(rboxs.split('#')[1]) # if radius < ddist: # print errormsg # print ' ' + fname + ': radius of the circle:',radius,' is smaller than ' + \ # 'the distance between grid points !!' # quit(-1) # else: # Nrad = int(radius/ddist) Nrad = int(circler) objfile = NetCDFFile(ncfile, 'r') lonobj = objfile.variables[lonn] latobj = objfile.variables[latn] timobj = objfile.variables[timn] if len(lonobj.shape) == 3: lonv = lonobj[0,:,:] latv = latobj[0,:,:] elif len(lonobj.shape) == 2: lonv = lonobj[:] latv = latobj[:] dimx = lonv.shape[1] dimy = lonv.shape[0] timv = timobj[:] if not searchInlist(objfile.variables, varn): print errormsg print ' ' + fname + ": variable name '" + varn + "' is not in file " + \ ncfile + '" !!!!!' quit(-1) varobj = objfile.variables[varn] # Selecting accordingly a trajectory ## Ttraj = file_nlines(trajfile,'#') if Ttraj > varobj.shape[0]: print errormsg print ' ' + fname + ': trajectory has ', Ttraj, ' time steps and data', \ varobj.shape[0], '" !!!!!' quit(-1) print ' ' + fname + ': Number of time-steps in trajectory file: ',Ttraj trajobj = open(trajfile,'r') gtrajvals = np.zeros((Ttraj,3), dtype=int) trajvals = np.zeros((Ttraj,3), dtype=np.float) varvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) lonvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) latvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) rvarvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) rlonvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) rlatvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) statvarvals = np.ones(tuple([Ttraj,6]), dtype=np.float) rstatvarvals = np.ones(tuple([Ttraj,6]), dtype=np.float) iline = 0 it = 0 for line in trajobj: slicev = [] slice2D = [] slicevnoT = [] ## Skipping first line ## if not iline == 0: ## it = iline - 1 ## it = iline # Slicing brings to reduce 1 time-step.... ??? if line[0:1] != '#': gtrajvals[it,0] = Tbeg + iline gtrajvals[it,1] = int(line.split(' ')[1]) gtrajvals[it,2] = int(line.split(' ')[2]) # print it,'t:',gtrajvals[it,0],'x y:', gtrajvals[it,1], gtrajvals[it,2] if timekind == 'wrf': gdate = datetimeStr_conversion(timv[gtrajvals[it,0]],'WRFdatetime', \ 'matYmdHMS') trajvals[it,0] = realdatetime1_CFcompilant(gdate, '19491201000000', \ 'hours') tunits = 'hours since 1949-12-01 00:00:00' elif timekind == 'cf': trajvals[it,0] = timv[gtrajvals[it,0]] tunits = timobj.getncattr('units') else: print errormsg print ' ' + fname + ' time kind: "' + timekind + '" not ready !!' quit(-1) trajvals[it,1] = lonv[gtrajvals[it,2],gtrajvals[it,1]] trajvals[it,2] = latv[gtrajvals[it,2],gtrajvals[it,1]] # print iline, it,'time:',trajvals[it,0],'lon lat:', trajvals[it,1], trajvals[it,2] # Assuming time as the first dimension in a fortran like way (t=0 as 1) # trajcut[0] = tstept - 1 # Assuming time as the first dimension in a C/python like way (t=0) slicev.append(gtrajvals[it,0]) # box values if gtrajvals[it,2]-box2 < 0 or gtrajvals[it,2]+box2 + 1 > dimy + 1 \ or gtrajvals[it,1]-box2 < 0 or gtrajvals[it,1]+box2 + 1 > dimx + 1: varvalst = np.ones((boxs, boxs), dtype=np.float)*fillValue if gtrajvals[it,2]-box2 < 0: yinit = 0 yinit2D = box2 - gtrajvals[it,2] else: yinit = gtrajvals[it,2]-box2 yinit2D = 0 if gtrajvals[it,2]+box2 + 1 > dimy + 1: yend = dimy + 1 yend2D = dimy + 1 - gtrajvals[it,2] + box2 else: yend = gtrajvals[it,2]+box2 + 1 yend2D = boxs if gtrajvals[it,1]-box2 < 0: xinit = 0 xinit2D = box2 - gtrajvals[it,1] else: xinit = gtrajvals[it,1]-box2 xinit2D = 0 if gtrajvals[it,1]+box2 + 1 > dimx + 1: xend = dimx + 1 xend2D = dimx + 1 - gtrajvals[it,1] - box2 else: xend = gtrajvals[it,1]+box2 + 1 xend2D = boxs slicev.append(slice(yinit, yend)) slicev.append(slice(xinit, xend)) slicevnoT.append(slice(yinit, yend)) slicevnoT.append(slice(xinit, xend)) slice2D.append(slice(yinit2D, yend2D)) slice2D.append(slice(xinit2D, xend2D)) varvalst[tuple(slice2D)] = varobj[tuple(slicev)] varvals[it,:,:] = varvalst # box stats values maskedvals = ma.masked_values (varvalst, fillValue) statvarvals[it,0] = varvalst[box2,box2] statvarvals[it,1] = maskedvals.min() statvarvals[it,2] = maskedvals.max() statvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals statvarvals[it,4] = maskedvals2.mean() statvarvals[it,5] = np.sqrt(statvarvals[it,4] - \ statvarvals[it,3]*statvarvals[it,3]) varvalst[tuple(slice2D)] = lonv[tuple(slicevnoT)] lonvals[it,:,:] = varvalst varvalst[tuple(slice2D)] = latv[tuple(slicevnoT)] latvals[it,:,:] = varvalst else: slicev.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2 + 1)) slicev.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2 + 1)) slicevnoT.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2+1)) slicevnoT.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2+1)) slice2D.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2 + 1)) slice2D.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2 + 1)) varvalst = varobj[tuple(slicev)] # box values varvals[it,:,:] = varvalst # print 'values at time t______' # print varvalst # box stats values statvarvals[it,0] = varvalst[box2,box2] statvarvals[it,1] = np.min(varvalst) statvarvals[it,2] = np.max(varvalst) statvarvals[it,3] = np.mean(varvalst) statvarvals[it,4] = np.mean(varvalst*varvalst) statvarvals[it,5] = np.sqrt(statvarvals[it,4] - \ statvarvals[it,3]*statvarvals[it,3]) varvalst = lonv[tuple(slicevnoT)] lonvals[it,:,:] = varvalst varvalst = latv[tuple(slicevnoT)] latvals[it,:,:] = varvalst # circle values slicev = [] slice2D = [] slicevnoT = [] slicev.append(gtrajvals[it,0]) circdist = radius_dist(dimy, dimx, gtrajvals[it,2], gtrajvals[it,1]) if gtrajvals[it,2]-Nrad < 0 or gtrajvals[it,2]+Nrad + 1 > dimy + 1 \ or gtrajvals[it,1]-Nrad < 0 or gtrajvals[it,1]+Nrad + 1 > dimx + 1: rvarvalst = np.ones((Nrad*2+1, Nrad*2+1), dtype=np.float)*fillValue if gtrajvals[it,2]-Nrad < 0: yinit = 0 yinit2D = Nrad - gtrajvals[it,2] else: yinit = gtrajvals[it,2]-Nrad yinit2D = 0 if gtrajvals[it,2]+Nrad + 1 > dimy + 1: yend = dimy + 1 yend2D = dimy + 1 - gtrajvals[it,2] + Nrad else: yend = gtrajvals[it,2]+Nrad + 1 yend2D = 2*Nrad+1 if gtrajvals[it,1]-Nrad < 0: xinit = 0 xinit2D = Nrad - gtrajvals[it,1] else: xinit = gtrajvals[it,1]-Nrad xinit2D = 0 if gtrajvals[it,1]+Nrad + 1 > dimx + 1: xend = dimx + 1 xend2D = dimx + 1 - gtrajvals[it,1] - Nrad else: xend = gtrajvals[it,1]+Nrad + 1 xend2D = 2*Nrad+1 slicev.append(slice(yinit, yend)) slicev.append(slice(xinit, xend)) slicevnoT.append(slice(yinit, yend)) slicevnoT.append(slice(xinit, xend)) slice2D.append(slice(yinit2D, yend2D)) slice2D.append(slice(xinit2D, xend2D)) rvarvalst[tuple(slice2D)] = varobj[tuple(slicev)] rvarvalst[tuple(slice2D)] = np.where(circdist[tuple(slice2D)] > \ np.float(Nrad), fillValue, rvarvalst[tuple(slice2D)]) rvarvals[it,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) rstatvarvals[it,0] = rvarvalst[Nrad,Nrad] rstatvarvals[it,1] = maskedvals.min() rstatvarvals[it,2] = maskedvals.max() rstatvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals rstatvarvals[it,4] = maskedvals2.mean() rstatvarvals[it,5] = np.sqrt(rstatvarvals[it,4] - \ rstatvarvals[it,3]*rstatvarvals[it,3]) rvarvalst[tuple(slice2D)] = lonv[tuple(slicevnoT)] rlonvals[it,:,:] = rvarvalst rvarvalst[tuple(slice2D)] = latv[tuple(slicevnoT)] rlatvals[it,:,:] = rvarvalst else: slicev.append(slice(gtrajvals[it,2]-Nrad, gtrajvals[it,2]+Nrad + 1)) slicev.append(slice(gtrajvals[it,1]-Nrad, gtrajvals[it,1]+Nrad + 1)) slicevnoT.append(slice(gtrajvals[it,2]-Nrad, gtrajvals[it,2]+Nrad+1)) slicevnoT.append(slice(gtrajvals[it,1]-Nrad, gtrajvals[it,1]+Nrad+1)) slice2D.append(slice(gtrajvals[it,2]-Nrad, gtrajvals[it,2]+Nrad + 1)) slice2D.append(slice(gtrajvals[it,1]-Nrad, gtrajvals[it,1]+Nrad + 1)) rvarvalst = varobj[tuple(slicev)] cdist = circdist[tuple(slicevnoT)] # circle values rvarvalst = np.where(cdist > np.float(Nrad), fillValue, rvarvalst) rvarvals[it,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) rstatvarvals[it,0] = rvarvalst[Nrad,Nrad] rstatvarvals[it,1] = maskedvals.min() rstatvarvals[it,2] = maskedvals.max() rstatvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals rstatvarvals[it,4] = maskedvals2.mean() rstatvarvals[it,5] = np.sqrt(rstatvarvals[it,4] - \ rstatvarvals[it,3]*rstatvarvals[it,3]) rvarvalst = lonv[tuple(slicevnoT)] rlonvals[it,:,:] = rvarvalst rvarvalst = latv[tuple(slicevnoT)] rlatvals[it,:,:] = rvarvalst it = it + 1 # print 'statistics:',rstatvarvals[it,:] iline = iline + 1 trajobj.close() # Creation of the netCDF file ## objofile = NetCDFFile(ofile, 'w') # Dimensions newdim = objofile.createDimension('x', boxs) newdim = objofile.createDimension('y', boxs) newdim = objofile.createDimension('xr', Nrad*2+1) newdim = objofile.createDimension('yr', Nrad*2+1) newdim = objofile.createDimension('time', None) # var dimensions newvar = objofile.createVariable('trlon', 'f8', ('time')) newattr = basicvardef(newvar,'trlon','trajectory longitude','degrees west_east') newvar[:] = trajvals[:,1] newvar = objofile.createVariable('trlat', 'f8', ('time')) newattr = basicvardef(newvar,'trlat','trajectory latitude','degrees north_south') newvar[:] = trajvals[:,2] newvar = objofile.createVariable('time', 'f8', ('time')) newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = trajvals[:,0] newvar = objofile.createVariable('lon', 'f8', ('time', 'y', 'x'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'longitude', 'longitude', 'degrees west_east') newvar[:] = lonvals newvar = objofile.createVariable('lat', 'f8', ('time', 'y', 'x'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'latitude', 'latitude', 'degrees north_south') newvar[:] = latvals # variable box values newvar = objofile.createVariable(varn + 'box', 'f4', ('time', 'y', 'x'), \ fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(varn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(varn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(varn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = varvals # center of the trajectory newvar = objofile.createVariable('trj_' + varn, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'trj_' + vsname, 'value along the trajectory of '+\ varn, vunits) newvar[:] = statvarvals[:,0] # variable box statistics ist = 0 statnames = ['minbox', 'maxbox', 'meanbox', 'mean2box', 'stdevbox'] vstlname = ['minimum value within', 'maximum value within', \ 'mean value within', 'squared mean value within', \ 'standard deviation value within'] for statn in statnames: newvar = objofile.createVariable(statn + '_' + varn, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the box ('+str(boxs)+'x'+str(boxs)+') of ' + varn, vunits) newvar[:] = statvarvals[:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 # variable circle values newvar = objofile.createVariable(varn + 'circle', 'f4', ('time', 'yr', 'xr'), \ fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(varn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(varn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(varn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = rvarvals # variable circle statistics ist = 0 statnames = ['mincircle', 'maxcircle', 'meancircle', 'mean2circle', 'stdevcircle'] for statn in statnames: newvar = objofile.createVariable(statn + '_' + varn, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the circle of radius ('+ str(circler)+') of ' + varn, vunits) newvar[:] = rstatvarvals[:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 # global attributes objofile.setncattr('author', 'L. Fita') objofile.setncattr('institution', 'Laboratire de Meteorologie Dynamique') objofile.setncattr('university', 'Pierre Marie Curie - Jussieu') objofile.setncattr('center', 'Centre National de Recherches Scientifiques') objofile.setncattr('city', 'Paris') objofile.setncattr('country', 'France') objofile.setncattr('script', 'nc_var_tools.py') objofile.setncattr('function', 'compute_tevolboxtraj') objofile.setncattr('version', '1.0') objofile.setncattr('data_file',ncfile) objfile.close() objofile.sync() objofile.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return #compute_tevolboxtraj('h', 'wrfout*', 'PSFC') #compute_tevolboxtraj('control/trajectory.dat@0,XLONG,XLAT,Times,wrf,3,3', \ # '../../superstorm/control/wrfout/wrfout_d01_2001-11-09_00:00:00', 'PSFC') #compute_tevolboxtraj('control/trajectory.dat@0,lon,lat,time,cf,5,5', \ # 'control/wss.nc', 'wss') def numVector_String(vec,char): """ Function to transform a vector of numbers to a single string [char] separated numVector_String(vec,char) vec= vector with the numerical values char= single character to split the values >>> print numVector_String(np.arange(10),' ') 0 1 2 3 4 5 6 7 8 9 """ fname = 'numVector_String' if vec == 'h': print fname + '_____________________________________________________________' print numVector_String.__doc__ quit() Nvals = len(vec) string='' for i in range(Nvals): if i == 0: string = str(vec[i]) else: string = string + char + str(vec[i]) return string def interpolate_locs(locs,coords,kinterp): """ Function to provide interpolate locations on a given axis interpolate_locs(locs,axis,kinterp) locs= locations to interpolate coords= axis values with the reference of coordinates kinterp: kind of interpolation 'lin': linear >>> coordinates = np.arange((10), dtype=np.float) >>> values = np.array([-1.2, 2.4, 5.6, 7.8, 12.0]) >>> interpolate_locs(values,coordinates,'lin') [ -1.2 2.4 5.6 7.8 13. ] >>> coordinates[0] = 0.5 >>> coordinates[2] = 2.5 >>> interpolate_locs(values,coordinates,'lin') [ -3.4 1.93333333 5.6 7.8 13. ] """ fname = 'interpolate_locs' if locs == 'h': print fname + '_____________________________________________________________' print interpolate_locs.__doc__ quit() Nlocs = locs.shape[0] Ncoords = coords.shape[0] dcoords = coords[Ncoords-1] - coords[0] intlocs = np.zeros((Nlocs), dtype=np.float) minc = np.min(coords) maxc = np.max(coords) for iloc in range(Nlocs): for icor in range(Ncoords-1): if locs[iloc] < minc and dcoords > 0.: a = 0. b = 1. / (coords[1] - coords[0]) c = coords[0] elif locs[iloc] > maxc and dcoords > 0.: a = (Ncoords-1)*1. b = 1. / (coords[Ncoords-1] - coords[Ncoords-2]) c = coords[Ncoords-2] elif locs[iloc] < minc and dcoords < 0.: a = (Ncoords-1)*1. b = 1. / (coords[Ncoords-1] - coords[Ncoords-2]) c = coords[Ncoords-2] elif locs[iloc] > maxc and dcoords < 0.: a = 0. b = 1. / (coords[1] - coords[0]) c = coords[0] elif locs[iloc] >= coords[icor] and locs[iloc] < coords[icor+1] and dcoords > 0.: a = icor*1. b = 1. / (coords[icor+1] - coords[icor]) c = coords[icor] print coords[icor], locs[iloc], coords[icor+1], ':', icor, '->', a, b elif locs[iloc] <= coords[icor] and locs[iloc] > coords[icor+1] and dcoords < 0.: a = icor*1. b = 1. / (coords[icor+1] - coords[icor]) c = coords[icor] if kinterp == 'lin': intlocs[iloc] = a + (locs[iloc] - c)*b else: print errormsg print ' ' + fname + ": interpolation kind '" + kinterp + "' not ready !!!!!" quit(-1) return intlocs def vertical_interpolation2D(varb,vart,zorigvals,znewval,kinterp): """ Function to vertically integrate a 3D variable vertical_interpolation2D(varb,vart,zorigvals,znewval) varb= values at the base of the interval of interpolation (2D field) vart= values at the top of the interval of interpolation (2D field) zorigvals= pair of original values (2, 2D field) znewval= new value to interpolate Possible cases: zorigvals[0,:,:] <= znewval < zorigvals[1,:,:] znewval <= zorigvals[0,:,:] < zorigvals[1,:,:] zorigvals[0,:,:] < zorigvals[1,:,:] <= znewval kinterp: kind of interpolation 'lin': linear >>> dx=5 >>> dy=7 >>> vals1 = np.ones((dy,dx), dtype=np.float) >>> vals2 = np.ones((dy,dx), dtype=np.float)*2. >>> zbase = np.zeros((2,dy,dx), dtype=np.float) >>> zbase[0,:,:] = 0.5 >>> zbase[1,:,:] = 1. >>> vertical_interpolation2D(vals1,vals2, zbase, newz,'lin') [[ 1.5 1.5 1.5 1.5 1.5] [ 1.5 1.5 1.5 1.5 1.5] [ 1.5 1.5 1.5 1.5 1.5] [ 1.5 1.5 1.5 1.5 1.5] [ 1.5 1.5 1.5 1.5 1.5] [ 1.5 1.5 1.5 1.5 1.5] [ 1.5 1.5 1.5 1.5 1.5]] """ fname = 'vertical_interpolation2D' if varb == 'h': print fname + '_____________________________________________________________' print vertical_interpolation2D.__doc__ quit() newvar = np.zeros((varb.shape), dtype=np.float) if kinterp == 'lin': ## print ' ' + fname + ' Vertical linear interpolation at',znewval # Standard linear interpolation (y = a + b*incz) # a = zorig[0], b = (vart-varb)/(zorig[1]-zorig[0])), incz = znewval - zorig[0] a = varb b = np.where(zorigvals[1,:,:] == zorigvals[0,:,:], 0., \ (vart - varb)/(zorigvals[1,:,:] - zorigvals[0,:,:])) incz = np.ones((varb.shape), dtype=np.float)*znewval - zorigvals[0,:,:] newvar = a + b*incz # Too code for not be used... but maybe? # dx=varb.shape[1] # dy=varb.shape[0] # for j in range(dy): # for i in range(dx): # if zorigvals[1,j,i] == zorigvals[0,j,i]: print 'equals!', \ # zorigvals[1,j,i], zorigvals[0,j,i],':',newvar[j,i],'@',vart[j,i] # if zorigvals[0,j,i] != zorigvals[0,j,i]: print '0 Nan', \ # zorigvals[0,j,i],':',newvar[j,i],'@',vart[j,i] # if zorigvals[1,j,i] != zorigvals[1,j,i]: print '1 Nan', \ # zorigvals[1,j,i],':',newvar[j,i],'@',vart[j,i] # if zorigvals[0,j,i] is None: print '0 None', zorigvals[0,j,i],':', \ # newvar[j,i],'@',vart[j,i] # if zorigvals[1,j,i] is None: print '1 None', zorigvals[1,j,i],':', \ # newvar[j,i],'@',vart[j,i] else: print errormsg print ' ' + fname + ": interpolation kind '" + kinterp + "' not ready !!!!!" quit(-1) return newvar #dx=5 #dy=7 #vals1 = np.ones((dy,dx), dtype=np.float) #vals2 = np.ones((dy,dx), dtype=np.float)*2. #zbase = np.zeros((2,dy,dx), dtype=np.float) #zbase[0,:,:] = 0.5 #for i in range(dx): # for j in range(dy): # zbase[1,j,i] = np.sqrt((j-dy/2.)**2. + (i-dx/2.)**2.) / \ # np.sqrt((dy/2.)**2.+(dy/2.)**2.) + 1. #zbase[1,:,:] = 1. #newz = 0.75 #print vertical_interpolation2D(vals1,vals2, zbase, newz,'lin') def vertical_interpolation(varb,vart,zorigvals,znewval,kinterp): """ Function to vertically integrate a 1D variable vertical_interpolation(varb,vart,zorigvals,znewval) varb= values at the base of the interval of interpolation vart= values at the top of the interval of interpolation zorigvals= pair of original values (2) znewval= new value to interpolate Possible cases: zorigvals[0,:] <= znewval < zorigvals[1,:] znewval <= zorigvals[0,:] < zorigvals[1,:] zorigvals[0,:] < zorigvals[1,:] <= znewval kinterp: kind of interpolation 'lin': linear >>> dx=5 >>> vals1 = np.ones((dx), dtype=np.float) >>> vals2 = np.ones((dx), dtype=np.float)*2. >>> zbase = np.zeros((2,dx), dtype=np.float) >>> zbase[0,:] = 0.5 >>> for i in range(dx): >>> zbase[1,i] = i + 1. >>> newz = 0.75 >>> vertical_interpolation2D(vals1,vals2, zbase, newz,'lin') [ 1.5 1.16666667 1.1 1.07142857 1.05555556] """ fname = 'vertical_interpolation' if varb == 'h': print fname + '_____________________________________________________________' print vertical_interpolation.__doc__ quit() newvar = np.zeros((varb.shape), dtype=np.float) if kinterp == 'lin': ## print ' ' + fname + ' Vertical linear interpolation at',znewval # Standard linear interpolation (y = a + b*incz) # a = zorig[0], b = (vart-varb)/(zorig[1]-zorig[0])), incz = znewval - zorig[0] a = varb b = np.where(zorigvals[1,:] == zorigvals[0,:], 0., \ (vart - varb)/(zorigvals[1,:] - zorigvals[0,:])) incz = np.ones((varb.shape), dtype=np.float)*znewval - zorigvals[0,:] newvar = a + b*incz else: print errormsg print ' ' + fname + ": interpolation kind '" + kinterp + "' not ready !!!!!" quit(-1) return newvar def interpolate_3D(zorigvs, varv, zintvs, kint): """ Function to interpolate a 3D variable interpolate_3D(zintvs, varv, zorigvals) zorigvs= original 3D z values varv= 3D variable to interpolate zintvs= new series of z values to interpolate kint= kind of interpolation 'lin': linear """ fname = 'interpolate_3D' if zorigvs == 'h': print fname + '_____________________________________________________________' print interpolate_3D.__doc__ quit() Ninv = len(zintvs) dimv = varv.shape # Sense of the original z-variable Lzorig = zorigvs.shape[0] incz = zorigvs[Lzorig-1,0,0] - zorigvs[0,0,0] varin = np.zeros((Ninv,dimv[1],dimv[2]), dtype=np.float) for ilev in range(Ninv): # print ' vertical interpolate level: ',zintvs[ilev] valinf = np.zeros((dimv[1], dimv[2]), dtype=np.float) valsup = np.zeros((dimv[1], dimv[2]), dtype=np.float) zorigv = np.zeros((2,dimv[1], dimv[2]), dtype=np.float) for j in range(valinf.shape[0]): for i in range(valinf.shape[1]): if np.min(zorigvs[:,j,i]) > zintvs[ilev] and incz > 0.: valinf[j,i] = varv[0,j,i] valsup[j,i] = varv[1,j,i] zorigv[0,j,i] = zorigvs[0,j,i] zorigv[1,j,i] = zorigvs[1,j,i] elif np.max(zorigvs[:,j,i]) < zintvs[ilev] and incz < 0.: valinf[j,i] = varv[0,j,i] valsup[j,i] = varv[1,j,i] zorigv[0,j,i] = zorigvs[0,j,i] zorigv[1,j,i] = zorigvs[1,j,i] elif np.max(zorigvs[:,j,i]) < zintvs[ilev] and incz > 0.: valinf[j,i] = varv[Lzorig-2,j,i] valsup[j,i] = varv[Lzorig-1,j,i] zorigv[0,j,i] = zorigvs[Lzorig-2,j,i] zorigv[1,j,i] = zorigvs[Lzorig-1,j,i] elif np.min(zorigvs[:,j,i]) > zintvs[ilev] and incz < 0.: valinf[j,i] = varv[Lzorig-2,j,i] valsup[j,i] = varv[Lzorig-1,j,i] zorigv[0,j,i] = zorigvs[Lzorig-2,j,i] zorigv[1,j,i] = zorigvs[Lzorig-1,j,i] # print 'top: ',i,j,':',zorigv[0,j,i], zintvs[ilev], zorigv[1,j,i] else: for z in range(Lzorig-1): if (zorigvs[z,j,i]-zintvs[ilev])*(zorigvs[z+1,j,i]- \ zintvs[ilev]) <= 0.: valinf[j,i] = varv[z,j,i] valsup[j,i] = varv[z+1,j,i] zorigv[0,j,i] = zorigvs[z,j,i] zorigv[1,j,i] = zorigvs[z+1,j,i] break # print 'zorigvs: ',zorigvs[:,0,0] # print ' valinf:', valinf[0,0] # print ' valsup:', valsup[0,0] # print ' zorigv 0:',zorigv[0,0,0] # print ' zorigv 1:',zorigv[1,0,0] varin[ilev,:,:] = vertical_interpolation2D(valinf,valsup,zorigv, \ zintvs[ilev], kint) # print ' varin:',varin[ilev,0,0] # quit() # quit() return varin def interpolate_2D(zorigvs0, varv0, zdim, zintvs, kint): """ Function to interpolate a 2D variable interpolate_2D(zintvs, varv, zorigvals) zorigvs= original 2D z values varv= 2D variable to interpolate zdim= number of the dimension with the z-values zintvs= new series of z values to interpolate kint= kind of interpolation 'lin': linear """ fname = 'interpolate_2D' if zorigvs0 == 'h': print fname + '_____________________________________________________________' print interpolate_2D.__doc__ quit() Ninv = len(zintvs) if zdim == 0: varv = varv0 zorigvs = zorigvs0 else: varv = np.transpose(varv0) zorigvs = np.transpose(zorigvs0) dimv = varv.shape # Sense of the original z-variable Lzorig = zorigvs.shape[0] incz = zorigvs[Lzorig-1,0] - zorigvs[0,0] varin = np.zeros((Ninv,dimv[1]), dtype=np.float) for ilev in range(Ninv): valinf = np.zeros((dimv[1]), dtype=np.float) valsup = np.zeros((dimv[1]), dtype=np.float) zorigv = np.zeros((2,dimv[1]), dtype=np.float) for i in range(valinf.shape[0]): if np.min(zorigvs[:,i]) > zintvs[ilev] and incz > 0.: valinf[i] = varv[0,i] valsup[i] = varv[1,i] zorigv[0,i] = zorigvs[0,i] zorigv[1,i] = zorigvs[1,i] elif np.max(zorigvs[:,i]) < zintvs[ilev] and incz < 0.: valinf[i] = varv[0,i] valsup[i] = varv[1,i] zorigv[0,i] = zorigvs[0,i] zorigv[1,i] = zorigvs[1,i] elif np.max(zorigvs[:,i]) < zintvs[ilev] and incz > 0.: valinf[i] = varv[Lzorig-2,i] valsup[i] = varv[Lzorig-1,i] zorigv[0,i] = zorigvs[Lzorig-2,i] zorigv[1,i] = zorigvs[Lzorig-1,i] elif np.min(zorigvs[:,i]) > zintvs[ilev] and incz < 0.: valinf[i] = varv[Lzorig-2,i] valsup[i] = varv[Lzorig-1,i] zorigv[0,i] = zorigvs[Lzorig-2,i] zorigv[1,i] = zorigvs[Lzorig-1,i] else: for z in range(Lzorig-1): if (zorigvs[z,i]-zintvs[ilev])*(zorigvs[z+1,i]- \ zintvs[ilev]) <= 0.: valinf[i] = varv[z,i] valsup[i] = varv[z+1,i] zorigv[0,i] = zorigvs[z,i] zorigv[1,i] = zorigvs[z+1,i] break varin[ilev,:] = vertical_interpolation(valinf,valsup,zorigv, \ zintvs[ilev], kint) if zdim == 0: varin0 = varin else: varin0 = np.transpose(varin) return varin0 def list_toVec(listv, kind): """ Function to transform from a list of values to a vector list_toVec(listv, kind) listv= list with the values kind= kind of values >>> list_toVec(['1', 2, '3', 4, 5, '6.9', 7, 9], 'npfloat') [ 1. 2. 3. 4. 5. 6.9 7. 9. ] """ fname = 'list_toVec' if listv == 'h': print fname + '_____________________________________________________________' print list_toVec.__doc__ quit() Nvals = len(listv) if kind == 'npfloat': vecv = np.zeros((Nvals), dtype=np.float) for iv in range(Nvals): vecv[iv] = np.float(listv[iv]) else: print errormsg print ' ' + fname + ': type of value "' + kind + '" not ready!!' quit(-1) return vecv def running_mean(values, run): """ Function to compute a running mean of a series of values running_mean(vals, int) [vals]: series of values [run]: interval to use to compute the running mean NOTE: resultant size would be the same but with None except at [run/2,size(vals)-run/2] >>> running_mean(np.arange(100),10) [None None None None None 4.5 5.5 6.5 7.5 8.5 9.5 10.5 11.5 12.5 13.5 14.5 15.5 16.5 17.5 18.5 19.5 20.5 21.5 22.5 23.5 24.5 25.5 26.5 27.5 28.5 29.5 30.5 31.5 32.5 33.5 34.5 35.5 36.5 37.5 38.5 39.5 40.5 41.5 42.5 43.5 44.5 45.5 46.5 47.5 48.5 49.5 50.5 51.5 52.5 53.5 54.5 55.5 56.5 57.5 58.5 59.5 60.5 61.5 62.5 63.5 64.5 65.5 66.5 67.5 68.5 69.5 70.5 71.5 72.5 73.5 74.5 75.5 76.5 77.5 78.5 79.5 80.5 81.5 82.5 83.5 84.5 85.5 86.5 87.5 88.5 89.5 90.5 91.5 92.5 93.5 None None None None None] """ fname = 'running_mean' if values == 'h': print fname + '_____________________________________________________________' print running_mean.__doc__ quit() runmean = np.ones((len(values)), dtype=np.float)*fillValue for ip in range(run/2,len(values)-run/2): runmean[ip] = np.mean(values[ip-run/2:ip+run/2]) runmean = np.where(runmean == fillValue, None, runmean) return runmean def slice_variable(varobj, dimslice): """ Function to return a slice of a given variable according to values to its dimensions slice_variable(varobj, dims) varobj= object wit the variable dimslice= [[dimname1]:[value1]|[[dimname2]:[value2], ...] pairs of dimension [value]: * [integer]: which value of the dimension * -1: all along the dimension * -9: last value of the dimension * [beg]:[end] slice from [beg] to [end] """ fname = 'slice_variable' if varobj == 'h': print fname + '_____________________________________________________________' print slice_variable.__doc__ quit() vardims = varobj.dimensions Ndimvar = len(vardims) Ndimcut = len(dimslice.split('|')) dimsl = dimslice.split('|') varvalsdim = [] dimnslice = [] for idd in range(Ndimvar): for idc in range(Ndimcut): dimcutn = dimsl[idc].split(':')[0] dimcutv = dimsl[idc].split(':')[1] if vardims[idd] == dimcutn: posfrac = dimcutv.find(':') if posfrac != -1: inifrac = int(dimcutv.split(':')[0]) endfrac = int(dimcutv.split(':')[1]) varvalsdim.append(slice(inifrac,endfrac)) dimnslice.append(vardims[idd]) else: if int(dimcutv) == -1: varvalsdim.append(slice(0,varobj.shape[idd])) dimnslice.append(vardims[idd]) elif int(dimcutv) == -9: varvalsdim.append(int(varobj.shape[idd])-1) else: varvalsdim.append(int(dimcutv)) break varvalues = varobj[tuple(varvalsdim)] return varvalues, dimnslice def operation_alongdims(ovar,dimvals,dimsoper,opkind): """ Function to operate along different dimensions of a variable operation_alongdims(ovar,dimvals,opkind) ovar= netCDF variable object dimvals= [dimname1]:[val1]|[dimdname2]:[val2]|[...[dimnameN]:[valN]] [value]: * [integer]: which value of the dimension * -1: all along the dimension * [beg]:[end] slice from [beg] to [end] dimsoper= [dimname1]:[dimname2]:[...[dimnameN]] names of the dimensions along which operation has to be done opkind= operation to perform along the dimensions with a range: max, mean, mean2, min, sum """ fname = 'operation_alongdims' if ovar == 'h': print fname + '_____________________________________________________________' print operation_alongdims.__doc__ quit() vardims = ovar.dimensions Nvardims = len(vardims) cutdims = dimvals.split('|') Ncutdims = len(cutdims) if Nvardims != Ncutdims: print errormsg print ' ' + fname + ': cutting for',Ncutdims,'but variable has:',Nvardims, \ '!!' quit(-1) for idc in range(Ncutdims): found = False for idv in range(Nvardims): if vardims[idv] == cutdims[idc].split(':')[0]: found = True break if not found: print errormsg print ' ' + fname + ": cutting by '",cutdims[idc].split(':')[0], \ "' but variable has not it !!!" print ' variable dims:',vardims quit(-1) slicedvar, dimslice = slice_variable(ovar, dimvals) # operation dimnsoper = dimsoper.split(':') if opkind == 'mean2': origvar = slicedvar*slicedvar else: origvar = slicedvar.copy() dimsnewvar = [] Ndimslice = len(dimslice) Ndimsoper = len(dimnsoper) noper=0 for ids in range(Ndimslice): found = False for ido in range(Ndimsoper): if dimslice[ids] == dimnsoper[ido]: if opkind == 'max': varoper = np.max(origvar, axis=ids-noper) elif opkind == 'mean': varoper = np.mean(origvar, axis=ids-noper) elif opkind == 'mean2': varoper = np.mean(origvar, axis=ids-noper) elif opkind == 'min': varoper = np.min(origvar, axis=ids-noper) elif opkind == 'sum': varoper = np.sum(origvar, axis=ids-noper) else: print errormsg print ' ' + fname + ': operation "' + opkind + '" not ready!!' quit(-1) del(origvar) origvar = varoper.copy() noper = noper + 1 found = True break if not found: dimsnewvar.append(dimslice[ids]) return origvar, dimsnewvar def file_oper_alongdims(values, ncfile, varn): """ Function to operate a file along different dimensions of a variable file_oper_alongdims(values, ncfile, varn) values= [dimvals],[dimsoper],[opkind],[dimvn] [dimvals]; [dimname1]:[val1]|[dimdname2]:[val2]|[...[dimnameN]:[valN]] [value]; * [integer]: which value of the dimension * -1: all along the dimension * [beg]:[end] slice from [beg] to [end] [dimsoper]; [dimname1]:[dimname2]:[...[dimnameN]] names of the dimensions along which operation has to be done [opkind]; operation to perform along the dimensions with a range: max, mean, mean2, min, sum [dimvn]; [varname1]:[varname2]:[...[varnameM]] variables with the values of the dimensions of the resultant operation ncfile= netCDF file varn= variable name """ fname = 'file_oper_alongdims' if values == 'h': print fname + '_____________________________________________________________' print file_oper_alongdims.__doc__ quit() dimvals = values.split(',')[0] dimsoper = values.split(',')[1] operkind = values.split(',')[2] dimvn = values.split(',')[3] ofile = 'file_oper_alongdims_' + operkind + '.nc' objnc = NetCDFFile(ncfile, 'r') if not objnc.variables.has_key(varn): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varn + '" !!!!' quit(-1) objvar = objnc.variables[varn] newvarv, newvardimns = operation_alongdims(objvar,dimvals,dimsoper,operkind) print 'dimsoper:',dimsoper print 'newvar shape',newvarv.shape,'newvardims:',newvardimns # Creation of output file ## objnewnc = NetCDFFile(ofile, 'w') # Dimensions for idim in range(len(newvardimns)): objdim = objnc.dimensions[newvardimns[idim]] if objdim.isunlimited(): objnewnc.createDimension(newvardimns[idim], None) else: objnewnc.createDimension(newvardimns[idim], len(objdim)) # Variables with dimension values vardims = dimvn.split(':') for ivdim in range(len(vardims)): objvdim = objnc.variables[vardims[ivdim]] dimsvdim = objvdim.dimensions dimvslice = [] dimvdims = [] for idmvd in range(len(dimsvdim)): found = False for idimv in range(len(newvardimns)): if dimsvdim[idmvd] == newvardimns[idimv]: dimvslice.append(slice(0,objvdim.shape[idmvd])) dimvdims.append(dimsvdim[idmvd]) found = True break if not found: dimvslice.append(len(objnc.dimensions[dimsvdim[idmvd]])/2) newvar = objnewnc.createVariable(vardims[ivdim], 'f8', tuple(dimvdims)) newvar[:] = objvdim[tuple(dimvslice)] dimvattrs = objvdim.ncattrs() for attrn in dimvattrs: attrv = objvdim.getncattr(attrn) newattr = set_attribute(newvar,attrn,attrv) # new variable oldvarattr = objvar.ncattrs() varname = variables_values(varn)[0] if searchInlist(oldvarattr, 'standard_name'): stdname = objvar.getncattr('standard_name') else: stdname = variables_values(varn)[1] if searchInlist(oldvarattr, 'long_name'): lname = objvar.getncattr('long_name') else: lname = variables_values(varn)[4].replace('|',' ') if searchInlist(oldvarattr, 'units'): uname = objvar.getncattr('units') else: uname = variables_values(varn)[5] dimsoperS = numVector_String(dimsoper.split(':'),', ') newvar = objnewnc.createVariable(varname + operkind, 'f4', tuple(newvardimns)) newattr = basicvardef(newvar, stdname + operkind, lname + ' '+ operkind + \ ' along ' + dimsoperS, uname) newvar[:] = newvarv for idim in range(len(objvar.shape)): dimn = dimvals.split('|')[idim].split(':')[0] dimv = '{:d}'.format(len(objnc.dimensions[dimn])) if idim == 0: origdimsS = dimn + '(' + dimv + ')' else: origdimsS = origdimsS + ' ' + dimn + '(' + dimv + ')' newattr = set_attribute(newvar, 'orig_dimensions', origdimsS) # global attributes objnewnc.setncattr('author', 'L. Fita') objnewnc.setncattr('institution', 'Laboratire de Meteorologie Dynamique') objnewnc.setncattr('university', 'Pierre Marie Curie - Jussieu') objnewnc.setncattr('center', 'Centre National de Recherches Scientifiques') objnewnc.setncattr('city', 'Paris') objnewnc.setncattr('country', 'France') objnewnc.setncattr('script', 'nc_var_tools.py') objnewnc.setncattr('function', 'file_oper_alongdims') objnewnc.setncattr('version', '1.0') objnewnc.setncattr('data_file', ncfile) gorigattrs = objnc.ncattrs() for attr in gorigattrs: attrv = objnc.getncattr(attr) atvar = set_attribute(objnewnc, attr, attrv) objnc.close() objnewnc.sync() objnewnc.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return #file_oper_alongdims('time:-1|z:-1|x:-1|y:-1,time:y,mean,pressure:lat', '/home/lluis/etudes/WRF_LMDZ/WaquaL/WRF_LMDZ/AR40/vertical_interpolation_WRFp.nc', 'WRFt') def variables_values(varName): """ Function to provide values to plot the different variables variables_values(varName) [varName]= name of the variable return: [var name], [std name], [minimum], [maximum], [long name]('|' for spaces), [units], [color palette] (following: http://matplotlib.org/1.3.1/examples/color/colormaps_reference.html) [varn]: original name of the variable NOTE: It might be better doing it with an external ASII file. But then we got an extra dependency... >>> variables_values('WRFght') ['z', 'geopotential_height', 0.0, 80000.0, 'geopotential|height', 'm2s-2', 'rainbow'] """ fname='variables_values' if varName == 'h': print fname + '_____________________________________________________________' print variables_values.__doc__ quit() # Variable name might come with a statistical surname... stats = ['min','max','mean','stdv','turb'] statsLong = {'min':'minimum', 'max':'maximum', 'mean':'mean', \ 'stdv':'standard deviation', 'turb':'turbulence'} ifst = False for st in stats: if varName.find(st) > -1: print ' '+ fname + ": varibale '" + varName + "' with a statistical "+\ " surname: '",st,"' !!" Lst = len(st) LvarName = len(varName) varn = varName[0:LvarName - Lst] ifst = True stname = st break if not ifst: varn = varName if varn[0:6] == 'varDIM': # Variable from a dimension (all with 'varDIM' prefix) Lvarn = len(varn) varvals = [varn[6:Lvarn+1], varn[6:Lvarn+1], 0., 1., \ "variable|from|size|of|dimension|'" + varn[6:Lvarn+1] + "'", '1', 'rainbox'] elif varn == 'a_tht' or varn == 'LA_THT': varvals = ['a_th', 'total_thermal_plume_cover', 0., 1., \ 'total|column|thermal|plume|cover', '1', 'YlGnBu'] elif varn == 'bils' or varn == 'LBILS': varvals = ['bils', 'surface_total_heat_flux', -100., 100., \ 'surface|total|heat|flux', 'Wm-2', 'seismic'] elif varn == 'landcat' or varn == 'category': varvals = ['landcat', 'land_categories', 0., 22., 'land|categories', '1', \ 'rainbow'] elif varn == 'c' or varn == 'QCLOUD' or varn == 'oliq' or varn == 'OLIQ': varvals = ['c', 'condensed_water_mixing_ratio', 0., 3.e-4, \ 'condensed|water|mixing|ratio', 'kgkg-1', 'BuPu'] elif varn == 'clt' or varn == 'CLT' or varn == 'cldt' or \ varn == 'Total cloudiness': varvals = ['clt', 'cloud_area_fraction', 0., 1., 'total|cloud|cover', '1', \ 'gist_gray'] elif varn == 'cll' or varn == 'cldl' or varn == 'LCLDL' or \ varn == 'Low-level cloudiness': varvals = ['cll', 'low_level_cloud_area_fraction', 0., 1., \ 'low|level|(p|>|680|hPa)|cloud|fraction', '1', 'gist_gray'] elif varn == 'clm' or varn == 'cldm' or varn == 'LCLDM' or \ varn == 'Mid-level cloudiness': varvals = ['clm', 'mid_level_cloud_area_fraction', 0., 1., \ 'medium|level|(440|<|p|<|680|hPa)|cloud|fraction', '1', 'gist_gray'] elif varn == 'clh' or varn == 'cldh' or varn == 'LCLDH' or \ varn == 'High-level cloudiness': varvals = ['clh', 'high_level_cloud_area_fraction', 0., 1., \ 'high|level|(p|<|440|hPa)|cloud|fraction', '1', 'gist_gray'] elif varn == 'dqajs' or varn == 'LDQAJS': varvals = ['dqajs', 'dry_adjustment_water_vapor_tendency', -0.0003, 0.0003, \ 'dry|adjustment|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqcon' or varn == 'LDQCON': varvals = ['dqcon', 'convective_water_vapor_tendency', -3e-8, 3.e-8, \ 'convective|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqdyn' or varn == 'LDQDYN': varvals = ['dqdyn', 'dynamics_water_vapor_tendency', -3.e-7, 3.e-7, \ 'dynamics|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqeva' or varn == 'LDQEVA': varvals = ['dqeva', 'evaporation_water_vapor_tendency', -3.e-6, 3.e-6, \ 'evaporation|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqlscst' or varn == 'LDQLSCST': varvals = ['dqlscst', 'stratocumulus_water_vapor_tendency', -3.e-7, 3.e-7, \ 'stratocumulus|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqlscth' or varn == 'LDQLSCTH': varvals = ['dqlscth', 'thermals_water_vapor_tendency', -3.e-7, 3.e-7, \ 'thermal|plumes|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqlsc' or varn == 'LDQLSC': varvals = ['dqlsc', 'condensation_water_vapor_tendency', -3.e-6, 3.e-6, \ 'condensation|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqphy' or varn == 'LDQPHY': varvals = ['dqphy', 'physics_water_vapor_tendency', -3.e-7, 3.e-7, \ 'physics|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqthe' or varn == 'LDQTHE': varvals = ['dqthe', 'thermals_water_vapor_tendency', -3.e-7, 3.e-7, \ 'thermal|plumes|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqvdf' or varn == 'LDQVDF': varvals = ['dqvdf', 'vertical_difussion_water_vapor_tendency', -3.e-8, 3.e-8,\ 'vertical|difussion|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqwak' or varn == 'LDQWAK': varvals = ['dqwak', 'wake_water_vapor_tendency', -3.e-7, 3.e-7, \ 'wake|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dtajs' or varn == 'LDTAJS': varvals = ['dtajs', 'dry_adjustment_thermal_tendency', -3.e-5, 3.e-5, \ 'dry|adjustment|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtcon' or varn == 'LDTCON': varvals = ['dtcon', 'convective_thermal_tendency', -3.e-5, 3.e-5, \ 'convective|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtdyn' or varn == 'LDTDYN': varvals = ['dtdyn', 'dynamics_thermal_tendency', -3.e-4, 3.e-4, \ 'dynamics|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dteva' or varn == 'LDTEVA': varvals = ['dteva', 'evaporation_thermal_tendency', -3.e-3, 3.e-3, \ 'evaporation|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtlscst' or varn == 'LDTLSCST': varvals = ['dtlscst', 'stratocumulus_thermal_tendency', -3.e-4, 3.e-4, \ 'stratocumulus|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtlscth' or varn == 'LDTLSCTH': varvals = ['dtlscth', 'thermals_thermal_tendency', -3.e-4, 3.e-4, \ 'thermal|plumes|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtlsc' or varn == 'LDTLSC': varvals = ['dtlsc', 'condensation_thermal_tendency', -3.e-3, 3.e-3, \ 'condensation|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtphy' or varn == 'LDTPHY': varvals = ['dtphy', 'physics_thermal_tendency', -3.e-4, 3.e-4, \ 'physics|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtthe' or varn == 'LDTTHE': varvals = ['dtthe', 'thermals_thermal_tendency', -3.e-4, 3.e-4, \ 'thermal|plumes|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtvdf' or varn == 'LDTVDF': varvals = ['dtvdf', 'vertical_difussion_thermal_tendency', -3.e-5, 3.e-5, \ 'vertical|difussion|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtwak' or varn == 'LDTWAK': varvals = ['dtwak', 'wake_thermal_tendency', -3.e-4, 3.e-4, \ 'wake|thermal|tendency', 'K/s', 'seismic'] elif varn == 'evspsbl' or varn == 'LEVAP' or varn == 'evap': varvals = ['evspsbl', 'water_evaporation_flux', 0., 1.5e-4, \ 'water|evaporation|flux', 'kgm-2s-1', 'Blues'] elif varn == 'h2o' or varn == 'LH2O': varvals = ['h2o', 'water_mass_fraction', 0., 3.e-2, \ 'mass|fraction|of|water', '1', 'Blues'] elif varn == 'hfls' or varn == 'LH' or varn == 'LFLAT' or varn == 'flat': varvals = ['hfls', 'surface_upward_latent_heat_flux', -400., 400., \ 'upward|latnt|heat|flux|at|the|surface', 'Wm-2', 'seismic'] elif varn == 'hfss' or varn == 'LSENS' or varn == 'sens': varvals = ['hfss', 'surface_upward_sensible_heat_flux', -150., 150., \ 'upward|sensible|heat|flux|at|the|surface', 'Wm-2', 'seismic'] elif varn == 'hus' or varn == 'WRFrh' or varn == 'LMDZrh' or varn == 'rhum' or \ varn == 'LRHUM': varvals = ['hus', 'specific_humidity', 0., 1., 'specific|humidty', '1', \ 'BuPu'] elif varn == 'huss' or varn == 'WRFrhs' or varn == 'LMDZrhs' or varn == 'rh2m' or\ varn == 'LRH2M': varvals = ['huss', 'specific_humidity', 0., 1., 'specific|humidty|at|2m', \ '1', 'BuPu'] elif varn == 'lat' or varn == 'XLAT' or varn == 'XLAT_M' or varn == 'latitude': varvals = ['lat', 'latitude', -90., 90., 'latitude', 'degrees North', \ 'seismic'] elif varn == 'lon' or varn == 'XLONG' or varn == 'XLONG_M': varvals = ['lon', 'longitude', -180., 180., 'longitude', 'degrees East', \ 'seismic'] elif varn == 'longitude': varvals = ['lon', 'longitude', 0., 360., 'longitude', 'degrees East', \ 'seismic'] elif varn == 'orog' or varn == 'HGT' or varn == 'HGT_M': varvals = ['orog', 'orography', 0., 3000., 'surface|altitude', 'm','terrain'] elif varn == 'pr' or varn == 'RAINTOT' or varn == 'precip' or \ varn == 'LPRECIP' or varn == 'Precip Totale liq+sol': varvals = ['pr', 'precipitation_flux', 0., 1.e-4, 'precipitation|flux', \ 'kgm-2s-1', 'BuPu'] elif varn == 'pracc' or varn == 'ACRAINTOT': varvals = ['pracc', 'precipitation_amount', 0., 100., \ 'accumulated|precipitation', 'kgm-2', 'BuPu'] elif varn == 'prc' or varn == 'LPLUC' or varn == 'pluc' or varn == 'WRFprc': varvals = ['prc', 'convective_precipitation_flux', 0., 2.e-4, \ 'convective|precipitation|flux', 'kgm-2s-1', 'Blues'] elif varn == 'pres' or varn == 'presnivs' or varn == 'pressure': varvals = ['pres', 'air_pressure', 0., 103000., 'air|pressure', 'Pa', \ 'Blues'] elif varn == 'prls' or varn == 'WRFprls' or varn == 'LPLUL' or varn == 'plul': varvals = ['prls', 'large_scale_precipitation_flux', 0., 2.e-4, \ 'large|scale|precipitation|flux', 'kgm-2s-1', 'Blues'] elif varn == 'prsn' or varn == 'SNOW' or varn == 'snow' or varn == 'LSNOW': varvals = ['prsn', 'snowfall', 0., 1.e-4, 'snowfall|flux', 'kgm-2s-1', 'BuPu'] elif varn == 'prw' or varn == 'WRFprh': varvals = ['prw', 'atmosphere_water_vapor_content', 0., 10., \ 'water|vapor"path', 'kgm-2', 'Blues'] elif varn == 'ps' or varn == 'psfc' or varn =='PSFC' or varn == 'psol' or \ varn == 'Surface Pressure': varvals=['ps', 'surface_air_pressure', 85000., 105400., 'surface|pressure', \ 'hPa', 'cool'] elif varn == 'psl' or varn == 'mslp' or varn =='WRFmslp': varvals=['psl', 'air_pressure_at_sea_level', 85000., 104000., \ 'mean|sea|level|pressure', 'Pa', 'Greens'] elif varn == 'q_th': varvals = ['q_th', 'thermal_plume_total_water_content', 0., 25., \ 'total|water|cotent|in|thermal|plume', 'mm', 'YlOrRd'] elif varn == 'r' or varn == 'QVAPOR' or varn == 'ovap' or varn == 'LOVAP': varvals = ['r', 'water_mixing_ratio', 0., 0.03, 'water|mixing|ratio', \ 'kgkg-1', 'BuPu'] elif varn == 'rsds' or varn == 'SWdnSFC' or varn == 'SWdn at surface' or \ varn == 'SWDOWN': varvals=['rsds', 'surface_downwelling_shortwave_flux_in_air', 0., 1200., \ 'downward|SW|surface|radiation', 'Wm-2' ,'Reds'] elif varn == 'rsdsacc': varvals=['rsdsacc', 'accumulated_surface_downwelling_shortwave_flux_in_air', \ 0., 1200., 'accumulated|downward|SW|surface|radiation', 'Wm-2' ,'Reds'] elif varn == 'rvor' or varn == 'WRFrvor': varvals = ['rvor', 'air_relative_vorticity', -2.5E-3, 2.5E-3, \ 'air|relative|vorticity', 's-1', 'seismic'] elif varn == 'rvors' or varn == 'WRFrvors': varvals = ['rvors', 'surface_air_relative_vorticity', -2.5E-3, 2.5E-3, \ 'surface|air|relative|vorticity', 's-1', 'seismic'] elif varn == 's_therm' or varn == 'LS_THERM': varvals = ['s_therm', 'thermals_excess', 0., 0.8, 'thermals|excess', 'K', \ 'Reds'] elif varn == 's_therm' or varn == 'LS_THERM': varvals = ['s_therm', 'thermals_excess', 0., 0.8, 'thermals|excess', 'K', \ 'Reds'] elif varn == 'ta' or varn == 'WRFt' or varn == 'temp' or varn == 'LTEMP' or \ varn == 'Air temperature': varvals = ['ta', 'air_temperature', 195., 320., 'air|temperature', 'K', \ 'YlOrRd'] elif varn == 'tas' or varn == 'T2' or varn == 't2m' or varn == 'T2M' or \ varn == 'Temperature 2m': varvals = ['tas', 'air_temperature', 240., 310., 'air|temperature|at|2m', ' \ K', 'YlOrRd'] elif varn == 'tds' or varn == 'TH2': varvals = ['tds', 'air_dew_point_temperature', 240., 310., \ 'air|dew|point|temperature|at|2m', 'K', 'YlGnBu'] elif varn == 'time'or varn == 'time_counter': varvals = ['time', 'time', 0., 1000., 'time', \ 'hours|since|1949/12/01|00:00:00', 'Reds'] elif varn == 'ua' or varn == 'vitu' or varn == 'U' or varn == 'Zonal wind' or \ varn == 'LVITU': varvals = ['ua', 'eastward_wind', -30., 30., 'eastward|wind', 'ms-1', \ 'seismic'] elif varn == 'uas' or varn == 'u10m' or varn == 'U10' or varn =='Vent zonal 10m': varvals = ['uas', 'eastward_wind', -30., 30., 'eastward|2m|wind', \ 'ms-1', 'seismic'] elif varn == 'va' or varn == 'vitv' or varn == 'V' or varn == 'Meridional wind' \ or varn == 'LVITV': varvals = ['va', 'northward_wind', -30., 30., 'northward|wind', 'ms-1', \ 'seismic'] elif varn == 'vas' or varn == 'v10m' or varn == 'V10' or \ varn =='Vent meridien 10m': varvals = ['vas', 'northward_wind', -30., 30., 'northward|2m|wind', 'ms-1', \ 'seismic'] elif varn == 'wake_h' or varn == 'LWAKE_H': varvals = ['wake_h', 'wake_height', 0., 1000., 'height|of|the|wakes', 'm', \ 'YlOrRd'] elif varn == 'wake_s' or varn == 'LWAKE_S': varvals = ['wake_s', 'wake_area_fraction', 0., 0.5, 'wake|spatial|fraction', \ '1', 'BuGn'] elif varn == 'wa' or varn == 'W' or varn == 'Vertical wind': varvals = ['wa', 'upward_wind', -10., 10., 'upward|wind', 'ms-1', \ 'seismic'] elif varn == 'wap' or varn == 'vitw' or varn == 'LVITW': varvals = ['wap', 'upward_wind', -3.e-10, 3.e-10, 'upward|wind', 'mPa-1', \ 'seismic'] elif varn == 'wss' or varn == 'SPDUV': varvals = ['wss', 'air_velocity', 0., 30., 'surface|horizontal|wind|speed', \ 'ms-1', 'Reds'] elif varn == 'xtime' or varn == 'XTIME': varvals = ['xtime', 'time', 0., 1.e5, 'time', \ 'minutes|since|simulation|start', 'Reds'] elif varn == 'x' or varn == 'X': varvals = ['x', 'x', 0., 100., 'x', '-', 'Reds'] elif varn == 'y' or varn == 'Y': varvals = ['y', 'y', 0., 100., 'y', '-', 'Blues'] elif varn == 'z' or varn == 'Z': varvals = ['z', 'z', 0., 100., 'z', '-', 'Greens'] elif varn == 'zg' or varn == 'WRFght' or varn == 'Geopotential height' or \ varn == 'geop' or varn == 'LGEOP': varvals = ['zg', 'geopotential_height', 0., 80000., 'geopotential|height', \ 'm2s-2', 'rainbow'] elif varn == 'zmax_th' or varn == 'LZMAX_TH': varvals = ['zmax_th', 'thermal_plume_height', 0., 4000., \ 'maximum|thermals|plume|height', 'm', 'YlOrRd'] elif varn == 'zmla' or varn == 's_pblh' or varn == 'LS_PBLH': varvals = ['zmla', 'atmosphere_boundary_layer_thickness', 0., 2500., \ 'atmosphere|boundary|layer|thickness', 'm', 'Blues'] else: print errormsg print ' ' + fname + ': variable ' + varn + ' not defined !!!' quit(-1) if ifst: varvals[0] = varvals[0] + stname ## Should be already there # varvals[4] = varvals[4] + statsLong[stname] return varvals def WRF_CFtime_creation(values, ncfile, varn): """ Function to add a CF-convention time unit in a WRF file WRF_CFtime_creation(values, ncfile, varn) [values]= [refdate],[units] [refdate]: reference date to compute the times in [YYYY][MM][DD][HH][MI][SS] format [units]: time units to use: 'days','hours','minutes','seconds' [ncfile]= WRF file to add the CF-convention time variable [varn]= name of the variable time to add """ fname='WRF_CFtime_creation' if values == 'h': print fname + '_____________________________________________________________' print WRF_CFtime_creation.__doc__ quit() refdate=values.split(',')[0] tunitsval=values.split(',')[1] wrfnc = NetCDFFile(ncfile, 'a') if wrfnc.variables.has_key(varn): print errormsg print ' ' + fname + ": WRF file '" + ncfile + "' already has variable '" + \ varn + "' !!!" quit(-1) timeobj = wrfnc.variables['Times'] timewrfv = timeobj[:] yrref=refdate[0:4] monref=refdate[4:6] dayref=refdate[6:8] horref=refdate[8:10] minref=refdate[10:12] secref=refdate[12:14] refdateS = yrref + '/' + monref + '/' + dayref + '_' + horref + ':' + minref + \ ':' + secref dt = timeobj.shape[0] cftimes = np.zeros((dt), dtype=np.float) for it in range(dt): wrfdates = datetimeStr_conversion(timewrfv[it,:],'WRFdatetime', 'matYmdHMS') cftimes[it] = realdatetime1_CFcompilant(wrfdates, refdate, tunitsval) tunits = tunitsval + ' since ' + refdateS newvar = wrfnc.createVariable('time','f4',('Time')) newattr = basicvardef(newvar, 'time','time',tunits) newvar[:] = cftimes newattr = set_attribute(newvar, 'calendar', 'standard') wrfnc.sync() wrfnc.close() return #WRF_CFtime_creation('19491201000000,hours', 'wrfout_d01_1995-01-15_00:00:00', 'time') def WRF_CFxtime_creation(values, ncfile, varn): """ Function to add a CF-convention time unit in a WRF file using variable 'XTIME' WRF_CFxtime_creation(values, ncfile, varn) [values]= [refdate],[units] [refdate]: reference date to compute the times in [YYYY][MM][DD][HH][MI][SS] format [units]: time units to use: 'days','hours','minutes','seconds' [ncfile]= WRF file to add the CF-convention time variable [varn]= name of the variable time to add """ fname='WRF_CFxtime_creation' if values == 'h': print fname + '_____________________________________________________________' print WRF_CFxtime_creation.__doc__ quit() refdate=values.split(',')[0] tunitsval=values.split(',')[1] wrfnc = NetCDFFile(ncfile, 'a') if not wrfnc.variables.has_key('XTIME'): print errormsg print ' ' + fname + ": WRF file '" + ncfile + "' does not have variable '" +\ "XTIME' !!" quit(-1) if wrfnc.variables.has_key(varn): print warnmsg print ' ' + fname + ": WRF file '" + ncfile + "' already has variable '" + \ varn + "' !!!" print ' over-writting values' if not searchInlist(wrfnc.ncattrs(), 'SIMULATION_START_DATE'): print errormsg print ' ' + fname + ": WRF file '" + ncfile + "' does not have attribute '"+\ "'SIMULATION_START_DATE' !!!" quit(-1) startdate = wrfnc.SIMULATION_START_DATE timeobj = wrfnc.variables['XTIME'] timewrfv = timeobj[:] yrref=refdate[0:4] monref=refdate[4:6] dayref=refdate[6:8] horref=refdate[8:10] minref=refdate[10:12] secref=refdate[12:14] refdateS = yrref + '-' + monref + '-' + dayref + '_' + horref + ':' + minref + \ ':' + secref dt = timeobj.shape[0] cftimes = np.zeros((dt), dtype=np.float) for it in range(dt): wrfdates = datetimeStr_conversion(str(timewrfv[it]) +',minutes since ' + \ startdate, 'cfTime', 'matYmdHMS') cftimes[it] = realdatetime1_CFcompilant(wrfdates, refdate, tunitsval) tunits = tunitsval + ' since ' + refdateS if wrfnc.variables.has_key('time'): newvar = wrfnc.variables['time'] else: newvar = wrfnc.createVariable('time','f4',('Time')) newattr = basicvardef(newvar, 'time','time',tunits) newvar[:] = cftimes newattr = set_attribute(newvar, 'calendar', 'standard') wrfnc.sync() wrfnc.close() return def insert_variable(onc, varn, varv, vardims, varvdims, onewnc): """ Function to insert a variable in an existing file insert_variable(onc, varn, varv, vardims, onewnc) [onc]= object of the original file [varn]= name of the variable [varv]= values of the variable [vardims]= dimension names of the variable [varvdims]= name of the variables with the values of the dimensions [onewnc]= object of the new file """ fname = 'insert_variable' if onc == 'h': print fname + '_____________________________________________________________' print insert_variable.__doc__ quit() ininewncdims = onewnc.dimensions ininewncvars = onewnc.variables # Creation of dimensions for idim in vardims: if not ininewncdims.has_key(idim): if not onc.dimensions.has_key(idim): print errormsg print ' ' + fname + ': initial file does not have dimension: "' + \ idim + '"' quit(-1) else: refdimv = onc.dimensions[idim] if refdimv.isunlimited(): dimsize = None else: dimsize = len(refdimv) onewnc.createDimension(idim, dimsize) # Variables with dimension values for ivdim in varvdims: if not ininewncvars.has_key(ivdim): if not onc.variables.has_key(ivdim): print errormsg print ' ' + fname + ': initial file does not have variable: "' + \ ivdim + '"' quit(-1) else: refvdimv = onc.variables[ivdim] refvdimvals = refvdimv[:] for dmn in refvdimv.dimensions: if not onewnc.dimensions.has_key(dmn): print 'dmn:',dmn if onc.dimensions[dmn].isunlimited(): newdim = onewnc.createDimension(dmn, None ) else: newdim = onewnc.createDimension(dmn,len(onc.dimensions[dmn])) newvar = onewnc.createVariable(ivdim, refvdimv.dtype, \ (refvdimv.dimensions)) varattrs = refvdimv.ncattrs() newvar[:] = refvdimvals for attrn in varattrs: attrv = refvdimv.getncattr(attrn) newattr = set_attribute(newvar, attrn, attrv) # Variable varvalues = variables_values(varn) if onewnc.variables.has_key(varvalues[0]): print warnmsg print ' ' + fname + ": file already has variable '" + varvalues[0] + "' !!" print ' re-writting values' newvar = onewnc.variables[varvalues[0]] else: newvar = onewnc.createVariable(varvalues[0], 'f4', (vardims)) newattr = basicvardef(newvar, varvalues[1], varvalues[4].replace('|',' '), \ varvalues[5]) newvar[:] = varv return def var_dim_dimv(dimvar, dimns, dimvns): """ Function to bring back which variables with the values should be use for a given variable var_dim_dimv(ovar, dimns, dimvns) [dimvar]= list of the dimensions of the variable [dimns]= names of the list of available dimensions [dimvns]= correspondant name of variables with the values for the dimensions >>> dimsvar = ['t', 'y', 'x'] >>> dimnames = ['t', 'z', 'l', 'y' , 'x'] >>> dimvarnames = ['time', 'pressure', 'soil_levels', 'lat' ,' lon'] >>> var_dim_dimv(dimsvar, dimnames, dimvarnames) ['time', 'lat', ' lon'] """ fname = 'var_dim_dimv' if dimvar == 'h': print fname + '_____________________________________________________________' print var_dim_dimv.__doc__ quit() dimvvar = list(dimvns) Ndimvar = len(dimvar) Ndimns = len(dimns) for indimv in range(Ndimns): for indima in range(Ndimvar): found = False print ' Lluis:',fname,dimns[indimv],dimvar[indima] if dimns[indimv] == dimvar[indima]: found = True break if not found: dimvvar.remove(dimvns[indimv]) return dimvvar def WRF_CFlonlat_creation(values, ncfile, varn): """ Function to add a CF-convention longitude/latitude variables in a WRF file WRF_CFlon_creation(values, ncfile, varn) [values]= [lonname],[latname] names of the variables to add (standard names 'longitude', 'latitude') [ncfile]= WRF file to add the CF-convention time variable [varn]= [lonname],[latname] name of the variables to use """ fname='WRF_CFlonlat_creation' if values == 'h': print fname + '_____________________________________________________________' print WRF_CFlonlat_creation.__doc__ quit() cflonn = values.split(',')[0] cflatn = values.split(',')[1] wlonn = varn.split(',')[0] wlatn = varn.split(',')[1] wrfnc = NetCDFFile(ncfile, 'a') if not searchInlist(wrfnc.variables.keys(),wlonn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not lon variable '" + \ wlonn + "' !!!" quit(-1) if not searchInlist(wrfnc.variables.keys(),wlatn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not lat variable '" + \ wlatn + "' !!!" quit(-1) if not searchInlist(wrfnc.dimensions.keys(),'west_east'): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not 'west_east' dimension !!!" quit(-1) if not searchInlist(wrfnc.dimensions.keys(),'south_north'): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not 'south_north' dimension !!!" quit(-1) lonobj = wrfnc.variables[wlonn] lonv = lonobj[:] latobj = wrfnc.variables[wlatn] latv = latobj[:] if len(lonv.shape) == 3: cflons = np.zeros((lonv.shape[1],lonv.shape[2]), dtype=np.float) cflons = np.where(lonv[0,:,:] < 0., lonv[0,:,:] + 360., lonv[0,:,:]) cflats = np.zeros((latv.shape[1],latv.shape[2]), dtype=np.float) cflats = latv[0,:,:] elif len(lonv.shape) == 2: cflons = np.zeros((lonv.shape), dtype=np.float) cflons = np.where(lonv < 0., lonv + 360., lonv) cflats = latv elif len(lonv.shape) == 1: cflons = np.zeros((lonv.shape), dtype=np.float) cflons = np.where(lonv < 0., lonv + 360., lonv) cflats = latv else: print errormsg print ' ' + fname + ": original longitude variable '" + varn + "' shape:" + \ lonv.shape + "not ready !!!" quit(-1) if len(lonv.shape) == 3: for ix in range(lonv.shape[2]): print ix, lonv[0,10,ix], cflons[10,ix] elif len(lonv.shape) == 2: for ix in range(lonv.shape[1]): print ix, lonv[10,ix], cflons[10,ix] if wrfnc.variables.has_key(cflonn): print warnmsg print ' ' + fname + ": file '" + ncfile + "' already has variable '" + \ cflonn + "' !!!" print ' over-writting values' newvar = wrfnc.variables[cflonn] else: newvar = wrfnc.createVariable(cflonn,'f4',('south_north','west_east')) newattr = basicvardef(newvar, 'longitude','longitude','degrees East') newvar[:] = cflons if wrfnc.variables.has_key(cflatn): print warnmsg print ' ' + fname + ": file '" + ncfile + "' already has variable '" + \ cflatn + "' !!!" print ' over-writting values' newvar = wrfnc.variables[cflatn] else: newvar = wrfnc.createVariable(cflatn,'f4',('south_north','west_east')) newattr = basicvardef(newvar, 'latitude','latitude','degrees North') newvar[:] = cflats wrfnc.sync() wrfnc.close() return #WRF_CFlon_creation('longitude', 'wrfout_d01_1980-03-01_00:00:00_Time_B0-E48-I1.nc', 'XLONG') def dimVar_creation(values, ncfile): """ Function to add a 1D variable with the size of a given dimension in a file dimVar_creation(values, ncfile) [values]= [dimname] name of the dimesion to use [ncfile]= file to add the variable """ fname='dimVar_creation' if values == 'h': print fname + '_____________________________________________________________' print dimVar_creation.__doc__ quit() ncobj = NetCDFFile(ncfile, 'a') varn = 'varDIM' + values if not searchInlist(ncobj.dimensions.keys(),values): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not '" + values + \ "' dimension !!" quit(-1) dimsize = len(ncobj.dimensions[values]) if ncobj.variables.has_key(varn): print warnmsg print ' ' + fname + ": file '" + ncfile + "' already has variable '" + \ varn + "' !!!" print ' over-writting values' newvar = ncobj.variables[varn] else: newvar = ncobj.createVariable(varn,'f4',(values)) newattr = basicvardef(newvar, varn,"variable from '" +values+ "' dimension" ,'-') newvar[:] = np.arange(dimsize)*1. ncobj.sync() ncobj.close() print fname + ": successfull creation of variable '" + varn + "' !!" return #dimVar_creation('bottom_top', 'wrfout_d01_1979-12-01_00:00:00_south_north_B3-E3-I1_west_east_B26-E26-I1.nc') def files_folder(folder,headfile): """ Function to retrieve a list of files from a folder [fold] and head [headfile] >>> files_folder('/media/data2/etudes/WRF_LMDZ/WL_HyMeX_HighRes/medic950116/', 'wrfout_d03') ['wrfout_d03_1995-01-13_02:00:00', 'wrfout_d03_1995-01-13_04:00:00', 'wrfout_d03_1995-01-13_06:00:00', 'wrfout_d03_1995-01-13_06:15:00', 'wrfout_d03_1995-01-13_08:15:00', 'wrfout_d03_1995-01-13_10:15:00', 'wrfout_d03_1995-01-13_12:15:00', 'wrfout_d03_1995-01-13_14:15:00', 'wrfout_d03_1995-01-13_16:15:00', (...) 'wrfout_d03_1995-01-17_18:15:00', 'wrfout_d03_1995-01-17_20:15:00', 'wrfout_d03_1995-01-17_22:15:00'] """ import subprocess as sub fname = 'files_folder' if folder == 'h': print fname + '_____________________________________________________________' print files_folder.__doc__ quit() # ins = folder + "/" + headfile + "*" ins = folder + "/" print 'ins:',ins files = sub.Popen(["/bin/ls","-1",ins], stdout=sub.PIPE) fileslist = files.communicate() listfiles0 = str(fileslist).split('\\n') # Filtering output Lheader=len(headfile) listfiles = [] for ifile in listfiles0: if ifile[0:Lheader] == headfile: listfiles.append(ifile) return listfiles def netcdf_fold_concatenation(values, ncfile, varn): """ Function to concatenate netCDF files in a given folder for a given set of variables netcdf_fold_concateation(values, ncfile, varn) [values]= [fold],[dimname] folder with the location of the netCDF files [fold]: folder with the location of the netCDF files [dimname]: dimension along which files should be concatenated [ncfile]= header of the name of the files to concatenate [ncfile]* [varn]= ',' separated list of variables to concatenate [var1],[var2],[...[varN]] or 'all' for all variables """ import subprocess as sub fname='netcdf_fold_concatenate' ofile = 'netcdf_fold_concatenated.nc' if values == 'h': print fname + '_____________________________________________________________' print netcdf_fold_concatenation.__doc__ quit() fold = values.split(',')[0] condim = values.split(',')[1] confiles = files_folder(fold,ncfile) Nfiles = len(confiles) print ' ' + fname +': concatenating:',Nfiles,'files' print confiles # Opening all files ncobjs = [] for filen in confiles: print 'charging: ',filen ncobjs.append(NetCDFFile(fold + '/' + filen, 'r')) # Looking for the new length of the concatenation dimension if not ncobjs[0].dimensions.has_key(condim): print errormsg print ' ' + fname + ": files do not have dimensions '" + condim + "' !!!" quit(-1) totcondim = 0 for ifile in range(Nfiles): ncobj = ncobjs[ifile] totcondim = totcondim + len(ncobjs[ifile].dimensions[condim]) print "total concatenated dimension '" + condim + "': ", totcondim # concatenated file creation ## newnc = NetCDFFile(ofile, 'w') # dimension creation ## if ncobjs[0].dimensions[condim].isunlimited(): newnc.createDimension(condim, None) else: newnc.createDimension(condim, totcondim) # Fake variable for the dimension print 'condim:',condim dims = [] dims.append(condim) newvar = newnc.createVariable('int' + condim, 'i4', tuple(dims)) newvar[:] = range(totcondim) # Looping variables ## if varn == 'all': desvars = ncobjs[0].variables else: desvars = varn.split(',') for dvar in desvars: if not ncobjs[0].variables.has_key(dvar): print errormsg print ' ' + fname + ": files do not have variable '" + dvar + "' !!!" quit(-1) for dvar in desvars: print ' ' + fname + ": concatenating '" + dvar + "'..." objvar = ncobjs[0].variables[dvar] # creation of dimensions vardims = objvar.dimensions for ndim in vardims: if not newnc.dimensions.has_key(ndim): vardimo = ncobjs[0].dimensions[ndim] if vardimo.isunlimited(): newnc.createDimension(ndim, None) else: newnc.createDimension(ndim, len(vardimo)) # creation of variable kvar = objvar.dtype newvar = newnc.createVariable(dvar, kvar, objvar.dimensions) varattrs = objvar.ncattrs() for atr in varattrs: attrv = objvar.getncattr(atr) newattr = set_attribute(newvar, atr, attrv) # variable values # vartotv = np.zeros((newvar.shape), dtype=kvar) objvar = ncobjs[0].variables[dvar] if searchInlist(list(objvar.dimensions),condim): begslicetot = 0 for ifile in range(Nfiles): slicevartot = [] if ncobjs[ifile].variables.has_key(dvar): objvar = ncobjs[ifile].variables[dvar] for dimn in objvar.dimensions: ldimfile = len(ncobjs[ifile].dimensions[dimn]) if dimn == condim: slicevartot.append(slice(begslicetot,begslicetot+ldimfile)) begslicetot = begslicetot + ldimfile else: slicevartot.append(slice(0,ldimfile)) newvar[tuple(slicevartot)] = objvar[:] newnc.sync() else: print errormsg print ' ' + fname + ": file '" + fold + '/' + confiles[ifile] + \ "' does not have variable '" + dvar + "' !!" quit(-1) else: newvar[:] = objvar[:] newnc.sync() # newvar[:] = vartotv gattrs = ncobjs[0].ncattrs() for attr in gattrs: attrv = ncobjs[0].getncattr(attr) newattr = set_attribute(newnc, attr, attrv) for ifile in range(Nfiles): ncobjs[ifile].close() newnc.sync() newnc.close() print "Successfull creation of concatenated file '" + ofile + "' !!!" return #netcdf_fold_concatenation('/media/data2/etudes/WRF_LMDZ/WL_HyMeX_HighRes/medic950116,Time', 'wrfout_d03', 'Times,XLONG,XLAT,T2,TH2,PSFC,U10,V10,MAPFAC_M') def index_mat(mat,val): """ Function to provide the coordinates of a given value inside a matrix index_mat(mat,val) mat= matrix with values val= value to search >>> index_mat(np.arange(27).reshape(3,3,3),22) [2 1 1] """ fname = 'index_mat' matshape = mat.shape matlist = list(mat.flatten()) ifound = matlist.index(val) Ndims = len(matshape) valpos = np.zeros((Ndims), dtype=int) baseprevdims = np.zeros((Ndims), dtype=int) for dimid in range(Ndims): baseprevdims[dimid] = np.product(matshape[dimid+1:Ndims]) if dimid == 0: alreadyplaced = 0 else: alreadyplaced = np.sum(baseprevdims[0:dimid]*valpos[0:dimid]) valpos[dimid] = int((ifound - alreadyplaced )/ baseprevdims[dimid]) return valpos def netcdf_concatenation(ncfile): """ Function to concatenate netCDF files for a given set of variables netcdf_concateation(values, ncfile, varn) [ncfile]= [filename1]@[varn1]|[filename1]@[varn1] '|' list of file names and variables to concatenate [varn]= ',' separated list of variables to concatenate [var1],[var2],[...[varN]] or 'all' for all variables """ fname='netcdf_concatenate' ofile = 'netcdf_concatenated.nc' if ncfile == 'h': print fname + '_____________________________________________________________' print netcdf_concatenation.__doc__ quit() confiles = ncfile.split('|') Nfiles = len(confiles) print ' ' + fname +': concatenating:',Nfiles,'files' print confiles # Opening all files filenames = [] ncobjs = [] for files in confiles: filen = files.split('@')[0] filenames.append(filen) if not os.path.isfile(filen): print errormsg print ' ' + fname + ": file '" + filen + "' does no exist!!" quit(-1) print 'charging: ',filen ncobjs.append(NetCDFFile(filen, 'r')) # concatenated file creation ## newnc = NetCDFFile(ofile, 'w') # Looping variables ## ifile = 0 for files in confiles: filen = files.split('@')[0] desvar = files.split('@')[1] if desvar == 'all': desvars = ncobjs[ifile].variables else: desvars = desvar.split(',') for dvar in desvars: if not ncobjs[ifile].variables.has_key(dvar): print errormsg print ' ' + fname + ": files do not have variable '" +dvar+ "' !!!" quit(-1) for dvar in desvars: if not newnc.variables.has_key(dvar): print ' ' + fname + ": concatenating '" + dvar + "' from '" + filen+\ "'..." objvar = ncobjs[ifile].variables[dvar] # creation of dimensions vardims = objvar.dimensions for ndim in vardims: if not newnc.dimensions.has_key(ndim): vardimo = ncobjs[ifile].dimensions[ndim] if vardimo.isunlimited(): newnc.createDimension(ndim, None) else: newnc.createDimension(ndim, len(vardimo)) # creation of variable kvar = objvar.dtype newvar = newnc.createVariable(dvar, kvar, objvar.dimensions) varattrs = objvar.ncattrs() for atr in varattrs: attrv = objvar.getncattr(atr) newattr = set_attribute(newvar, atr, attrv) # variable values # vartotv = np.zeros((newvar.shape), dtype=kvar) objvar = ncobjs[ifile].variables[dvar] newvar[:] = objvar[:] newnc.sync() # newvar[:] = vartotv ifile = ifile + 1 gattrs = ncobjs[0].ncattrs() for attr in gattrs: attrv = ncobjs[0].getncattr(attr) newattr = set_attribute(newnc, attr, attrv) filesnvec = numVector_String(filenames,', ') newattr = set_attribute(newnc, 'concatenated', filesnvec) for ifile in range(Nfiles): ncobjs[ifile].close() newnc.sync() newnc.close() print "Successfull creation of concatenated file '" + ofile + "' !!!" return #netcdf_concatenation('mean_pluc.nc@all|mean_dtcon.nc@all') def field_stats(values, ncfile, varn): """ Function to retrieve statistics from a field field_stats(values, ncfile, varn) [values]= kind of statistics 'full': all statistics given variable [ncfile]= name of the netCDF file to use [varn]= variable name to use ('all' for all variables) """ fname='field_stats' if values == 'h': print fname + '_____________________________________________________________' print field_stats.__doc__ quit() ncobj = NetCDFFile(ncfile, 'r') if varn == 'all': varstats = list(ncobj.variables) else: varstats = [varn] print ' ' + fname + '______ ______ _____ ____ ___ __ _' for vn in varstats: if not ncobj.variables.has_key(vn): print errormsg print ' ' + fname + ": file do not have variable '" + vn + "' !!" quit(-1) objfield = ncobj.variables[vn] field = objfield[:] print ' ' + vn + '... .. .' if values == 'full': minv = np.min(field) maxv = np.max(field) meanv = np.mean(field) mean2v = np.mean(field*field) varv = np.sqrt(mean2v - meanv*meanv) print ' Fstats shape:',field.shape print ' Fstats min:', minv print ' Fstats max:', maxv print ' Fstats mean:', meanv print ' Fstats mean2:', mean2v print ' Fstats variability:', varv else: print errormsg print ' ' + fname + ": kind of statistics '" + values + "' not ready!!" print ' valid statistics: full' quit(-1) ncobj.close() return #field_stats('full', 'geo_em.d01.nc', 'HGT_M') def check_arguments(funcname,Nargs,args,char,expectargs): """ Function to check the number of arguments if they are coincident check_arguments(funcname,Nargs,args,char) funcname= name of the function/program to check Nargs= theoretical number of arguments args= passed arguments char= character used to split the arguments """ fname = 'check_arguments' Nvals = len(args.split(char)) if Nvals != Nargs: print errormsg print ' ' + fname + ': wrong number of arguments:',Nvals," passed to '", \ funcname, "' which requires:",Nargs,'!!' print ' given arguments:',args.split(char) print ' expected arguments:',expectargs quit(-1) return def Str_Bool(val): """ Function to transform from a String value to a boolean one >>> Str_Bool('True') True >>> Str_Bool('0') False >>> Str_Bool('no') False """ fname = 'Str_Bool' if val == 'True' or val == '1' or val == 'yes': boolv = True elif val == 'False' or val == '0' or val== 'no': boolv = False else: print errormsg print ' ' + fname + ": value '" + val + "' not ready!!" quit(-1) return boolv def get_namelist_vars(values, namelist): """ Function to get namelist-like values ([varname] = [value]) get_namelist_vars(namelist) values= [sectionname],[kout] [sectionname]: name of the section from which one want the values ('all', for all) [kout]: kind of output 'tex3': printed output as LaTeX table of three columns \verb+namelist_name+ & value 'column': printed output as namelist_name value 'dict': as two python dictionary object (namelistname, value and namelistname, sectionname) namelist= namelist_like file to retrieve values >>> get_namelist_vars('geogrid,dic','/home/lluis/etudes/domains/medic950116/namelist.wps') {'e_sn': '32, 97,', 'stand_lon': '0.', 'opt_geogrid_tbl_path': "'./'", 'geog_data_path': "'/home/lluis/DATA/WRF/geog'", 'pole_lat': '90.0', 'ref_lat': '35.0,', 'map_proj': "'lat-lon',", 'parent_id': '1, 1,', 'geog_data_res': "'2m','2m',", 'e_we': '32, 112,', 'dx': '0.35,', 'dy': '0.35,', 'parent_grid_ratio': '1, 3,', 'pole_lon': '0.0', 'ref_lon': '20,', 'j_parent_start': '1, 17,', 'i_parent_start': '1, 31,'} """ fname = 'get_namelist_vars' # List of characters which split pairs name/value valuessep = ['='] # List of characters which indicate a comment commentchars = ['#'] if values == 'h': print fname + '_____________________________________________________________' print get_namelist_vars.__doc__ quit() expectargs = '[sectionname],[kout]' check_arguments(fname,len(expectargs.split(',')),values,',',expectargs) secname = values.split(',')[0] kout = values.split(',')[1] if not os.path.isfile(namelist): print errormsg print ' ' + fname + ": namelist file '" + namelist + "' does not exist !!" quit(-1) ncml = open(namelist, 'r') sections = {} namelistvals = {} namelistsecs = {} sectionnames = [] namessec = [] allnames = [] namelistvalssec = {} namelistsecssec = {} nmlname = '' sectionname = '' for line in ncml: linevals = reduce_spaces(line) Nvals = len(linevals) if Nvals >= 1 and linevals[0][0:1] == '&': if len(sectionnames) > 1: sections[sectionname] = namessec sectionname = linevals[0][1:len(linevals[0])+1] # print ' ' + fname + ": new section '" + sectionname + "' !!!" sectionnames.append(sectionname) namessec = [] nmlname = '' elif Nvals >= 1 and not searchInlist(commentchars,linevals[0][0:1]): if Nvals >= 3 and searchInlist(valuessep,linevals[1]): nmlname = linevals[0] nmlval = numVector_String(linevals[2:Nvals],' ') elif Nvals == 1: for valsep in valuessep: if linevals[0].find(valsep) != -1: nmlname = linevals[0].split(valsep)[0] nmlval = linevals[0].split(valsep)[1] break elif Nvals == 2: for valsep in valuessep: if linevals[0].find(valsep) != -1: nmlname = linevals[0].split(valsep)[0] nmlval = linevals[1] break elif linevals[1].find(valsep) != -1: nmlname = linevals[0] nmlval = linevals[1].split(valsep)[0] break else: print warnmsg print ' ' + fname + ': wrong number of values', Nvals, \ 'in the namelist line!' print ' line: ',line print ' line values:',linevals # quit(-1) namelistvals[nmlname] = nmlval namelistsecs[nmlname] = sectionname namessec.append(nmlname) allnames.append(nmlname) if len(sectionname) > 1: sections[sectionname] = namessec if secname != 'all': if not searchInlist(sections.keys(),secname): print errormsg print ' ' + fname + ": section '" + values + "' does not exist !!" print ' only found:',sectionnames quit(-1) namestouse = [] for nml in allnames: for nnml in sections[secname]: namelistvalssec[nnml] = namelistvals[nnml] namelistsecssec[nnml] = secname if nml == nnml: namestouse.append(nml) else: namestouse = allnames namelistvalssec = namelistvals namelistsecssec = namelistsecs if kout == 'tex3': ofile='get_namelist_vars_3col.tex' fobj = open(ofile, 'w') vals = namestouse Nvals = len(vals) Nvals3 = int(Nvals/3) latextab = '\\begin{center}\n\\begin{tabular}{lclclc}\n' for il in range(2): latextab = latextab + '{\\bfseries{name}} & {\\bfseries{value}} & ' latextab= latextab+ '{\\bfseries{name}} & {\\bfseries{value}} \\\\ \\hline\n' if np.mod(Nvals,3) != 0: Nvals0 = Nvals - np.mod(Nvals,3) else: Nvals0 = Nvals for ival in range(0,Nvals0,3): line = '' print ' ival:',ival for il in range(2): line = line + '\\verb+' + vals[ival+il] + '+ & ' + \ namelistvalssec[vals[ival+il]].replace('_','\\_') +' & ' line = line + '\\verb+' + vals[ival+2] + '+ & ' + \ namelistvalssec[vals[ival+2]].replace('_','\\_') + ' \\\\\n' latextab = latextab + line latextab = latextab + '%not multiple of three!!!!\n' print 'mod:', np.mod(Nvals,3),Nvals0,Nvals if np.mod(Nvals,3) != 0: ival = Nvals0 line = '' for il in range(np.mod(Nvals,3)): print 'ival:',ival + il line = line + '\\verb+' + vals[ival+il] + '+ & ' + \ namelistvalssec[vals[ival+il]].replace('_','\\_') + ' & ' for il in range(2-np.mod(Nvals,3)): line = line + ' & & ' latextab = latextab + line + ' & \\\\\n' latextab = latextab + '\\end{tabular}\n\\end{center}\n' # print latextab fobj.write(latextab) fobj.close() print fname + ": successful writen '" + ofile + "' LaTeX tale file !!" return elif kout == 'column': for dictv in namestouse: print dictv + ' = ' + namelistvalssec[dictv] return elif kout == 'dict': return namelistvalssec, namelistsecssec else: print errormsg print ' ' + fname + ": kind of output '" + kout + "' not ready!!!" quit(-1) return def WRF_d0Nref(values, geofold): """ Function for the generation of an extra WRF domain from a given one field_stats(values, ncfile, varn) [values]= [namelist],[minLON],[minLAT],[maxLON],[maxLAT],[Ndomref],[factor],[geogres] [namelist]: namelist.wps to use [minLON],[minLAT]: minimum longitude and latitude [maxLON],[maxLAT]: maximum longitude and latitude [Ndomref]: domain reference number [factor]: resolution factor with respect d01 [geogres]: which topography resolution to use [geofold]= folder where to found the geo_em.d0[N-1].nc file """ fname='WRF_d0Nref' if values == 'h': print fname + '_____________________________________________________________' print WRF_d0Nref.__doc__ quit() expectargs = '[namelist],[minLON],[minLAT],[maxLON],[maxLAT],[Ndomref],[factor],' expectargs = expectargs + '[geogres]' check_arguments(fname,len(expectargs.split(',')),values,',',expectargs) namelist = values.split(',')[0] minLON = np.float(values.split(',')[1]) minLAT = np.float(values.split(',')[2]) maxLON = np.float(values.split(',')[3]) maxLAT = np.float(values.split(',')[4]) Ndomref = int(values.split(',')[5]) factor = int(values.split(',')[6]) geogres = values.split(',')[7] onml = 'namelist.wps_d0' + str(Ndomref+1) # Namelist variables to which new value is a copy of the last one repeatvalue = ['start_date' ,'end_date'] # Namelist variables to which value is increased by 1 (single value) add1 = ['max_dom'] # Namelist variables to which new value is an increase by 1 of the last one add1value = ['parent_id'] # ncfile = geofold + '/geo_em.d' + zeros(Ndomref,2) + '.nc' ncfile = geofold + '/geo_em.d0' + str(Ndomref) + '.nc' if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': geo_em file "' + ncfile + '" does not exist !!' quit(-1) ncobj = NetCDFFile(ncfile, 'r') nlon='XLONG_M' nlat='XLAT_M' lonobj = ncobj.variables[nlon] latobj = ncobj.variables[nlat] lon = lonobj[0,:,:] lat = latobj[0,:,:] ncobj.close() diffmin = np.sqrt((lon - minLON)**2. + (lat - minLAT)**2.) diffmax = np.sqrt((lon - maxLON)**2. + (lat - maxLAT)**2.) posdiffmin = index_mat(diffmin,np.min(diffmin)) posdiffmax = index_mat(diffmax,np.min(diffmax)) # print 'min vals. min:',np.min(diffmin),'max:',np.min(diffmax) # print 'min:', minLON, ',', minLAT, ':', posdiffmin, '.', lon[tuple(posdiffmin)], \ # ',', lat[tuple(posdiffmin)] # print 'max:', maxLON, ',', maxLAT, ':', posdiffmax, '.', lon[tuple(posdiffmax)], \ # ',', lat[tuple(posdiffmax)] print ' ' + fname +': coordinates of the SW corner:', posdiffmin, ' NE corner:',\ posdiffmax nmlobj = open(namelist, 'r') onmlobj = open(onml, 'w') for line in nmlobj: listline = reduce_spaces(line) nvals = len(listline) if nvals > 0: if searchInlist(add1,listline[0]): nmlval = int(listline[nvals-1].replace(',','')) + 1 onmlobj.write(' ' + listline[0] + ' = ' + str(nmlval) + ',\n') elif searchInlist(add1value,listline[0]): if Ndomref == 1: nmlval = 1 else: nmlval = int(listline[nvals-1].replace(',','')) + 1 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1], ' ') + ' ' +str(nmlval)+',\n') elif searchInlist(repeatvalue,listline[0]): nmlval = listline[nvals-1] onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+'\n') elif listline[0] == 'parent_grid_ratio': nmlval = factor onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'i_parent_start': nmlval = posdiffmin[1] - 10 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'j_parent_start': nmlval = posdiffmin[0] - 10 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'e_we': nmlval = (posdiffmax[1] - posdiffmin[1] + 20)*factor nmlval = nmlval + np.mod(nmlval,factor) + 1 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'e_sn': nmlval = (posdiffmax[0] - posdiffmin[0] + 20)*factor nmlval = nmlval + np.mod(nmlval,factor) + 1 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'geog_data_res': nmlval = "'" + geogres + "'" onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') else: onmlobj.write(line) else: onmlobj.write('\n') nmlobj.close() onmlobj.close() print fname + ": successfully written new namelist '" + onml + "' !!" return def filter_2dim(values, ncfile, varn): """ Function to filter along 2 dimensions (moving grid-box means of a given size) values of the netCDF file filter_2dim(values, ncfile, variable) values= [Ngrid],[dimnh1],[dimnh2],[dimvnh1],[dimvnh2] Ngrid: box size to make means dimnh[1/2]: name of the 2 dimensions to make the averages dimvnh[1/2]: name of the 2 variables with the values of the dimensions to make the averages ncfile= netCDF file to use varn= name of the variable to filter ('all' for all variables) filter_2dim('80,y,x,lon,lat', 'tahiti_5m_ll.grd', 'z') """ fname = 'filter_2dim' if values == 'h': print fname + '_____________________________________________________________' print filter_2dim.__doc__ quit() expectargs = '[Ngrid],[dimnh1],[dimnh2],[dimvnh1],[dimvnh2]' check_arguments(fname,len(expectargs.split(',')),values,',',expectargs) Ngrid = int(values.split(',')[0]) dimnh1 = values.split(',')[1] dimnh2 = values.split(',')[2] dimvnh1 = values.split(',')[3] dimvnh2 = values.split(',')[4] Ngrid2 = Ngrid/2 dimhorns = [dimnh1, dimnh2] dimvhorns = [dimvnh1, dimvnh2] if varn == 'all': varns = ncobj.variables ofile = 'filter_hor.nc' else: varns = [varn] ofile = 'filter_hor_' + varn + '.nc' ncobj = NetCDFFile(ncfile,'r') newnc = NetCDFFile(ofile, 'w') for vn in varns: varvobj = ncobj.variables[vn] vardims = varvobj.dimensions print ' ' + fname + ' var:', vn, 'variable shape:', varvobj.shape for dimn in vardims: dimobj = ncobj.dimensions[dimn] if not newnc.dimensions.has_key(dimn): if dimobj.isunlimited(): newnc.createDimension(dimn, None) else: newnc.createDimension(dimn, len(dimobj)) if searchInlist(varvobj.ncattrs(),'_FillValue'): vfillValue = varvobj._FillValue else: vfillValue = fillValue vtype = varvobj.dtype if searchInlist(dimvhorns,vn): print ' ' + fname + ': dimension values variable!' print " keeping original values as '" + vn + "' and filtered as '" + \ vn+'_flt'+str(Ngrid)+'x'+str(Ngrid) + "'" newvar = newnc.createVariable(vn, vtype, vardims) newvar[:] = varvobj[:] vattr = varvobj.ncattrs() for nattr in vattr: attrv = varvobj.getncattr(nattr) if nattr != '_FillValue': newattr = set_attribute(newvar,nattr,attrv) newvar = newnc.createVariable(vn+'_flt'+str(Ngrid)+'x'+str(Ngrid), \ vtype, vardims) else: newvar = newnc.createVariable(vn, vtype, vardims, fill_value = fillValue) if searchInlist(vardims,dimnh1) or searchInlist(vardims,dimnh2): varv = varvobj[:] varfilt = varvobj[:] dimh1id = -1 dimh2id = -1 if searchInlist(vardims,dimnh1): dimh1id= vardims.index(dimnh1) if searchInlist(vardims,dimnh2): dimh2id= vardims.index(dimnh2) if dimh1id != -1 and dimh2id != -1: print ' ' + fname + ': 2D filtered variable' dh1 = varvobj.shape[dimh1id] dh2 = varvobj.shape[dimh2id] for i in range(Ngrid2,dh1-Ngrid2+1): for j in range(Ngrid2,dh2-Ngrid2+1): percendone(i*(dh1-Ngrid)*1.+j,(dh1-Ngrid)*(dh2-Ngrid)*1., 1., \ vn + ' filtered: ' + str(i) + ', ' + str(j)) varslice = [] varslicesum = [] for dimn in vardims: if dimn == dimnh1: varslicesum.append(slice(i-Ngrid2,i+Ngrid2+1)) varslice.append(slice(i,i+1)) elif dimn == dimnh2: varslicesum.append(slice(j-Ngrid2,j+Ngrid2+1)) varslice.append(slice(j,j+1)) else: varslicesum.append(slice(0, \ len(ncobj.dimensions[dimn]))) varslice.append(slice(0, len(ncobj.dimensions[dimn]))) # Doing nothing for _FillValue values varvalssum = varv[tuple(varslicesum)] if varvalssum.mask.all(): varfilt[tuple(varslice)] = fillValue else: numberValues = len(varvalssum.compressed()) if dimh1id < dimh2id: varfilt[tuple(varslice)] = np.sum(np.sum(varvalssum, \ axis=dimh1id), axis=dimh2id-1)/(numberValues*1.) else: varfilt[tuple(varslice)] = np.sum(np.sum(varvvlssum, \ axis=dimh2id), axis=dimh1id-1)/(numberValues*1.) print ' ' + fname + ': 2D Finished!!' elif dimh1id != -1 and dimh2id == -1: print ' ' + fname + ': 1D filtered variable' for i in range(Ngrid2,varvobj.shape[dimh1id]-Ngrid2+1): percendone(i,(dh1-Ngrid), 1, vn + ' filtered') varslice = [] varslicesum = [] for dimn in vardims: if dimn == dimhn1: varslicesum.append(slice(i-Ngrid2,i+Ngrid2+1)) else: varslicesum.append(slice(0, len(ncobj.dimensions[dimn]))) varvalssum = varv[tuple(varslicesum)] if varvalssum.mask.all(): varfilt[tuple(varslice)] = fillValue else: numberValues = len(varvalssum.compressed()) varfilt[tuple(varslice)] = np.sum(varvalssum, \ axis=dimh1id)/numberValues*1. print ' ' + fname + ': 1D Finished!!' elif dimh2id != -1 and dimh1id == -1: print ' ' + fname + ': 1D filtered variable' for j in range(Ngrid2,varvobj.shape[dimh2id]-Ngrid2+1): percendone(j,(dh2-Ngrid), 1, vn + ' filtered') varslice = [] varslicesum = [] for dimn in vardims: if dimn == dimhn2: varslicesum.append(slice(j-Ngrid2,j+Ngrid2+1)) else: varslicesum.append(slice(0, len(ncobj.dimensions[dimn]))) varvalssum = varv[tuple(varslicesum)] if varvalssum.mask.all(): varfilt[tuple(varslice)] = fillValue else: numberValues = len(varvalssum.compressed()) varfilt[tuple(varslice)] = np.sum(varvalssum, \ axis=dimh2id)/numberValues print ' ' + fname + ': 1D Finished!!' newvar[:] = varfilt else: newvar[:] = varvobj[:] vattr = varvobj.ncattrs() for nattr in vattr: attrv = varvobj.getncattr(nattr) if nattr != '_FillValue': newattr = set_attribute(newvar, nattr, attrv) gattr = ncobj.ncattrs() for nattr in gattr: attrv = ncobj.getncattr(nattr) newattr = set_attribute(newnc, nattr, attrv) ncobj.close() newnc.sync() newnc.close() print fname + ": successfull creation of '" + ofile + "' !!!" return #filter_2dim('80,y,x,lon,lat', 'tahiti_5m_ll.grd', 'z') def ncreplace(values, ncfile, varn): """ Function to replace something from a netCDF file values= [kind],[kindn],[netcdffileref] kind: element to substitute 'var': variable 'vattr': variable attributes 'gattr': global attributes kindn: name of the 'something' to replace ('all', for all possible values, [varn]@[attrn] for attribute of a given variable) netcdffileref: netCDF file from which replace values will be taken ncfile= netCDF file to replace something varn= name to use (accordingly to '[kind]') ncreplace('var,HGT_M,Polynesie_filtered_nn.nc', 'geo_em.d04.nc', 'HGT_M') """ fname = 'ncreplace' if values == 'h': print fname + '_____________________________________________________________' print ncreplace.__doc__ quit() expectargs = '[kind],[kindn],[netcdffileref]' check_arguments(fname,len(expectargs.split(',')),values,',',expectargs) kind = values.split(',')[0] kindn = values.split(',')[1] netcdffileref = values.split(',')[2] print ' ' + fname + ": replacing '" + kind + "' in '" + ncfile + "' using '" + \ netcdffileref + "' ..." if not os.path.isfile(netcdffileref): print errormsg print ' ' + fname + ": reference file '" + netcdffileref + \ "' does not exist !!" quit(-1) ncrefobj = NetCDFFile(netcdffileref, 'r') ncobj = NetCDFFile(ncfile, 'a') if kind == 'var' and kindn != 'all': if not ncrefobj.variables.has_key(kindn): print errormsg print ' ' + fname + ": reference file does not have variable '" + kindn \ + "' !!" print ' it has: ',ncrefobj.variables.keys() quit(-1) elif kind == 'vattr' and kindn != 'all': if kindn.index('@') == -1: print errormsg print ' ' + fname + ": an pair [var]@[attrn] is not given! '" + kindn + \ + "' !!" quit(-1) varn = kindn.split('@')[0] attrn = kindn.split('@')[1] if not ncrefobj.variables.has_key(varn): print errormsg print ' ' + fname + ": reference file does not have variable '" + varn +\ "' !!" quit(-1) varobj = ncrefobj.variables[varn] if not searchInlist(varobj.ncattrs(), attrn): print errormsg print ' ' + fname + ": reference variable '" + varn + \ "' does not have attribute '" + attrn + "' !!" quit(-1) else: print errormsg print ' ' + fname + ": kind to replace '" + kind + "' not ready !!" quit(-1) if kind == 'var': if kindn == 'all': varrefns = ncrefobj.variables varns = ncobj.variables else: varrefns = [kindn] varns = [varn] # On kindn= 'all' & varn= 'all' one has to assume same variables' names for vnref in varrefns: if not ncrefobj.variables.has_key(vnref): print errormsg print ' ' + fname + ": reference file does not have variable '" + \ vnref + "' !!" quit(-1) for vn in varns: if not ncobj.variables.has_key(vn): print errormsg print ' ' + fname + ": file does not have variable '" + vn + \ "' !!" quit(-1) if kindn == 'all' and vnref == vn: print ' ' + fname + ": replacing all now '" + vn + "' with '"+\ vnref + "' ..." varv = ncobj.variables[vn] varrefv = ncrefobj.variables[vnref][:] varv[:] = varrefv ncobj.sync() break else: vdimns = numVector_String(ncobj.variables[vn].shape,':') vrefdimns = numVector_String(ncrefobj.variables[vnref].shape,':') if vdimns != vrefdimns: print errormsg print ' ' + fname + ': values can not be replaced!' print ' different shapes! original:', \ ncobj.variables[vn].shape,'reference:', \ ncrefobj.variables[vnref].shape # In case of HGT_M... # quit(-1) quit(-1) print ' ' + fname + ": replacing '" + vn + "' with '" + \ vnref + "' ..." varv = ncobj.variables[vn] varrefv = ncrefobj.variables[vnref][:] # In case of HGT_M... varv[0,:,:] = varrefv varv[:] = varrefv ncobj.sync() else: print errormsg print ' ' + fname + ": kind to replace '" + kind + "' not exactly ready !!" quit(-1) ncobj.sync() ncobj.close() ncrefobj.close() return #quit() """operation to make: addfattr, add a attributes to a variable from another file: addfattr -S [file]:[var] addfdim, add a new dimension from another file: addfdim -S [file]:[dimension] addfgattr, add global attribute from another file: addfgattr -S [file] addfvar, add a new variable from another file: addfvar -S [file]:[variable] addgattr, add a new global attribute: addatr -S [attrname]|[attrvalue] addgattrk, add a new global attribute: addatr -S [attrname]|[attrvalue]|[kind(S (!, white spaces),I,R,D)] addref, add a new variable with dimension and attributes from an already existing 'variable ref' in the file: addref -S [variable ref]:[attr name]@[value]:[[attr2]@[value2], ...]:[value/file with values] mname, modify name -S newname addvals, Function to add values to a given variable at a given dimension: addvals -S [dimension]:[position]:[Nvals]:[addval], *[position]: position within the dimension at which the variable wants to be increased 'first': first position of the dimension 'middle': middle position of the dimension 'last': last position of the dimension 'num': before given value *[addval]: value to give to the added values 'val': a given value 'missing': as a missing value addvattr, add a new attribute to any given variable: addvattr -S [attrname]|[attrvalue] addvattrk, add a new attribute to any given variable: addvattrk -S [attrname]|[attrvalue]|[kind(S (!, white spaces),I,R,D)] checkNaNs, checks for NaN values over all variables in a file checkAllValues, check for variables with along all their dimensions with the same value in a file flipdim, flips the value following one dimension: flipdim -S Ndim infgattrs, give the values of all the global attributes of the file: infgattrs infsinggattrs, give the value of a single global attribute of the file: infsinggattrs -S [attribute name] infsingvattrs, give the value of a single attribute of the variable: infsingvattrs -S [attribute name] inftime, give all the information of the variable time: inftime infvars, give the names of all the variables of the file: infvars infvattrs, give the values of all the attributes of a variable: infvattrs mdname, modify dimension name: mdname -S olddname:newdname means, computes the spatial mean of the variable: means meanseas, compute the seasonal mean of a variable: meanseas -S [timename] meant, computes the temporal mean of the variable: meant -S [power](power of the polynomial fit) mname, modify name: mname -S newname out, output values: out -S inidim1,[inidim2,...]:enddim1,[enddim2,...] rgattr, read a global attribute: rgattr -S [attrname] rvattr, read a variable attribute: rvattr -S [attrname] rmgattr, remove a global attribute: rmgattr -S [attrname] rmvariable, remove a variable: rmvariable rmvattr, remove an attribute to any given variable: rmvattr -S [attrname] subsetmns, retrieve a subset of values based on months: subsetmns -S [MM1]:[...:[MMn]] or [MMi]-[MMe] for a period (output as 'subsetmns.nc') subsetyrs, retrieve a subset of values based on years: subsetyrs -S [YYYY1]:[...:[YYYYn]] or [YYYYi]-[YYYYe] for a period (output as 'subsetyrs.nc') timescheck, checks time-steps of a given file: timescheck -S [FirstDate]:[LastDate][freq]:[[auxmonth]] [FirstDate] = first date within file (in [YYYY][MM][DD][HH][MI][SS] format) [LastDate] = last date within file (in [YYYY][MM][DD][HH][MI][SS] format) [freq] = frequency of the data within the file (in time units) or 'month' (monthly values) [auxmonth] = in case of 'month' midmon: dates are given according to the mid time value of the month real: dates are given increassing a month from the previous date timesort, sort a variable time: timesort timestepchg, changes a time-step value from a file, using a given time-step from another one: timestepchg -S [origtime_step]:[newfile]:[newtime_step] valmod, modifiy values of variable -S [modification]: sumc,[constant]: add [constant] to variables values subc,[constant]: substract [constant] to variables values mulc,[constant]: multipy by [constant] to variables values divc,[constant]: divide by [constant] to variables values lowthres,[threshold],[newvalue]: modify all values below [threshold] to [newvalue] upthres,[threshold],[newvalue]: modify all values above [threshold] to [newvalue] valmod_dim(values, ncfile, varn) vartimeshidft, temporaly shift a number of time-steps a given variable inside a netCDF file: vartimeshift -S [nsteps]:[[FillValue]] varmask, mask a variable using a mask: varmask -S [maskfilename]:[maskvar]:[dimsmask]:[timename]. It is assumed that mask[...,dimM,dimJ,dimK] and var[...,dimM,dimJ,dimK] share the last dimensions 'addfattr': fattradd(opts.varname, opts.values, opts.ncfile) 'addfdim': fdimadd(opts.values, opts.ncfile) 'addfgattr': fgaddattr(opts.values, opts.ncfile) 'addfvar': fvaradd(opts.values, opts.ncfile) 'addgattr': gaddattr(opts.values, opts.ncfile) 'addgattrk': gaddattrk(opts.values, opts.ncfile) 'addref': varaddref(opts.values, opts.ncfile, opts.varname) 'addvattr': varaddattr(opts.values, opts.ncfile, opts.varname) 'addvattrk': varaddattrk(opts.values, opts.ncfile, opts.varname) 'DataSetSection': datasetsection(opts.values,opts.ncfile) 'dimflip': flipdim(opts.values,opts.ncfile,opts.varname) 'infgattrs': igattr(opts.ncfile) 'infsinggattrs': isgattrs(opts.values, opts.ncfile) 'infsingvattrs': isvattrs(opts.values, opts.ncfile, opts.varname) 'inftime': timeinf(opts.ncfile, opts.varname) 'infvars': ivars(opts.ncfile) 'infvattrs': ivattr(opts.ncfile, opts.varname) 'mdname': chdimname(opts.values, opts.ncfile, opts.varname) 'means': spacemean(opts.ncfile, opts.varname) 'meanseas': spacemean(opst.values, opts.ncfile, opts.varname) 'meant': timemean(opts.values, opts.ncfile, opts.varname) 'mname': chvarname(opts.values, opts.ncfile, opts.varname) 'out': varout(opts.values, opts.ncfile, opts.varname) 'rgattr': grattr(opts.values, opts.ncfile) 'rvattr': vrattr(opts.values, opts.ncfile, opts.varname) 'rmgattr': grmattr(opts.values, opts.ncfile) 'rmvariable': varrm(opts.ncfile, opts.varname) 'rmvattr': varrmattr(opts.values, opts.ncfile, opts.varname) 'subsetmns': submns(opts.values, opts.ncfile, opts.varname) 'subsetyrs': subyrs(opts.values, opts.ncfile, opts.varname) 'timescheck': check_times_file(opts.values, opts.ncfile, opts.varname) 'timesort': sorttimesmat(opts.ncfile, opts.varname) 'timestepchg': chgtimestep(opts.values, opts.ncfile, opts.varname) 'valsadd': addvals(opts.values, opts.ncfile, opts.varname) 'valmod': valmod(opts.values, opts.ncfile, opts.varname) 'varmask': maskvar(opts.values, opts.ncfile, opts.varname) 'vartimeshift': timeshiftvar(opts.values, opts.ncfile, opts.varname) """