# Python tools to manage netCDF files. # From L. Fita work in different places: CCRC (Australia), LMD (France) # More information at: http://www.xn--llusfb-5va.cat/python/PyNCplot # # pyNCplot and its component nc_var_tools.py comes with ABSOLUTELY NO WARRANTY. # This work is licendes under a Creative Commons # Attribution-ShareAlike 4.0 International License (http://creativecommons.org/licenses/by-sa/4.0) # import numpy as np from netCDF4 import Dataset as NetCDFFile import os import re import numpy.ma as ma # Importing generic tools file 'generic_tools.py' import generic_tools as gen import subprocess as sub ####### Contents: # addDim: Operation to add a new dimension in a file # add_dims: Function to add dimensions from a given netCDF object to another one # add_globattrs: Function to add global attributes from a given netCDF object to another one # add_global_PyNCplot: Function to add global attributes from 'PyNCplot' to a given netCDF # add_varattrs: Function to add variables attributes from a given netCDF object to other variables in another one # add_vars: Function to add variables from a given netCDF object to another one # addVar: Operation to add a new variable in a file using exisintg dimensions # addvals: Function to add values to a given variable at a given dimension # attrinvar: Check existence of an attribute in a netCDF variable object # basicvardef: Function to give the basic attributes to a variable (std_name, long_name, units) # CDO_toCF: Function to pass a CDO output file to CF-conventions # changevartype: Function to change the type of a variable (when possible) # chdimname: Changing the name of the dimension # checkAllValues: Function to check for variables with along all their dimensions with the same value in a file # checkallvars: Function to check all variables of a file # checkNaNs: Function to check for NaN values over all variables in a file # check_times_file: Function to check time-steps of a given file # chgtimestep: Function to change the values of a given time-step of a variable inside a netCDF for values from a nother file # chvarname: Changing the name of the variable # cleaning_varsfile: Function to keep a list of varibales from a file # cls_time_information: Classs to provide information about variable time # compute_deaccum: Function to compute deaccum: deaccumulation of a variable (VAR[t+1]-VAR[t]) # compute_opersvarsfiles: Function to compute opersvarfiles: operation of variables from different files (OPER1.FILE1_VAR1 OPER2.FILE2_VAR2), operations are going to be secuentially made # compute_opervaralltime: Function to compute opervaralltime: operation of variable successible allover the time-steps # compute_opervartimes: Function to compute opervartimes: operation of variable for a given sub-set of time-steps # computevar_model: Function to provide the way to compute a CF-variable providing its name # compute_tevolboxtraj: Function to compute tevolboxtraj: temporal evolution at a given point along a box following a trajectory # compute_tevolboxtraj_radialsec: Function to compute tevolboxtraj_radialsec: temporal evolution at a given point along a number of radii at a given frequency following a trajectory # compute_WRFtime: function to copmute CFtimes from WRFtime variable # curve_section: Function to provide a section of a file following a given curve # DataSetSection: Function to get a section (values along a dimension) of a given data-set # DataSetSection_multidims: Function to get a section (values along multiple dimensions) of a given data-set # DataSetSection_multivars: Function to get a section (values along multiple variables) of a given data-set # DatesFiles: Function to find different time values on a series of WRF files in a folder # DimsLoop: Function to provide the shape of the dimensions which are not selected # DYNAMICO_toCF: Function to pass a DYNAMICO original file to CF-conventions # dimToUnlimited: Operation to create an unlimited dimension from an existing one # dimVar_creation: Function to add a 1D variable with the size of a given dimension in a file # fattradd: Adding attributes from a reference file # fgaddattr: Adding global attributes from a reference file # fdimadd: Adding dimension from another reference file # field_stats: Function to retrieve statistics from a field # field_stats_dim: Function to retrieve statistics from a field along dimensions # file_creation: Operation to create a file with one variable with a given set of dimensions # file_oper_alongdims: Function to operate a file along different dimensions of a variable # fill_ncvariable_lastdims: Function to fill the last [Nlastdims] dimensions of a variable [varn] from a netCDF object at the other dimensions at [prevdims] # filter_2dim: Function to filter along 2 dimensions (moving grid-box means of a given size) values of the netCDF file # flipdim: flips the value following one dimension [Nflipdim] # fvaradd: Adding variable (and all its attributes and dimensions) from a reference file to a file # gaddattr: Add a global attribute to a netCDF. Removes previous attribute if it exist # gaddattrk: Add a global attribute to a netCDF caring about the type. Removes previous attribute if it exist # genericNCvariable: Class to fake a netCDF varibale # get_1str_nc: Function to get 1 string value in a netCDF variable as a chain of 1char values # get_attribute: Function to get an attribute from a netCDF file # get_namelist_vars: Function to get namelist-like values ([varname] = [value]) # get_str_nc: Function to get string values in a netCDF variable as a chains of 1char values # get_Variables: Function to get a list of variables from an existing file # getvalues_lonlat: Function to retrieve the values from the closest grid point to a set of longitude, latitude values # get_varunits: Getting the units of a variable object # getvars_tofile: Function to get variables from a file and pass them to another one # grattr: Function to read a global atribute # grmattr: Removing a global attribute # idims: Give all the dimensions names of a file # igattrs: Give all the global attributes of a file # increaseDimvar: Function to increase with 1 dimension an existing variable within a netcdf file. Values of the variable will be repeated along the new dimension # insert_variable: Function to insert a variable in an existing file # interp_curve: Function to interpolate (as weighted mean with 4-closest ones) a variable following a curve # isgattrs: Give a single global attribute of a file and its type # isvattrs: Give a single attribute of a variable # ivars: Give all the variable names of a file # ivattrs: Give all the attributes of a variable and its type # load_ncvariable_lastdims: Function to load the last [Nlastdims] dimensions of a variable [varn] from a netCDF object at the other dimensions at [prevdims] # lonlatProj: Function to compute longitudes and latitudes for a given projection following subroutine calc_resolution_in from ORCHIDEE/src_global/module_InterpWeight # lonlat_polygon: Function to define a lon/lat region giving the coordinates of the vertexs of a given polygon # maskvar: Function to mask a variable using a mask. It is assumed that mask[...,dimM,dimJ,dimK] and var[...,dimM,dimJ,dimK] share the last dimensions # merge_files: Function to merge variables from two files # ModelChar: Class object to determine major characterisitcs of a given model output # model_characteristics: Functino to provide major characterisitcs of a given model output # mthDYNAMICO_toCF: Function to pass a mthDYNAMICO original file to CF-conventions # ncreplace: Function to replace something from a netCDF file # ncstepdiff: Function to compute differencies between time-steps (deacumulate) a netCDF file # nctype: Function to provide the string for the variable creation in a netCDF file # netcdf_concatenation: Function to concatenate netCDF files for a given set of variable # netcdf_fold_concatenation: Function to concatenate netCDF files in a given folder for a given set of variables # netcdf_fold_concatenation_HMT: Function to concatenate netCDF files in a given folder for a given set of variables giving Header, Middle, Tail for the name files # operation_alongdims: Function to operate along different dimensions of a variable # operdim: Function to operate along a series of dimensions # ovar_onc: Function to copy an object variable to a nother netcdf object # Partialmap_Entiremap: Function to transform from a partial global map (e.g.: only land points) to an entire one Coincidence of points is done throughout a first guess from fractions of the total domain of search # Partialmap_EntiremapFor: Function to transform from a partial global map (e.g.: only land points) to an entire one usinf Fortran code # Partialmap_EntiremapForExact: Function to transform from a partial global map (e.g.: only land points) to an entire one using Fortran code with exact location # pinterp: Function to vertically interpolate using subroutines from the p_interp.F90 NCAR program # put_variable_slice: Function to add a variable from a netcdf object to a new one following a slice # reconstruct_matrix_from_vector: Function to reconstruct a 2D matrix from a pair of syncronized vectors with the positions on x and y coordinates # remapnn: Function to remap to the nearest neightbor a variable using projection from another file # remapnn_old: Function to remap to the nearest neightbor a variable using projection from another fil # reproject: Function to reproject values to another one # rm_FillValue: Operation to remove the _FillValue from a variable inside a netCDF file # seasmean: Function to compute the seasonal mean of a variable # sellonlatbox: Function to select a lotlan box from a data-set # sellonlatboxold: Function to select a lotlan box from a data-set # sellonlatlevbox: Function to select a lotlan box and a given level from a data-set # selvar: Function to select a series of variables from a netcdf file. Variables with the values of the associated dimensions will be also retrieved # set_attribute: Sets a value of an attribute of a netCDF variable. Removes previous attribute value if exists # set_attributek: Sets a value of an attribute of a netCDF variable with a kind. Removes previous attribute value if exists # setvar_asciivalues: Function to set de values of a variable with an ASCII file (common Fortran-like format) # setvar_nc: Operation to set values of a variable from values of an other one # SliceVar: Function to slice a given variable throughout a given list of dimensions # SliceVarDict: Function to slice a given variable throughout a dictionary # slice_createArray: Function to create a variable from a list which contains slicing-variable values # slice_variable: Function to return a slice of a given variable according to values to its dimensions # sorttimesmat: Function to sort the time values of a given file # spacemean: Function to retrieve a space mean series from a multidimensional variable of a file # SpatialWeightedMean: Function to compute the spatial mean using weights from a netCDF file # statcompare_files: Python script to statistically compare two different files # submns: Function to retrieve a series of months from a file # subyrs: Function to retrieve a series of years from a file # testvarinfile: Check existence of a variable inside a netCDF file # TimeInf: Function to print all the information from the variable time # time_information: Function to provide information about variable time # timemean: Function to retrieve a time mean series from a multidimensional variable of a file # time_reset: Function to re-set the time axis of a file # timeshiftvar: Function to temporaly shift a number of time-steps a given variable inside a netCDF file # TimeSplitmat: Function to transform a file with CFtimes to a matrix [Nyear,Nmonth,Nday,Nhour,Nminute,Nsecond] # uploading_timestep: Function to upload a given time-step of a variable inside a netCDF # valmod: Function to modify the value of a variable # valmod_dim: Function to modify the value of a variable at a given dimension and value # var_3desc: Function to provide std_name, long_name and units from an object variable # varaddattr: Add an attribute to a variable. Removes previous attribute if it exists # varaddattrk: Add an attribute to a variable caring about the type # varaddref: Function to add a variable in an existing file copying characteristics from an existing one # var_creation: Operation to create a new variable in a file with a given set of dimensions # variable_inf: Class to provide the information from a given variable # varinfile: Check existence of a variable inside a netCDF file # varrmattr: Removing an attribute from a variable # VarVal_FillValue: Function to transform a given value from a given variable to _FillValue in a netCDF file # var_dim_dimv: Function to bring back which variables with the values should be use for a given variable # varDimension: Function to find the variable with the values of a dimension. It is assumed, that given variable # var_model: Function to provide the way to compute a CF-variable providing its name # varout: Function when we want to output variable values # varoutold: Function when we want to output variable values # varrm: Removing a variable from a file # vrattr: Function to remove an atribute from a variable # writing_1str_nc: Function to write 1 string value in a netCDF variable as a chain of 1char values # writing_str_nc: Function to write string values in a netCDF variable as a chain of 1char values # WRF_CFlonlat_creation: Function to add a CF-convention longitude/latitude variables in a WRF file # WRF_CFtime_creation: Function to add a CF-convention time unit in a WRF file # WRF_CFxtime_creation: Function to add a CF-convention time unit in a WRF file using variable 'XTIME' # WRF_d0Nref: Function for the generation of an extra WRF domain from a given one # WRF_toCF: Function to pass a WRF original file to CF-conventions main = 'nc_var_tools.py' errormsg = 'ERROR -- error -- ERROR -- error' warnmsg = 'WARNING -- warning -- WARNING -- warning' infmsg = 'INFORMATION -- information -- INFORMATION -- information' ## Constants # Radius of the Earth (m) EarthR = 6378000. class genericNCvariable(object): """ Class to fake a netCDF varibale d[x/y/z/t]: dimensions of the variable dn[x/y/z/t]: name of the dimensions of the variable >>> variable = genericNCvariable(5,4,3,2,'lon','lat','pres','time') """ def __init__(self,dx,dy,dz,dt,dnx,dny,dnz,dnt): from collections import OrderedDict totshape = dt*dz*dy*dx dimdict = {dnt: dt, dnz: dz, dny: dy, dnx: dx} self.values = np.arange((totshape), dtype=np.float).reshape(dt,dz,dy,dx) # Here for a fake netCDF file # self.dimensions = OrderedDict(sorted(dimdict.items(), key=lambda t: t[0])) self.dimensions = (dnt, dnz, dny, dnx) self.shape = (dt, dz, dy, dx) self.standard_name = 'general_var' self.long_name = 'generic netcdf variable' self.units = '-' def __getitem__(self, val): return self.values[val] def varinfile(ncf, filen, emsg, vmsg, varn): """ Check existence of a variable inside a netCDF file ncf= netCDF object filen= file name emsg= general error message vmsg= message before generic text of inexistence of variable varn= variable name """ if not ncf.variables.has_key(varn): print emsg print ' file "' + filen + '" does not have ' + vmsg + ' variable "' + varn + '" !!' print emsg ncf.close() quit(-1) return class testvarinfile(object): """ Check existence of a variable inside a netCDF file ncf= netCDF object filen= file name emsg= general error message vmsg= message before generic text of inexistence of variable varn= variable name self.exist= boolean value of existence self.message= error message """ def __init__(self, ncf, filen, emsg, vmsg, varn): if ncf is None: self.exist= None self.message = None else: if not ncf.variables.has_key(varn): self.exist = False self.message = emsg + '\n' + ' file "' + filen + '" does not have "' + vmsg + '" variable "' + varn + '" !!\n' + \ emsg else: self.exist = True return def attrinvar(varobj, emsg, vmsg, attrn): """ Check existence of an attribute in a netCDF variable object varobj= netCDF variable object emsg= general error message vmsg= message before generic text of inexistence of variable attrrn= attribute name """ varattrs = varobj.ncattrs() if not isInlist(varattrs, attrn): print emsg print ' variable "' + varobj.name + '" does not have attribute ' + attrn + '" !!' print emsg ncf.close() quit(-1) return def add_dims(oc,nc,dnames): """ Function to add dimensions from a given netCDF object to another one oc: source netcdfile object nc: netcdfile object to add dimensions dnames: list of name of dimensions to add """ fname = 'add_dims' for dn in dnames: if not gen.searchInlist(nc.dimensions, dn): print ' ' + fname + ': adding dim:',dn,' ...' if oc.dimensions[dn].isunlimited(): newdim = nc.createDimension(dn, None) else: newdim = nc.createDimension(dn, len(oc.dimensions[dn])) nc.sync() return def add_vars(oc,nc,vnames): """ Function to add variables from a given netCDF object to another one oc: source netcdfile object nc: netcdfile object to add dimensions vnames: list of name of variables to add """ fname = 'add_vars' for vn in vnames: if not gen.searchInlist(nc.variables,vn): print ' ' + fname + ': adding var:',vn,' ...' varo = oc.variables[vn] vartype = varo.dtype vardims = varo.dimensions varattrs = varo.ncattrs() for vdn in vardims: if not gen.searchInlist(nc.dimensions,vdn): print warnmsg print ' ' + fname + ": adding dimension '" + vdn + \ "' from variable '" + vdn + "' which is not in file !!" add_dims(oc,nc,[vdn]) if gen.searchInlist(varattrs,'_FillValue'): newvar = nc.createVariable(vn, vartype, vardims, \ fill_value=varo.getncattr('_FillValue')) else: newvar = nc.createVariable(vn, vartype, vardims) for attrn in varattrs: attrv = varo.getncattr(attrn) newattr = set_attribute(newvar, attrn, attrv) newvar[:] = varo[:] nc.sync() return def add_varattrs(oc,nc,attrvnames,vnames): """ Function to add variables attributes from a given netCDF object to other variables in another one oc: source netcdfile object nc: netcdfile object to add dimensions attrvnames: list of variables to take the attributes vnames: list of name of variables to add """ fname = 'add_varattrs' ivar = 0 for vn in attrvnames: ovn = vnames[ivar] if not oc.variables.has_key(vn): print errormsg print ' ' +fname+ ": source file does not have variable '" + vn + "' !!" quit(-1) if not nc.variables.has_key(ovn): print errormsg print ' ' +fname+ ": target file does not have variable '" + ovn + "' !!" quit(-1) varo = oc.variables[vn] ovaro = nc.variables[ovn] varattrs = varo.ncattrs() for attrn in varattrs: if attrn != '_FillValue': attrv = varo.getncattr(attrn) newattr = set_attribute(ovaro, attrn, attrv) nc.sync() ivar = ivar + 1 return def add_globattrs(oc,nc,attrns): """ Function to add global attributes from a given netCDF object to another one oc: source netcdfile object nc: netcdfile object to add dimensions vnames: list of name of dimensions to add ('all', for all attributes) """ fname = 'add_globattrs' allfattrs = oc.ncattrs() if attrns == 'all': fattrns = allfattrs else: fattrns = attrns for an in fattrns: if not gen.searchInlist(allfattrs,an): print errormsg print ' ' + fname + ": file has not global attribute '" + an + "' !!" quit(-1) print ' ' + fname + ': adding attr:',an,' ...' av = oc.getncattr(an) newattr = set_attribute(nc, an, av) nc.sync() return def valmod(values, ncfile, varn): """ Function to modify the value of a variable values = modins,[modval1],[...,[modvalN]] modins = instruction: 'sumc', add [modval1] 'subc', substraction [modval1] 'mulc', multiply by [modval1] 'divc', divide by [modval1] 'lowthres': modify all values below [modval1] to [modval2] 'upthres': modify all values above [modval1] to [modval2] 'potc': [val] ** [modval1] ncfile = netCDF file name varn = name of the variable """ fname='valmod' if values == 'h': print fname + '_____________________________________________________________' print valmod.__doc__ quit() vals = values.split(',') modins = vals[0] modval = float(vals[1]) if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ": File '" + ncfile + "' does not have variable '" + \ varn + "' !!" print errormsg ncf.close() quit(-1) var = ncf.variables[varn] varshape = var.shape vartype = var.dtype Ndims = len(varshape) varshapevals = list(varshape) Nvars = len(varshape) if not Nvars == 0: if Nvars == 1: varvals = np.zeros(varshape[0], dtype=vartype) elif Nvars == 2: varvals = np.zeros((varshape[0], varshape[1]), dtype=vartype) elif Nvars == 3: varvals = np.zeros((varshape[1], varshape[2]), dtype=vartype) elif Nvars == 4: varvals = np.zeros((varshape[2], varshape[3]), dtype=vartype) elif Nvars == 5: varvals = np.zeros((varshape[3], varshape[4]), dtype=vartype) elif Nvars == 6: varvals = np.zeros((varshape[4], varshape[5]), dtype=vartype) else: print errormsg print ' ' + fname + ': variable size ',Nvars,' is not ready!!!!' # Allocating all the necessary memory (just in case) if Nvars <= 2: varvals = var[:] var[:] = gen.valmodoper(varvals, values) elif Nvars == 3: for i in range(varshape[0]): varvals[:] = var[i,:,:] var[i,:,:] = gen.valmodoper(varvals, values) elif Nvars == 4: for i in range(varshape[0]): for j in range(varshape[1]): varvals[:] = var[i,j,:,:] var[i,j,:,:] = gen.valmodoper(varvals, values) elif Nvars == 5: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): varvals[:] = var[i,j,k,:,:] var[i,j,k,:,:] = gen.valmodoper(varvals, values) elif Nvars == 6: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): for l in range(varshape[3]): varvals[:] = var[i,j,k,l,:,:] var[i,j,k,l,:,:] = gen.valmodoper(varvals, values) ncf.sync() ncf.close() return def varaddref(values, ncfile, varn): """ Function to add a variable in an existing file copying characteristics from an existing one values = [variable ref]:[attr name]@[value][:[attr2]@[value2], ...]:[value/file with values] add a new variable [varn] with dimension and attributes from an already existing [variable ref] with attributes [[attr name]@[value][:[attr2]@[value2], ...]] in the file [netcdf] and value [value/file with values] netcdf = netCDF file name varn = new variable name """ fname = 'varaddref' if values == 'h': print fname + '_____________________________________________________________' print varaddref.__doc__ quit() varvalues = values.split(':') Nvarvalues = len(varvalues) varprev = varvalues[0] newattrs = {} for iattr in range(Nvarvalues - 2): attrv = varvalues[iattr+1] newattrs[attrv.split('@')[0]] = attrv.split('@')[1] ncf = NetCDFFile(ncfile,'a') if ncf.variables.has_key(varn): print errormsg print ' varaddref: File already has the varible ' + varn + ' !!!' print errormsg ncf.close() quit(-1) if not ncf.variables.has_key(varprev): print errormsg print ' varaddref: File does not have variable ' + varprev + ' !!!!' print errormsg ncf.close() quit(-1) varp = ncf.variables[varprev] varpVal = varp[:] varpshape = varp.shape Nvarpshape = len(varpshape) varpattrs = varp.ncattrs() varptype = varp.dtype varpdims = varp.dimensions varpndims = varp.ndim ## print ' shape of the variable used as reference: ',varpshape # # variable characteristics if len(varpshape) == 4: dimpt = varpshape[0] dimpz = varpshape[1] dimpy = varpshape[2] dimpx = varpshape[3] elif len(varpshape) == 3: dimpt = varpshape[0] dimpy = varpshape[1] dimpx = varpshape[2] elif len(varpshape) == 3: dimpy = varpshape[0] dimpx = varpshape[1] newvar = ncf.createVariable(varn, varptype, dimensions=varpdims) newvar = ncf.variables[varn] # # Values varplen=1 for idim in range(varpndims): varplen=varplen*varpshape[idim] if varptype == 'float' or varptype == 'float32' or varptype == 'float64' or vartype == np.float(1.) or vartype == np.float32(1.): newvals = np.reshape(np.arange(varplen), varpshape)*1. else: newvals = np.reshape(np.arange(varplen), varpshape) if not os.path.isfile(varvalues[Nvarvalues-1]): ## print ' Using constant value' newvals[:] = float(varvalues[Nvarvalues-1]) else: ## print ' Using 2-D values from ' + varvalues[Nvarvalues-1] asciif = open(varvalues[Nvarvalues-1], 'r') fdimx = len(asciif.readlines()) asciif.seek(0) if len(varpshape) == 4: if not fdimx == dimpx: print errormsg print ' varaddref: Provided file has dimx=', fdimx, ' and variable has dimx=', dimpx, ' !!!' print errormsg ncf.close() quit(-1) iline = 0 idx = 0 for fline in asciif: line = fline.replace('\n','') yvals = [] for iyval in line.split(' '): yvals.append(float(iyval)) if iline == 0: fdimy = len(yvals) if not fdimy == dimpy: print errormsg print ' varaddref: Provided file has dimy=', fdimy, ' and variable has dimy= ', dimpy, ' !!!' print errormsg ncf.close() quit(-1) for it in range(dimpt): for iz in range(dimpz): newvals[it,iz,:,idx] = yvals idx = idx+1 iline = iline+1 elif len(varpshape) == 3: if not fdimx == dimpx: print errormsg print ' varaddref: Provided file has dimx=', fdimx, ' and variable has dimx=', dimpx, ' !!!' print errormsg ncf.close() quit(-1) iline = 0 idx = 0 for fline in asciif: line = fline.replace('\n','') yvals = [] for iyval in line.split(' '): yvals.append(float(iyval)) if iline == 0: fdimy = len(yvals) if not fdimy == dimpy: print errormsg print ' varaddref: Provided file has dimy=', fdimy, ' and variable has dimy= ',dimpy, ' !!!' print errormsg ncf.close() quit(-1) for it in range(dimpt): newvals[it,:,idx] = yvals idx = idx+1 iline = iline+1 elif len(varpshape) == 2: if not fdimx == dimpx: print errormsg print ' varaddref: Provided file has dimx=', fdimx, ' and variable has dimx=', dimpx, ' !!!' print errormsg ncf.close() quit(-1) iline = 0 idx = 0 for fline in asciif: line = fline.replace('\n','') yvals = [] for iyval in line.split(' '): yvals.append(float(iyval)) if iline == 0: fdimy = len(yvals) if not fdimy == dimpy: print errormsg print ' varaddref: Provided file has dimy=', fdimy, ' and variable has dimy= ',dimpy, ' !!!' print errormsg ncf.close() quit(-1) newvals[:,idx] = yvals idx = idx+1 iline = iline+1 asciif.close() newvar[:] = newvals # # Attributes for iattr in range(len(varpattrs)): attrn = varpattrs[iattr] attrval = varp.getncattr(attrn) if attrn in newattrs: newattr = newvar.setncattr(attrn, newattrs[attrn]) else: newattr = newvar.setncattr(attrn, attrval) ncf.sync() ncf.close() return def varoutold(values, ncfile, varn): """ Function when we want to output variable values values = [optsIrange]:[optsErange] [optsIrange]: val1,val2,...,valN inital value for the 'N' dimensions of the variable [optsErange]: val1,val2,...,valN ending value for the 'N' dimensions of the variable ncfile = netCDF file name varn = variable name """ import numpy as np from netCDF4 import Dataset as NetCDFFile optsIrange = values.split(':')[0] optsErange = values.split(':')[1] inirange=optsIrange.split(',') endrange=optsErange.split(',') irange = [int(val) for val in inirange] erange = [int(val) for val in endrange] if not len(irange) == len(erange): print errormsg print ' varout: Different number of values in each range!' print ' varout: initial range: ' + optsIrange print ' varout: ending range: ' + optsErange print errormsg quit(-1) else: ndims=len(irange) if not os.path.isfile(ncfile): print errormsg print ' varout: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'r') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' varout: File "' + ncfile + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) var = ncf.variables[varn] varshape = var.shape Nvarshape = len(varshape) if not Nvarshape == ndims: print errormsg print ' varout: Provided number of values of the range ' + ndims + ' is different of the shape of the variable ' + Nvarshape + ' !!!' print errormsg ncf.close() quit(-1) if Nvarshape == 1: varValrange = var[irange[0]:rangedim(erange[0], varshape[0])] elif Nvarshape == 2: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1]), varshape[1]] elif Nvarshape == 3: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1], varshape[1]), irange[2]:rangedim(erange[2], varshape[2])] elif Nvarshape == 4: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1], varshape[1]), irange[2]:rangedim(erange[2], varshape[2]), \ irange[3]:rangedim(erange[3], varshape[3])] elif Nvarshape == 5: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1], varshape[1]), irange[2]:rangedim(erange[2], varshape[2]), \ irange[3]:rangedim(erange[3], varshape[3]), irange[4]:rangedim(erange[4], varshape[4])] elif Nvarshape == 6: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1], varshape[1]), irange[2]:rangedim(erange[2], varshape[2]), \ irange[3]:rangedim(erange[3], varshape[3]), irange[4]:rangedim(erange[4], varshape[4]), \ irange[5]:rangedim(erange[5], varshape[5])] ncf.close() Nshaperange = len(varValrange.shape) Noneshape = 0 for ir in range(Nshaperange): if varValrange.shape[ir] == 1: Noneshape = Noneshape + 1 if Noneshape == Nshaperange - 1: for i in range(len(varValrange)): print '%2s %f' % ( 'NC', varValrange[i] ) return def varout(values, ncfile, varn): """ Function when we want to output variable values values = [dimname1]:[valdim1]|[dimname2]:[valdim2]|[...,[dimnameN]:[valdimN]] [valdim]: * [integer]: which value of the dimension * -1: all along the dimension * -9: last value of the dimension * [beg]@[end]@[freq] slice from [beg] to [end] every [freq] * NOTE, no dim name all the dimension size ncfile = netCDF file name varn = variable name """ fname = 'varout' if values == 'h': print fname + '_____________________________________________________________' print varout.__doc__ quit() ncobj = NetCDFFile(ncfile, 'r') if not ncobj.variables.has_key(varn): print errormsg print ' ' + fname + ': file "' + ncfile + '" does not have variable "' + \ varn + '" !!' quit(-1) vobj = ncobj.variables[varn] newvar, newdims = slice_variable(vobj, values) Nnewdims = len(newdims) if Nnewdims == 0 or len(newvar.shape) == 0: print '%2s %f' % ( 'NC',newvar ) elif Nnewdims == 1: for i in range(newvar.shape[0]): print '%2s %f' % ( 'NC',newvar[i] ) elif Nnewdims == 2: for i0 in range(newvar.shape[0]): row = numVector_String(newvar[i0,:],' ') print 'NC ' + row else: for i0 in range(newvar.shape[Nnewdims-2]): varslice = [] dimsinf = '' for idim in range(Nnewdims - 2): for d in range(newvar.shape[idim]): if idim == 0: dimsinf = newdims[idim] + ': ' + str(d) else: dimsinf = dimsinf + ' ' + newdims[idim] + ': ' + str(d) varslice.append(d) varslice.append(i0) varslice.append(0,newvar.shape[Nnewdims-1]) row = numVector_String(newvar[tuple(varslice)],' ') print 'NCinf ' + dimsinf + '_________________' print 'NC ' + row ncobj.close() return def chdimname(values, ncfile): """ Changing the name of the dimension values = [olddimname]:[newdimname] [olddimname]: old name of the dimension [newdimname]: new name of the dimension ncfile = netCDF file name """ fname = 'chdimname' if values == 'h': print fname + '_____________________________________________________________' print chdimname.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ": File '" + ncfile + "' does not exist !!" quit(-1) ncf = NetCDFFile(ncfile,'a') olddimname=values.split(':')[0] newdimname=values.split(':')[1] if not ncf.dimensions.has_key(olddimname): print warnmsg print ' ' + fname + ": File '" + ncfile + "' does not have dimension '" + \ olddimname + "' !!" ncf.close() quit(-1) if not olddimname == newdimname and not ncf.dimensions.has_key(newdimname): newname = ncf.renameDimension(olddimname, newdimname) ncf.sync() ncf.close() else: print warnmsg print ' ' + fname + ": File '" + ncfile + "' already has dimension name '" + \ newdimname + '" ' print ' ' + fname + ": modifying all variables which use the old '" + \ olddimname + "' dimension..." filevars = ncf.variables for fvarn in filevars: if ncf.variables.has_key(fvarn): fvar = ncf.variables[fvarn] fvardims = fvar.dimensions if gen.searchInlist(fvardims, olddimname): # print ' variable "' + fvarn + '" uses dimension "' + olddimname + '" ' varinf = variable_inf(fvar) newdims = tuple(fvardims) change = {olddimname: newdimname} # From http://stackoverflow.com/questions/9067043/python-replace-list-values-using-a-tuple newdims = tuple([ change.get(x,x) for x in fvardims ]) newvar = ncf.createVariable(fvarn + 'tmpdname', varinf.dtype, \ newdims, fill_value=varinf.FillValue) varv = fvar[:] newvar[:] = varv for attrn in varinf.attributes: newvarattr = newvar.ncattrs() if not gen.searchInlist(newvarattr, attrn): attrv = fvar.getncattr(attrn) newar = newvar.setncattr(attrn, attrv) fvar = ncf.renameVariable(fvarn, fvarn + 'rmdname') ncf.sync() newvar = ncf.renameVariable(fvarn + 'tmpdname', fvarn) ncf.sync() ncf.close() varrm(ncfile, fvarn + 'rmdname') ncf = NetCDFFile(ncfile,'a') return def chvarname(values, ncfile, varn): """Changing the name of the variable values = new variable name ncfile = netCDF file varn = name of the variable """ fname = 'chvarname' if values == 'h': print fname + '_____________________________________________________________' print chvarname.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print warnmsg print ' ' + fname + ': File does not have variable "' + varn + '" !!!!' ncf.close() quit(-1) if not varn == values and not ncf.variables.has_key(values): newname = ncf.renameVariable(varn, values) ncf.sync() ncf.close() return def set_attribute(ncv, attrname, attrvalue): """ Sets a value of an attribute of a netCDF variable. Removes previous attribute value if exists ncv = object netcdf variable attrname = name of the attribute attrvalue = value of the attribute """ fname = 'set_attribute' attvar = ncv.ncattrs() if gen.searchInlist(attvar, attrname): attr = ncv.delncattr(attrname) attr = ncv.setncattr(attrname, attrvalue) return attr def set_attributek(ncv, attrname, attrval, attrkind): """ Sets a value of an attribute of a netCDF variable with a kind. Removes previous attribute value if exists ncvar = object netcdf variable attrname = name of the attribute attrvalue = value of the attribute atrtrkind = kind of attribute: 'S', string ('!' as spaces); 'U', unicode ('!' as spaces); 'I', integer; 'Inp32', numpy integer 32; 'R', real; ' npfloat', np.float; 'D', np.float64 """ fname = 'set_attributek' validk = {'S': 'string', 'U': 'unicode', 'I': 'integer', \ 'Inp32': 'integer long (np.int32)', 'R': 'float', 'npfloat': 'np.float', \ 'D': 'float long (np.float64)'} if type(attrkind) == type('s'): if attrkind == 'S': attrvalue = str(attrval.replace('!', ' ')) elif attrkind == 'U': attrvalue = unicode(attrval.replace('!',' ')) elif attrkind == 'I': attrvalue = np.int(attrval) elif attrkind == 'R': attrvalue = float(attrval) elif attrkind == 'npfloat': attrvalue = np.float(attrval) elif attrkind == 'D': attrvalue = np.float64(attrval) else: print errormsg print ' ' + fname + ": '" + attrkind + "' kind of attribute is not ready!" print ' valid values: _______' for key in validk.keys(): print ' ', key,':', validk[key] quit(-1) else: if attrkind == type(str('a')): attrvalue = str(attrval.replace('!', ' ')) elif attrkind == type(unicode('a')): attrvalue = unicode(attrval.replace('!',' ')) elif attrkind == type(np.int(1)): attrvalue = np.int(attrval) elif attrkind == np.dtype('i'): attrvalue = np.int32(attrval) elif attrkind == type(float(1.)): attrvalue = float(attrval) elif attrkind == type(np.float(1.)): attrvalue = np.float(attrval) elif attrkind == np.dtype('float32'): attrvalue = np.array(attrval, dtype='f') elif attrkind == type(np.float32(1.)): attrvalue = np.float32(attrval) elif attrkind == type(np.float64(1.)): attrvalue = np.float64(attrval) elif attrkind == type(np.array([1.,2.])): attrvalue = attrval else: print errormsg print ' ' + fname + ": '" + attrkind + "' kind of attribute is not ready!" print ' valid values: _______' for key in validk.keys(): print ' ', key,':', validk[key] quit(-1) attvar = ncv.ncattrs() if gen.searchInlist(attvar, attrname): attr = ncv.delncattr(attrname) if attrname == 'original_subroutines_author': attrvalue = 'Cindy Bruyere' attr = ncv.setncattr(attrname, attrvalue) return attr def add_global_PyNCplot(ObjFile, pyscript, funcname, version): """ Function to add global attributes from 'PyNCplot' to a given netCDF ObjFile= netCDF file object to which add the global attributes funcname= name of the function from which file comes from version= version of the function """ fname = 'add_global_PyNCplot' # Global values ObjFile.setncattr('author', 'L. Fita') #newattr = set_attributek(ObjFile, 'institution', unicode('Laboratoire ' + \ # 'de M' + unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') #ObjFile.setncattr('university', 'Pierre et Marie Curie - Jussieu, Paris 06') #ObjFile.setncattr('center', 'Centre National de Recherches Scientifiques') #ObjFile.setncattr('city', 'Paris') #ObjFile.setncattr('country', 'France') newattr = set_attributek(ObjFile, 'institution', unicode('Centro de ' + \ 'Investigaciones del Mar y la Atm' + unichr(243) + 'sfera (CIMA)'), 'U') newattr = set_attributek(ObjFile, 'institution2', unicode('Instituto Franco-' + \ 'Argentino sobre Estudios de Clima y sus Impactos (CNRS, UMI-3351-IFAECI'), 'U') newattr = set_attributek(ObjFile, 'center', unicode('Consejo Nacional de ' + \ 'Investigaciones Cient' + unichr(237) + 'ficas y T' + unichr(233) + \ 'cnicas (CONICET)'), 'U') ObjFile.setncattr('university', 'Universidad de Buenos Aires (UBA)') ObjFile.setncattr('city', 'Buenos Aires') ObjFile.setncattr('country', 'Argentina') ObjFile.setncattr('tool', 'PyNCplot') ObjFile.setncattr('url', 'http://www.xn--llusfb-5va.cat/python/PyNCplot') ObjFile.setncattr('script', pyscript) if funcname is not None: ObjFile.setncattr('function', funcname) ObjFile.setncattr('version', version) ObjFile.sync() return def basicvardef(varobj, vstname, vlname, vunits): """ Function to give the basic attributes to a variable varobj= netCDF variable object vstname= standard name of the variable vlname= long name of the variable vunits= units of the variable """ attr = varobj.setncattr('standard_name', vstname) attr = varobj.setncattr('long_name', vlname) attr = varobj.setncattr('units', vunits) return def gaddattr(values, ncfile): """ Add a global attribute to a netCDF. Removes previous attribute if it exist values = [attrname]|[attrvalue ('!' as spaces)] ncfile = netCDF file """ if not os.path.isfile(ncfile): print errormsg print ' gaddattr: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') attrvals=values.split('|') attrn=attrvals[0] attrv=attrvals[1].replace('!', ' ') ncf = set_attribute(ncf, attrn, attrv) ncf.sync() ncf.close() return def gaddattrk(values, ncfile): """ Add a global attribute to a netCDF caring about the type. Removes previous attribute if it exist values = [attrname]|[attrvalue]|[attrk] attrname = name of the attribute attrvalue = value of the attribute attrk = 'S', string ('!' as spaces); 'I', integer; 'Inp32', numpy integer 32; 'R', real; 'D', double ncfile = netCDF file """ if not os.path.isfile(ncfile): print errormsg print ' gaddattrk: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') attrvals=values.split('|') attrn=attrvals[0] attrv0=attrvals[1] attrk=attrvals[2] if attrk == 'S': attrv = str(attrv0.replace('!', ' ')) elif attrk == 'I': attrv = int(attrv0) elif attrk == 'Inp32': attrv = np.int32(attrv0) elif attrk == 'R': attrv = float(attrv0) elif attrk == 'D': attrv = np.float64(attrv0) else: print errormsg print ' gaddattrk: "' + attrk + '" kind of attribute is not ready!' ncf.close() quit(-1) ncf = set_attribute(ncf, attrn, attrv) ncf.sync() ncf.close() return def varaddattr(values, ncfile, varn): """ Add an attribute to a variable. Removes previous attribute if it exists values = [attrname]|[attrvalue('!' as spaces)] ncfile = netCDF file name varn = name of the variable """ fname = 'varaddattr' if values == 'h': print fname + '_____________________________________________________________' print varaddattr.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname +": File '" + ncfile + "' does not exist !!" print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname +': File "' + ncfile + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) attrvals=values.split('|') attrn=attrvals[0] attrv=attrvals[1].replace('!', ' ') var = ncf.variables[varn] var = set_attribute(var, attrn, attrv) ncf.sync() ncf.close() return def varaddattrk(values, ncfile, varn): """ Add an attribute to a variable caring about the type values = [attrname]|[attrvalue]|[attrk] attrname = name of the attribute attrvalue = value of the attribute attrk = 'S', string ('!' as spaces); 'I', integer; 'R', real; 'D', double; 'npR', numpy Real, 'npR32', numpy Real-32 ncfile = netCDF file varn = variable name """ fname = 'varaddattrk' if values == 'h': print fname + '_____________________________________________________________' print varaddattrk.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ': File "' + ncfile + '"does not have variable ' + varn + ' !!!!' print errormsg ncf.close() quit(-1) attrvals=values.split('|') attrn=attrvals[0] attrv0=attrvals[1] attrk=attrvals[2] var = ncf.variables[varn] if attrk == 'S': attrv = str(attrv0.replace('!', ' ')) elif attrk == 'I': attrv = int(attrv0) elif attrk == 'R': attrv = np.float(attrv0) elif attrk == 'D': attrv = np.float64(attrv0) elif attrk == 'npR': attrv = np.float(attrv0) elif attrk == 'npR32': attrv = np.float32(attrv0) else: print errormsg print ' ' + fname + ': "' + attrk + '" kind of attribute is not ready!' ncf.close() quit(-1) var = set_attribute(var, attrn, attrv) ncf.sync() ncf.close() return def varrmattr(values, ncfile, varn): """ Removing an attribute from a variable values = attribute name ncfile = netCDF file name varn = variable name """ fname = 'varrmattr' if values == 'h': print fname + '_____________________________________________________________' print varrmattr.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname +': File "' + ncfile + '" does not exist !!' quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname +': File "' + ncfile + '" does not have variable "' + varn + '" !!!!' ncf.close() quit(-1) var = ncf.variables[varn] attvar = var.ncattrs() if searchInlist(attvar, values): attr = var.delncattr(values) else: print warnmsg print ' ' + fname +': "' + varn + '" does not have attribute: ' + values ncf.sync() ncf.close() return def grmattr(values, ncfile): """ Removing a global attribute values = attribute name ncfile = netCDF file """ if not os.path.isfile(ncfile): print errormsg print ' grmattr: File "' + ncfile + '" does not exist !!' quit(-1) ncf = NetCDFFile(ncfile,'a') attvar = ncf.ncattrs() if searchInlist(attvar, values): attr = ncf.delncattr(values) else: print warnmsg print ' grmattr: "' + ncfile + '" does not have attribute: ' + values ncf.sync() ncf.close() return def fvaradd(values, ncfile): """ Adding variable (and all its attributes and dimensions) from a reference file to a file values = [netCDFref],[varnref] netCDFref = netCDF file name as reference for the variable to add varnref = name of the variable from [netCDFref] to be added ncfile = netCDF file name """ fname = 'fvaradd' if values == 'h': print fname + '_____________________________________________________________' print fvaradd.__doc__ quit() refnc = values.split(',')[0] refvar = values.split(',')[1] if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' quit(-1) if not os.path.isfile(refnc): print errormsg print ' ' + fname + ': Reference file "' + refnc + '" does not exist !!' quit(-1) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') refvars = ncref.variables if gen.searchInlist(refvars, refvar): refvarv = ncref.variables[refvar] else: print errormsg print ' ' + fname + ': File "' + refnc + '" does not have variable: ' + refvar ncf.close() ncref.close() quit(-1) vardims = refvarv.dimensions vartype = refvarv.dtype varattr = refvarv.ncattrs() # Checking dimensions ## newdims = ncf.dimensions for rdim in vardims: if not gen.searchInlist(newdims, rdim): print ' ' + fname + ': Adding dimension ' + rdim ncf.close() ncref.close() fdimadd(refnc + ',' + rdim, ncfile) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') # Checking fill value ## if gen.searchInlist(varattr, '_FillValue'): varfil = refvarv._FillValue else: varfil = False if not ncf.variables.has_key(refvar): print ' ' + fname + ': Adding refvar:', refvar, 'shape: ', refvarv.shape var = ncf.createVariable(refvar, vartype, vardims, fill_value=varfil) else: print warnmsg print ' ' + fname + ': updating var:', refvar, 'shape: ', refvarv.shape var = ncf.variables[refvar] varshape = refvarv.shape varvals = np.zeros(tuple(varshape), dtype=vartype) Nvars = len(varshape) # Allocating all the necessary memory (just in case) var[:] = varvals if Nvars <= 2: varvals = refvarv[:] var[:] = varvals elif Nvars == 3: for i in range(varshape[0]): varvals[i,:,:] = refvarv[i,:,:] var[i,:,:] = varvals[i,:,:] elif Nvars == 4: for i in range(varshape[0]): for j in range(varshape[1]): varvals[i,j,:,:] = refvarv[i,j,:,:] var[i,j,:,:] = varvals[i,j,:,:] elif Nvars == 5: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): varvals[i,j,k,:,:] = refvarv[i,j,k,:,:] var[i,j,k,:,:] = varvals[i,j,k,:,:] elif Nvars == 6: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): for l in range(varshape[3]): varvals[i,j,k,l,:,:] = refvarv[i,j,k,l,:,:] var[i,j,k,l,:,:] = varvals[i,j,k,l,:,:] newvar = ncf.variables[refvar] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = refvarv.getncattr(attr) if not gen.searchInlist(newvarattrs, attr): newvar.setncattr(attr, attrv) ncf.sync() ncf.close() ncref.close() return def fdimadd(values, ncfile): """ Adding dimension from another reference file values = [refnc],[refdim] refnc = netCDF file name as reference refdim = name of the dimension to be added ncfile = netCDF file name """ fname='fdimadd' refnc = values.split(',')[0] refdim = values.split(',')[1] if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' quit(-1) if not os.path.isfile(refnc): print errormsg print ' ' + fname + ': Reference file "' + refnc + '" does not exist !!' quit(-1) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') refdims = ncref.dimensions if gen.searchInlist(refdims, refdim): refdimv = ncref.dimensions[refdim] else: print errormsg print ' ' + fname + ': File "' + refnc + '" does not have dimension: "' + refdim + '"' ncf.close() ncref.close() quit(-1) if refdimv.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(refdimv) print ' ' + fname + ': Adding refdim:', refdim, 'size:', dimsize dim = ncf.createDimension(refdim, dimsize) ncf.sync() ncf.close() ncref.close() return def fattradd(var, values, ncfile): """ Adding attributes from a reference file var = variable to which has to be added the attribute values = [refnc]:[refvar] refnc = netCDF file name as reference refvar = variable from the reference file ncfile = netCDF file name """ refnc = values.split(':')[0] refvar = values.split(':')[1] if not os.path.isfile(ncfile): print errormsg print ' fattradd: File "' + ncfile + '" does not exist !!' quit(-1) if not os.path.isfile(refnc): print errormsg print ' fattradd: Reference file "' + refnc + '" does not exist !!' quit(-1) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') vars = ncf.variables if gen.searchInlist(vars, var): varv = ncf.variables[var] else: print ' fattradd: File "' + ncfile + '" does not have variable: "' + var + '"' ncf.close() ncref.close() quit(-1) refvars = ncref.variables if gen.searchInlist(refvars, refvar): refvarv = ncref.variables[refvar] else: print ' fattradd: File "' + refnc + '" does not have variable: "' + refvar + '"' ncf.close() ncref.close() quit(-1) refvarattrs = refvarv.ncattrs() Nattrs = len(refvarattrs) print ' fattradd: Adding ', Nattrs,' atributes from:', refvar for attr in refvarattrs: attrv = refvarv.getncattr(attr) atvar = set_attribute(varv, attr, attrv) ncf.sync() ncf.close() ncref.close() return def fgaddattr(values, ncfile): """ Adding global attributes from a reference file values = netCDF file name as reference ncfile = netCDF file name """ fname = 'fgaddattr' if values == 'h': print fname + '_____________________________________________________________' print fgaddattr.__doc__ quit() refnc = values if not os.path.isfile(ncfile): print errormsg print ' fgaddattr: File "' + ncfile + '" does not exist !!' quit(-1) if not os.path.isfile(refnc): print errormsg print ' fgaddattr: Reference file "' + refnc + '" does not exist !!' quit(-1) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') refgattrs = ncref.ncattrs() Nattrs = len(refgattrs) print ' fgaddattr: Adding ', Nattrs,' global atributes' for attr in refgattrs: attrv = ncref.getncattr(attr) atvar = set_attribute(ncf, attr, attrv) ncf.sync() ncf.close() ncref.close() return def varrm(ncfile, var): """ Removing a variable from a file ncfile = netCDF object file var = variable name to remove """ import shutil as shu fname = 'varrm' if var == 'h': print fname + '_____________________________________________________________' print varrm.__doc__ quit() ncf = NetCDFFile(ncfile,'a') ncvars = ncf.variables.keys() ncf.close() if not gen.searchInlist(ncvars, var): print ' ' + fname + ": File '" + ncfile + "' does not have variable: '" + \ var + "' !!" print ' variables:', ncvars quit(-1) tmpncf = NetCDFFile('tmp_py.nc' , 'w') gtmpattr = set_attribute(tmpncf, 'copy', 'temporal') tmpncf.sync() tmpncf.close() for varn in ncvars: if not varn == var: fvaradd(ncfile + ',' + varn, 'tmp_py.nc') fgaddattr(ncfile, 'tmp_py.nc') shu.copyfile('tmp_py.nc', ncfile) os.remove('tmp_py.nc') return def seasmean(timename, filename, varn): """ Function to compute the seasonal mean of a variable timename= name of the variable time in the file filename= netCDF file name varn= name of the variable """ fillValue=1.e20 ncf = NetCDFFile(filename,'a') ofile = 'seasmean_' + varn + '.nc' if not ncf.variables.has_key(varn): print errormsg print ' seasmean: File "' + filename + '" does not have variable ' + varn + ' !!!!' print errormsg ncf.close() quit(-1) varobj = ncf.variables[varn] varinf = variable_inf(varobj) timeobj = ncf.variables[timename] timeinf = cls_time_information(ncf, timename) timevals=timeobj[:] netcdftimes = netCDFdatetime_realdatetime(timeinf.units + ' since ' + \ timeinf.Urefdate, timeinf.calendar, timevals) timeseasons=times_4seasons(netcdftimes, True) timeseasvals=seasonal_means(netcdftimes, timevals) ncfnew = NetCDFFile(ofile,'w') ncfnew.createDimension('seastime', timeinf.dimt/4) ncfnew.createDimension('seas', 4) newvarobj = ncfnew.createVariable('seastime', 'f4', ('seastime', 'seas',), \ fill_value=fillValue) newvarobj[:]=timeseasvals attr = newvarobj.setncattr('calendar', timeinf.calendar) attr = newvarobj.setncattr('units', timeinf.units) newvarobj = ncfnew.createVariable('seasn', str, ('seas')) attr = newvarobj.setncattr('standard_name', 'seasn') attr = newvarobj.setncattr('long_name', 'name of the season') attr = newvarobj.setncattr('units', '3-months') newvarobj[0]='DJF' newvarobj[1]='MAM' newvarobj[2]='JJA' newvarobj[3]='SON' newdims=[] idim = 0 for ndim in varinf.dimns: if ndim == timename: newdims.append(unicode('seastime')) if not idim == 0: print errormsg print ' ' + fname + ': File "' + filename + '" does not have ' + \ 'time dimension "' + timename + '" as the first one. It has it as #'\ , idim, ' !!!!' ncf.close() quit(-1) else: newdims.append(ndim) if not ncfnew.dimensions.has_key(ndim): ncfnew.sync() ncfnew.close() ncf.close() fdimadd(filename + ',' + ndim, ofile) ncf = NetCDFFile(filename,'r') ncfnew = NetCDFFile(ofile,'a') print ncfnew.dimensions namedims=tuple(newdims) print namedims varobj = ncf.variables[varn] varinf = variable_inf(varobj) newvarobj=ncfnew.createVariable(varn + '_seasmean', 'f4', namedims, \ fill_value=fillValue) attr = newvarobj.setncattr('standard_name', varn + '_seasmean') attr = newvarobj.setncattr('long_name', 'seasonal mean of ' + varn) attr = newvarobj.setncattr('units', varinf.units) newvar = np.ones(varinf.dims[0], dtype=varinf.dtype) if varinf.Ndims == 1: newvar = newvarobj[:] newvarobj[:] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 2: for i in range(varinf.dims[1]): newvar = newvarobj[:,i] newvarobj[:,i] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 3: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): newvar = newvarobj[:,i,j] newvarobj[:,i,j] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 4: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): newvar = newvarobj[:,i,j,k] newvarobj[:,i,j,k] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 5: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): newvar = newvarobj[:,i,j,k,l] newvarobj[:,i,j,k,l] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 6: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): for m in range(varinf.dims[5]): newvar = newvarobj[:,i,j,k,l,m] newvarobj[:,i,j,k,l,m] = seasonal_means(netcdftimes, newvar) else: print errormsg print ' seasmean: number of dimensions ', varinf.Ndims, ' not ready !!!!' print errormsg ncf.close() quit(-1) ncfnew.sync() ncfnew.close() print 'seasmean: Seasonal mean file "' + ofile + '" written!' return def ivars(ncfile): """Give all the variable names of a file ncfile= netCDFfile name from which all variables will be given """ fname = 'ivars' if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' quit(-1) ncf = NetCDFFile(ncfile,'r') ncvars = ncf.variables allvars='' ivar = 0 for var in ncvars: print '@' + var + '@' if ivar == 0: allvars = var else: allvars = allvars + ':' + var ivar = ivar + 1 print ' # allvars= ' + allvars ncf.close() return ncvars def idims(ncfile): """Give all the dimensions names of a file ncfile= netCDFfile name from which all variables will be given """ fname = 'idims' if ncfile == 'h': print fname + '_____________________________________________________________' print idims.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' quit(-1) ncf = NetCDFFile(ncfile,'r') ncdims = ncf.dimensions alldims='' idim = 0 for dim in ncdims: print dim if idim == 0: alldims=dim else: alldims = alldims + ':' + dim idim = idim + 1 print ' # alldims= ' + alldims ncf.close() return list(ncdims) def igattrs(ncfile): """Give all the global attributes of a file """ if not os.path.isfile(ncfile): print errormsg print ' igattrs: File "' + ncfile + '" does not exist !!' print errormsg quit() ncf = NetCDFFile(ncfile,'r') ncattrs = ncf.ncattrs() allattrs='' allattrsdic = {} for attr in ncattrs: attrv=ncf.getncattr(attr) if type(attrv) == type(str('1')): attrk = 'S' elif type(attrv) == type(unicode('1')): attrk = 'S' elif type(attrv) == type(int(1)): attrk = 'I' elif type(attrv) == type(np.int(1)): attrk = 'Inp' elif type(attrv) == type(np.int32(1)): attrk = 'Inp32' elif type(attrv) == type(float(1.)): attrk = 'R' elif type(attrv) == type(np.float32(1.)): attrk = 'Rnp32' elif type(attrv) == type(np.float64(1.)): attrk = 'Rnp64' else: print errormsg print ' igattr: Reading attribute "', type(attrv), '" not ready! value:', attrv ncf.close() quit(-1) print attr, '|', attrv, '|', attrk allattrsdic[attr] = [attrv, attrk] allattrs=allattrs + ':' + attr + '|' + str(attrv) + '|' + attrk print '####### ###### ##### #### ### ## #' print '# allgattrs= ' + allattrs ncf.close() return allattrsdic def isgattrs(values, ncfile): """Give a single global attribute of a file and its type values = attribute name ncfile = netCDF file name output: attribute name, '|', attribute value, '|', attribute kind ('S', string '!' as spaces; 'I', integer; 'R', real; 'D', double ) """ if not os.path.isfile(ncfile): print errormsg print ' isgattrs: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'r') ncattrs = ncf.ncattrs() if not searchInlist(ncattrs, values): print ' isgattrs: File "' + ncfile + '" does not have global attribute: "' + values + '" ' ncf.close() quit(-1) attrv=ncf.getncattr(values) if type(attrv) == type(str('1')): attrk = 'S' attrv = attrv.replace(' ','!') elif type(attrv) == type(unicode('1')): attrk = 'S' attrv = attrv.replace(' ','!') elif type(attrv) == type(int(1)): attrk = 'I' elif type(attrv) == type(np.int(1)): attrk = 'I' elif type(attrv) == type(np.int32(1)): attrk = 'I' elif type(attrv) == type(float(1.)): attrk = 'R' elif type(attrv) == type(np.float32(1.)): attrk = 'R' elif type(attrv) == type(np.float64(1.)): attrk = 'D' else: print errormsg print ' isgattr: Reading attribute "', type(attrv), '" not ready! value:', attrv ncf.close() quit(-1) print values, '|', attrv, '|', attrk ncf.close() return [values, attrv, attrk] def ivattrs(ncfile, varn): """Give all the attributes of a variable and its type ncfile = netCDF file name var = variable name output: attribute name, '|', attribute value, '|', attribute kind ('S', string '!' as spaces; 'I', integer; 'R', real; 'D', double ) """ fname = 'ivattrs' if varn == 'h': print fname + '_____________________________________________________________' print ivattrs.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ": File '" + ncfile + "' does not exist !!" quit(-1) ncf = NetCDFFile(ncfile,'r') ncvars = ncf.variables if not gen.searchInlist(ncvars, varn): print ' ' + fname + ": File '" + ncfile + "' does not have variable: '" + \ varn + "' !!" ncf.close() quit(-1) varval = ncf.variables[varn] ncattrs = varval.ncattrs() allattrs='' allattrsdic = {} iattr = 0 for attr in ncattrs: attrv=varval.getncattr(attr) if type(attrv) == type(str('1')): attrk = 'S' attrv = attrv.replace(' ','!') elif type(attrv) == type(unicode('1')): attrk = 'S' attrv = attrv.replace(' ','!') elif type(attrv) == type(int(1)): attrk = 'I' elif type(attrv) == type(np.int(1)): attrk = 'I' elif type(attrv) == type(np.int32(1)): attrk = 'I' elif type(attrv) == type(float(1.)): attrk = 'R' elif type(attrv) == type(np.float32(1.)): attrk = 'R' elif type(attrv) == type(np.float64(1.)): attrk = 'D' else: print errormsg print ' ' + fname + ": Attribute type '", type(attrv), "' value:", attrv, \ "' not ready !!" ncf.close() quit(-1) print attr, '|', attrv, '|', attrk allattrsdic[attr] = [attrv, attrk] if iattr == 0: allattrs = attr + '|' + str(attrv) + '|' + attrk else: allattrs = allattrs + ':' + attr + '|' + str(attrv) + '|' + attrk iattr=iattr + 1 print '####### ###### ##### #### ### ## #' print '# allvattrs= ' + allattrs ncf.close() return allattrsdic def isvattrs(values, ncfile, varn): """Give a single attribute of a variable values = attribute name ncfile = netCDF file name varn = variable name """ fname = 'isvattrs' if values == 'h': print fname + '_____________________________________________________________' print isvattrs.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' isvattrs:File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'r') ncvars = ncf.variables if not searchInlist(ncvars, varn): print errormsg print ' isvattrs:"' + ncfile + '" does not have variable: "' + varn + '" ' ncf.close() quit(-1) varval = ncf.variables[varn] ncattrs = varval.ncattrs() if not searchInlist(ncattrs, values): print errormsg print ' isvattrs:' + ncfile + ' does not have global attribute: "' + values + '" ' ncf.close() quit(-1) attrv=varval.getncattr(values) if type(attrv) == type(str('1')): attrk = 'S' elif type(attrv) == type(unicode('1')): attrk = 'S' elif type(attrv) == type(int(1)): attrk = 'I' elif type(attrv) == type(np.int(1)): attrk = 'I' elif type(attrv) == type(np.int32(1)): attrk = 'I' elif type(attrv) == type(float(1.)): attrk = 'R' elif type(attrv) == type(np.float(1.)): attrk = 'R' elif type(attrv) == type(np.float32(1.)): attrk = 'R' elif type(attrv) == type(np.float64(1.)): attrk = 'D' else: print errormsg print ' isvattr: Reading attribute "', type(attrv), '" not ready! value:', attrv ncf.close() quit(-1) print values, '|', attrv, '|', attrk ncf.close() return [values, attrv, attrk] def grattr(values, ncfile): """ Function to read a global atribute values = attribute name ncfile = netCDF file name """ fname = 'grattr' if values == 'h': print fname + '_____________________________________________________________' print grattr.__doc__ quit() ncf = NetCDFFile(ncfile,'r') glob_attrs = ncf.ncattrs() attrPos = searchInlist(glob_attrs, values) if not attrPos: print errormsg print ' grattr: File "' + ncfile + '" does not have attribute "' + values + '" !!!!' print errormsg ncf.close() quit(-1) print ncf.getncattr(values) ncf.close() return def vrattr(values, ncfile, varn): """ Function to remove an atribute from a variable values = attribute name ncfile = netCDF file name varn = variable name """ fname = 'vrattr' if values == 'h': print fname + '_____________________________________________________________' print vrattr.__doc__ quit() ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ": File '" + ncfile + "' does not have variable '" + \ varn + "' !!!!" ncf.close() quit(-1) var = ncf.variables[varn] var_attrs = var.ncattrs() if not searchInlist(var_attrs, values): print errormsg print ' ' + fname + ": Variable '" + varn + "' does not have attribute '" + \ values + "' !!!!" ncf.close() quit(-1) else: attr = var.delncattr(values) ncf.close() return class cls_time_information(object): """ Classs to provide information about variable time ncfu = netCDF unit name tname = name of the variable time in [ncfu] self.calendar: calendar of the variable time self.unitsval: units value as it appears in attributes self.units: units of variable time self.Urefdate: reference date as it appears in the original 'units' section self.Srefdate: reference date as string [YYYY][MM][DD][HH][MI][SS] self.refdate: reference date self.attributes: attributes of time variable self.dimt: dimension of the time variable self.dt: distance between first two time-steps in self.tunits self.firstTu: first time value in self.units self.firstTt: first time value as datetime object self.firstTS: first time value as string [YYYY][MM][DD][HH][MI][SS] self.firstTm: first time value as matrix (from datetime; [Y], [M], [D], [H], [M], [S]) self.lastTu: last time value in self.units self.firstTt: last time value as datetime object self.lastTS: last time value as string [YYYY][MM][DD][HH][MI][SS] self.lastTm: last time value as matrix (from datetime; [Y], [M], [D], [H], [M], [S]) """ def __init__(self, ncfu, tname): import datetime as dt fname = 'cls_time_information' if ncfu is None: self.unitsval = None self.units = None self.calendar = None self.Urefdate = None self.Srefdate = None self.refdate = None self.attributes = None self.dimt = None self.dt = None self.firstTu = None self.firstTt = None self.firstTS = None self.firstTm = None self.lastTu = None self.lastTt = None self.lastTS = None self.lastTm = None else: times = ncfu.variables[tname] attvar = times.ncattrs() self.attributes = attvar if not searchInlist(attvar, 'units'): print warnmsg print ' ' + fname + ': time variable "' + tname + \ '" does not have attribute: "units"' units = None self.unitsval = None self.units = None self.calendar = None self.Urefdate = None self.Srefdate = None self.refdate = None if len(times.shape) == 1: self.dimt = len(times[:]) self.dt = times[1] - times[0] self.firstTu = times[0] self.firstTt = None self.firstTS = None self.firstTm = None self.lastTu = times[self.dimt] self.lastTt = None self.lastTS = None self.lastTm = None elif len(times.shape) == 2: if times.dimensions[1] == 'DateStrLen' and times.shape[1] == 19: print warnmsg print ' ' + fname + ": WRF 'Times' variable !!" self.unitsval = 'seconds since 1949-12-01 00:00:00' self.units = 'seconds' self.calendar = 'standard' self.Urefdate = '1949-12-01 00:00:00' self.Srefdate = '19491201000000' self.refdate = datetimeStr_datetime('1949-12-01_00:00:00') self.dimt = times.shape[0] self.firstTS = datetimeStr_conversion(times[0,:], \ 'WRFdatetime', 'YmdHMS') self.firstTm = datetimeStr_conversion(times[0,:], \ 'WRFdatetime', 'YmdHMS') self.lastTS = datetimeStr_conversion(times[self.dimt-1,:], \ 'WRFdatetime', 'YmdHMS') self.lastTm = datetimeStr_conversion(times[self.dimt-1,:], \ 'WRFdatetime', 'matYmdHMS') secondTS = datetimeStr_conversion(times[1,:], 'WRFdatetime', \ 'YmdHMS') secondTt = dt.datetime.strptime(secondTS, '%Y%m%d%H%M%S') self.firstTt = dt.datetime.strptime(self.firstTS, \ '%Y%m%d%H%M%S') self.lastTu = None self.lastTt = dt.datetime.strptime(self.lastTS, \ '%Y%m%d%H%M%S') difft = self.firstTt - secondTt self.dt = difft.days*3600.*24. + difft.seconds + \ difft.microseconds/10.e6 difft = self.firstTt - self.refdate self.firstTu = difft.days*3600.*24. + difft.seconds + \ difft.microseconds/10.e6 difft = self.lastTt - self.refdate self.lastTu = difft.days*3600.*24. + difft.seconds + \ difft.microseconds/10.e6 else: self.dimt = None self.dt = None self.firstTu = None self.firstTt = None self.firstTS = None self.firstTm = None self.lastTu = None self.lastTt = None self.lastTS = None self.lastTm = None else: units = times.getncattr('units') self.unitsval = units if not searchInlist(attvar, 'calendar'): print warnmsg print ' cls_time_information: time variable "', tname, \ '" does not have attribute: "calendar"' self.calendar = '-' else: self.calendar = times.getncattr('calendar') txtunits = units.split(' ') self.units = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: # print ' refdate with time!' self.refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2]+\ '_' + Srefdate) self.Urefdate = txtunits[len(txtunits) - 2] + ' ' + Srefdate else: self.refdate = datetimeStr_datetime(Srefdate + '_00:00:00') self.Urefdate = Srefdate self.Srefdate = self.refdate.strftime("%Y%m%d%H%M%S") timev = times[:] self.dimt = times.shape[0] self.dt = timev[1]-timev[0] self.firstTu = timev[0] self.firstTt = timeref_datetime(self.refdate, self.firstTu,self.units) self.firstTS = self.firstTt.strftime("%Y%m%d%H%M%S") self.firstTm = timeref_datetime_mat(self.refdate, self.firstTu, \ self.units) self.lastTu = timev[self.dimt-1] self.lastTt = timeref_datetime(self.refdate, self.lastTu, self.units) self.lastTS = self.lastTt.strftime("%Y%m%d%H%M%S") self.lastTm = timeref_datetime_mat(self.refdate, self.lastTu, \ self.units) return def time_information(ncfu, tname): """ Function to provide information about variable time ncfu = netCDF unit name tname = name of the variable time in [ncfu] """ times = ncfu.variables[tname] timeinf = [] attvar = times.ncattrs() if not searchInlist(attvar, 'units'): print errormsg print ' "time" does not have attribute: "units"' quit(-1) else: units = times.getncattr('units') txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: # print ' refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) timeinf.append(tunits) timeinf.append(Srefdate) timeinf.append(refdate) return timeinf class variable_inf(object): """ Class to provide the information from a given variable var = object netCDF variable self.name: name of the variable self.dtype: type of the variable self.attributes: list with the name of attributes self.FillValue: value of the missing value self.dimns: name of the dimensions of the variable self.dims: dimensions of the variable self.Ndims: number of dimensions self.dimx: length of dimension along x-axis self.dimy: length of dimension along y-axis self.sname: standard name self.lname: long name self.corr: attribute 'coordinates' self.units: units of the variable """ def __init__(self, var): if var is None: self.name = None self.dimns = None self.dims = None self.Ndims = None self.dimx = None self.dimy = None self.sname = None self.lname = None self.corr = None self.units = None self.FillValue = None self.dtype = None self.attributes = None else: self.name = var._name self.dimns = var.dimensions self.dtype = var.dtype self.attributes = var.ncattrs() self.dims = var.shape if gen.searchInlist(self.attributes, 'standard_name'): self.sname = var.getncattr('standard_name') else: self.sname = None if gen.searchInlist(self.attributes, 'long_name'): self.lname = var.getncattr('long_name') else: self.lname = None if gen.searchInlist(self.attributes, 'coordinates'): self.coor = var.getncattr('coordinates') else: self.coor = None if gen.searchInlist(self.attributes, 'units'): self.units = var.getncattr('units') else: self.units = None if gen.searchInlist(self.attributes, '_FillValue'): self.FillValue = var.getncattr('_FillValue') else: self.FillValue = None self.Ndims = len(self.dims) if self.Ndims == 1: self.dimx=self.dims[0] if self.Ndims == 2: self.dimy=self.dims[0] self.dimx=self.dims[1] if self.Ndims == 3: self.dimy=self.dims[1] self.dimx=self.dims[2] if self.Ndims == 4: self.dimy=self.dims[2] self.dimx=self.dims[3] return def subyrs(values, ncfile, varn): """ Function to retrieve a series of years from a file values = [year1]:[[year2]:...[yearn]] values for years [year1], [year2], ... [yearn] [yearI]-[yearE] values for the period between [yearI] and [yearN] ncfile = netCDF file name varn = variable name """ import datetime as dt import calendar as cal ofile = 'subsetyrs.nc' ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' subyrs: File does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) times = ncf.variables['time'] timevals = times[:] realdates = CFtimes_datetime(ncf, 'time') dimt = len(timevals) # Checking years ## isper = values.find('-') desiredvalues = np.array(dimt, dtype=bool) desiredvaluesi = np.array(dimt, dtype=bool) desiredvaluese = np.array(dimt, dtype=bool) if not isper == -1: # Years list given as a period [YYYYi]-[YYYYf] print ' subyrs: There is a period of years "' + values + '"' iyr = int(values.split('-')[0]) eyr = int(values.split('-')[1]) desiredvaluesi = np.array(realdates[:,0] >= iyr) desiredvaluese = np.array(realdates[:,0] <= eyr) desiredvalues = desiredvaluesi*desiredvaluese else: yrs = values.split(':') nyr = 1 for iyr in range(len(yrs)): desiredvaluesi = np.array(realdates[:,0] == int(yrs[iyr])) if nyr == 1: desiredvalues = desiredvaluesi else: desiredvalues = desiredvaluesi+desiredvalues nyr = nyr + 1 Ndesiredvalues = len(realdates[desiredvalues]) print ' subyrs: N values: ', Ndesiredvalues if Ndesiredvalues == 0: print errormsg print ' subyrs: No values found for "' + values + '"!!' ncf.close() quit(-1) # Variable values (assuming time as first dimension) ## var = ncf.variables[varn] # varvals = var[:] vardims = var.shape varshape = len(vardims) print ' subyrs: Shape of data: ',varshape, ':' , vardims if varshape == 1: vardesiredvalues = np.arange(Ndesiredvalues) vardesiredvalues = var[desiredvalues] elif varshape == 2: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]).reshape(Ndesiredvalues,vardims[1]) vardesiredvalues = var[desiredvalues, :] elif varshape == 3: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]).reshape(Ndesiredvalues, \ vardims[1],vardims[2]) vardesiredvalues = var[desiredvalues, :, :] elif varshape == 4: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]).reshape(Ndesiredvalues, \ vardims[1],vardims[2],vardims[3]) vardesiredvalues = var[desiredvalues, :, :, :] elif varshape == 5: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]*vardims[4]).reshape(Ndesiredvalues, \ vardims[1],vardims[2],vardims[3],vardims[4]) vardesiredvalues = var[desiredvalues, :, :, :, :] elif varshape == 6: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]*vardims[4]* \ vardims[5]).reshape(Ndesiredvalues,vardims[1],vardims[2],vardims[3],vardims[4],vardims[5]) vardesiredvalues = var[desiredvalues, :, :, :, :, :] else: print errormsg print ' subyrs: ', varshape, ' shape of matrix not prepared !' ncf.close() quit(-1) # print ' shape of desired values: ', vardesiredvalues.shape # Creation of file ## ncfo = NetCDFFile( ofile, 'w') vardims = var.dimensions vartype = var.dtype varattr = var.ncattrs() # Checking dimensions ## newdims = ncfo.dimensions for rdim in vardims: if not searchInlist(newdims, rdim): if not rdim == 'time': print ' subyrs: Adding dimension ' + rdim ncfo.sync() ncf.close() ncfo.close() fdimadd(ncfile + ',' + rdim, ofile) ncf = NetCDFFile(ncfile,'r') ncfo = NetCDFFile(ofile,'a') else: ncfo.createDimension('time', None) # Checking fill value ## if searchInlist(varattr, '_FillValue'): varfil = var._FillValue else: varfil = False newvar = ncfo.createVariable(varn, vartype, vardims, fill_value=varfil) if not varshape == 0: newvar[:] = vardesiredvalues newvar = ncfo.variables[varn] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = var.getncattr(attr) if not searchInlist(newvarattrs, attr): newvar.setncattr(attr, attrv) vardims = times.dimensions vartype = times.dtype varattr = times.ncattrs() newvar = ncfo.createVariable('time', vartype, vardims, fill_value=varfil) newvar = ncfo.variables['time'] newvar[:] = timevals[desiredvalues] ncf.close() ncfo.sync() ncfo.close() fattradd('time', ncfile + ':time', ofile) fvaradd(ncfile + ',lon', ofile) fvaradd(ncfile + ',lat', ofile) ncfo = NetCDFFile(ofile,'a') newvar = ncfo.variables['time'] newvarattrs = newvar.ncattrs() if searchInlist(newvarattrs, 'bounds'): if newvar.getncattr('bounds') == 'time_bnds': ncf = NetCDFFile(ncfile,'r') tbnds = ncf.variables['time_bnds'] vardims = tbnds.dimensions vartype = tbnds.dtype varattr = tbnds.ncattrs() ncfo.createDimension('bnds', 2) newvar = ncfo.createVariable('time_bnds', vartype, vardims, fill_value=varfil) newvar[:] = tbnds[desiredvalues,:] ncf.close() ncfo.sync() ncfo.close() fattradd('time_bnds', ncfile + ':time_bnds', ofile) else: ncfo.close() else: ncfo.close() fgaddattr(ncfile, ofile) print ' subyrs: File "' + ofile + '" with a subset of ' + values + ' has been created' return def submns(values, ncfile, varn): """ Function to retrieve a series of months from a file values = [mon1]:[[mion2]:...[monn]] values for months [mon1], [mon2], ... [monn] [monI]-[monE] values for the period between [monI] and [monN] ncfile = netCDF file name varn = variable name """ import datetime as dt import calendar as cal ofile = 'subsetmns.nc' ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' submns: File "' + ncfile + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) times = ncf.variables['time'] timevals = times[:] realdates = CFtimes_datetime(ncf, 'time') dimt = len(timevals) # Checking months ## isper =values.find('-') desiredvalues = np.array(dimt, dtype=bool) desiredvaluesi = np.array(dimt, dtype=bool) desiredvaluese = np.array(dimt, dtype=bool) if not isper == -1: # Months list given as a period [MMi]-[MMf] print ' submns: There is a period of months "' + values + '"' imn = int(values.split('-')[0]) emn = int(values.split('-')[1]) desiredvaluesi = np.array(realdates[:,1] >= imn) desiredvaluese = np.array(realdates[:,1] <= emn) desiredvalues = desiredvaluesi*desiredvaluese else: mns = values.split(':') nmn = 1 for imn in range(len(mns)): desiredvaluesi = np.array(realdates[:,1] == int(mns[imn])) if nmn == 1: desiredvalues = desiredvaluesi else: desiredvalues = desiredvaluesi+desiredvalues nmn = nmn + 1 Ndesiredvalues = len(realdates[desiredvalues]) print ' submns: N values: ', Ndesiredvalues if Ndesiredvalues == 0: print errormsg print ' submns: No values found for "' + values + '"!!' ncf.close() quit(-1) # Variable values (assuming time as first dimension) ## var = ncf.variables[varn] # varvals = var[:] vardims = var.shape varshape = len(vardims) # print ' submns: Shape of data: ',varshape, ':' , vardims if varshape == 1: vardesiredvalues = np.arange(Ndesiredvalues) vardesiredvalues = var[desiredvalues] elif varshape == 2: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]).reshape(Ndesiredvalues,vardims[1]) vardesiredvalues = var[desiredvalues, :] elif varshape == 3: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]).reshape(Ndesiredvalues, \ vardims[1],vardims[2]) vardesiredvalues = var[desiredvalues, :, :] elif varshape == 4: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]).reshape(Ndesiredvalues, \ vardims[1],vardims[2],vardims[3]) vardesiredvalues = var[desiredvalues, :, :, :] elif varshape == 5: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]*vardims[4]).reshape(Ndesiredvalues, \ vardims[1],vardims[2],vardims[3],vardims[4]) vardesiredvalues = var[desiredvalues, :, :, :, :] elif varshape == 6: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]*vardims[4]* \ vardims[5]).reshape(Ndesiredvalues,vardims[1],vardims[2],vardims[3],vardims[4],vardims[5]) vardesiredvalues = var[desiredvalues, :, :, :, :, :] else: print errormsg print ' submns: ', varshape, ' shape of matrix not prepared !' ncf.close() quit(-1) # print ' shape of desired values: ', vardesiredvalues.shape # Creation of file ## ncfo = NetCDFFile( ofile, 'w') vardims = var.dimensions vartype = var.dtype varattr = var.ncattrs() # Checking dimensions ## newdims = ncfo.dimensions for rdim in vardims: if not searchInlist(newdims, rdim): if not rdim == 'time': print ' submns: Adding dimension ' + rdim ncfo.sync() ncf.close() ncfo.close() fdimadd(ncfile + ',' + rdim, ofile) ncf = NetCDFFile(ncfile,'r') ncfo = NetCDFFile(ofile,'a') else: ncfo.createDimension('time', None) # Checking fill value ## if searchInlist(varattr, '_FillValue'): varfil = var._FillValue else: varfil = False newvar = ncfo.createVariable(varn, vartype, vardims, fill_value=varfil) if not varshape == 0: newvar[:] = vardesiredvalues newvar = ncfo.variables[varn] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = var.getncattr(attr) if not searchInlist(newvarattrs, attr): newvar.setncattr(attr, attrv) vardims = times.dimensions vartype = times.dtype varattr = times.ncattrs() newvar = ncfo.createVariable('time', vartype, vardims, fill_value=varfil) newvar = ncfo.variables['time'] newvar[:] = timevals[desiredvalues] ncf.close() ncfo.sync() ncfo.close() fattradd('time', ncfile + ':time', ofile) fvaradd(ncfile + ',lon', ofile) fvaradd(ncfile + ',lat', ofile) ncfo = NetCDFFile(ofile,'a') newvar = ncfo.variables['time'] newvarattrs = newvar.ncattrs() if searchInlist(newvarattrs, 'bounds'): if newvar.getncattr('bounds') == 'time_bnds': ncf = NetCDFFile(ncfile,'r') tbnds = ncf.variables['time_bnds'] vardims = tbnds.dimensions vartype = tbnds.dtype varattr = tbnds.ncattrs() ncfo.createDimension('bnds', 2) newvar = ncfo.createVariable('time_bnds', vartype, vardims, fill_value=varfil) newvar[:] = tbnds[desiredvalues,:] ncf.close() ncfo.sync() ncfo.close() fattradd('time_bnds', ncfile + ':time_bnds', ofile) else: ncfo.close() else: ncfo.close() fgaddattr(ncfile, ofile) print ' submns: File "' + ofile + '" with a subset of ' + values + ' has been created' return def spacemean(ncfile, varn): """ Function to retrieve a space mean series from a multidimensional variable of a file ncfile = netCDF file name varn = variable name """ import datetime as dt import calendar as cal ofile = 'spacemean_' + varn + '.nc' varfil=1.e20 statsn = ['min', 'max', 'mean', 'mean2', 'stdv', 'meanwgt', 'mean2wgt', 'stdvwgt', 'quant'] statslongn = ['minimum', 'maximum', 'mean', 'quadratic mean', 'standard deviation', 'weigthed mean', 'weigthed quadratic mean', 'weigthed standard deviation', 'quantiles'] ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' spacemean: File "' + ncfile + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) times = ncf.variables['time'] timevals = times[:] attvar = times.ncattrs() if not searchInlist(attvar, 'units'): print errormsg print ' spacemean: "time" does not have attribute: "units"' ncf.close() quit(-1) else: units = times.getncattr('units') txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: print ' spacemean: refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) dimt = len(timevals) realdates = np.zeros((dimt, 6), dtype=int) print realdates.shape ## Not in timedelta # if tunits == 'years': # for it in range(dimt): # realdate = refdate + dt.timedelta(years=float(times[it])) # realdates[it] = int(realdate.year) # elif tunits == 'months': # for it in range(dimt): # realdate = refdate + dt.timedelta(months=float(times[it])) # realdates[it] = int(realdate.year) if tunits == 'weeks': for it in range(dimt): realdate = refdate + dt.timedelta(weeks=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'days': for it in range(dimt): realdate = refdate + dt.timedelta(days=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'hours': for it in range(dimt): realdate = refdate + dt.timedelta(hours=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'minutes': for it in range(dimt): realdate = refdate + dt.timedelta(minutes=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'seconds': for it in range(dimt): realdate = refdate + dt.timedelta(seconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'milliseconds': for it in range(dimt): realdate = refdate + dt.timedelta(milliseconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) else: print errormsg print ' spacemean: time units "' + tunits + '" not ready!!!!' ncf.close() quit(-1) # Variable values (assuming time as first dimension) ## var = ncf.variables[varn] varinf = variable_inf(var) if searchInlist(varinf.attributes, 'scale_factor'): scalefact = var.getncattr('scale_factor') print ' spacemean: data has a scale factor of ', scalefact varinf.dtype=type(np.float(1.)) else: scalefact = 1. if searchInlist(varinf.attributes, 'add_offset'): offset = var.getncattr('add_offset') print ' spacemean: data has an offset of ', offset else: offset = 0. # varvals = var[:] vardims = var.shape varshape = len(vardims) vardimns = var.dimensions ## print ' spacemean: Shape of data: ',varshape, ':' , vardims dimt=vardims[0] vardnames = [] # Spatial average spatial weigthed ## lonvar = ncf.variables['lon'] latvar = ncf.variables['lat'] if not len(lonvar.shape) == 2: lonv = lonvar[:] latv = latvar[:] dx=lonvar.shape[0] dy=latvar.shape[0] lonval = np.zeros((dy, dx), dtype=np.float64) latval = np.zeros((dy, dx), dtype=np.float64) for iy in range(dy): lonval[iy,:] = lonv for ix in range(dx): latval[:,ix] = latv else: lonval = lonvar[:] latval = latvar[:] weightsv = abs(np.cos(latval*np.pi/180.)) if varinf.Ndims == 1: print errormsg print ' spacemean: You can not compute a space mean for a ', varinf.Ndims, 'D var!!!' ncf.close() quit(-1) elif varinf.Ndims== 2: print errormsg print ' spacemean: You can not compute a space mean for a ', varinf.Ndims, 'D var!!!' ncf.close() quit(-1) elif varinf.Ndims == 3: varstats = np.ones((dimt, 8), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((dimt, 21), dtype=np.float64) varquant = varquant*varfil vardnames.append(vardimns[0]) dy=vardims[1] dx=vardims[2] for it in range(dimt): if not varinf.FillValue is None: varvl = var[it,:,:] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[it,:,:] percendone(it,dimt,5,'computed space statistics') varst = statsVal(varval) varstwgt = statsValWeigthed(varval, weightsv) varstats[it,0] = varst.minv varstats[it,1] = varst.maxv varstats[it,2] = varst.meanv varstats[it,3] = varst.mean2v varstats[it,4] = varst.stdv varquant[it,:] = varst.quantilesv varstats[it,5] = varstwgt.meanv varstats[it,6] = varstwgt.mean2v varstats[it,7] = varstwgt.stdv elif varshape == 4: varstats = np.ones((dimt, vardims[1], 8), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((dimt, vardims[1], 21), dtype=np.float64) varquant = varquant*varfil vardimnames = (str(vardimns[0]), str(vardimns[1])) vardnames.append(vardimns[0]) vardnames.append(vardimns[1]) dy=vardims[2] dx=vardims[3] for it in range(dimt): for ik in range(vardims[1]): if not varinf.FillValue is None: varvl = var[it,ik,:,:] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[it,ik,:,:] percendone(it*vardims[1]+ik,dimt*vardims[1],5,'computed space statistics') varst = statsVal(varval) varstwgt = statsValWeigthed(varval, weightsv) varstats[it,ik,0] = varst.minv varstats[it,ik,1] = varst.maxv varstats[it,ik,2] = varst.meanv varstats[it,ik,3] = varst.mean2v varstats[it,ik,4] = varst.stdv varquant[it,ik,:] = varst.quantilesv varstats[it,ik,5] = varstwgt.meanv varstats[it,ik,6] = varstwgt.mean2v varstats[it,ik,7] = varstwgt.stdv elif varshape == 5: varstats = np.ones((dimt, vardims[1], vardims[2], 8), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((dimt, vardims[1], vardims[2], 21), dtype=np.float64) varquant = varquant*varfil vardnames.append(vardimns[0]) vardnames.append(vardimns[1]) vardnames.append(vardimns[2]) dy=vardims[3] dx=vardims[4] for it in range(dimt): for ik in range(vardims[1]): for il in range(vardims[2]): if not varinf.FillValue is None: varvl = var[it,ik,il,:,:] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[it,ik,il,:,:] percendone(it*vardims[1]*vardims[2]+ik*vardims[1]+il,dimt*vardims[1]*vardims[2],5,'computed space statistics') varst = statsVal(varval) varstwgt = statsValWeigthed(varval, weightsv) varstats[it,ik,il,0] = varst.minv varstats[it,ik,il,1] = varst.maxv varstats[it,ik,il,2] = varst.meanv varstats[it,ik,il,3] = varst.mean2v varstats[it,ik,il,4] = varst.stdv varquant[it,ik,il,:] = varst.quantilesv varstats[it,ik,il,5] = varstwgt.meanv varstats[it,ik,il,6] = varstwgt.mean2v varstats[it,ik,il,7] = varstwgt.stdv else: print errormsg print ' spacemean: ', varshape, ' shape of matrix not prepared !' ncf.close() quit(-1) vardimnames = tuple(vardnames) # print ' shape of desired values: ', vardesiredvalues.shape # Creation of file ## ncfo = NetCDFFile( ofile, 'w') vartype = var.dtype varattr = var.ncattrs() # Checking dimensions ## newdims = ncfo.dimensions Nvardnames = len(vardimnames) for idim in range(Nvardnames): rdim = vardimnames[idim] if not searchInlist(newdims, rdim): if not rdim == 'time': print ' spacemean: Adding dimension ' + rdim ncfo.sync() ncf.close() ncfo.close() fdimadd(ncfile + ',' + rdim, ofile) ncf = NetCDFFile(ncfile,'r') ncfo = NetCDFFile(ofile,'a') else: ncfo.createDimension('time', None) # Checking fill value ## if searchInlist(varattr, '_FillValue'): varfil = var._FillValue else: varfil = False Nstats = len(statsn) for ist in range(Nstats): if statsn[ist] == 'quant': print ist, statsn[ist]##, ': ', varquant[int(dimt/2),10] newdim = ncfo.createDimension('quant', 21) newvardnames = list(vardnames) newvardnames.append('quant') newvar = ncfo.createVariable(varn + statsn[ist], varinf.dtype, tuple(newvardnames), fill_value=varfil) newvar[:] = varquant else: print ist, statsn[ist]##, ': ', varstats[int(dimt/2),ist] newvar = ncfo.createVariable(varn + statsn[ist], varinf.dtype, vardimnames, fill_value=varfil) if varshape == 3: newvar[:] = varstats[:,ist]*1. elif varshape == 4: newvar[:] = varstats[:,:,ist]*1. elif varshape == 5: newvar[:] = varstats[:,:,:,ist]*1. newvar = ncfo.variables[varn + statsn[ist]] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = var.getncattr(attr) if not searchInlist(newvarattrs, attr): if attr != 'scale_factor' and attr != 'add_offset' and attr != 'valid_range' \ and attr != 'unpacked_valid_range' and attr != 'actual_range' : # if not statsn[ist] == 'stdv' and not statsn[ist] == 'stdvwgt': print 'attr:', attr newvar.setncattr(attr, attrv) # else: # newvar.setncattr(attr, attrv) newvar.setncattr('cell_methods', 'space ' + statslongn[ist] + ' all domain ' + str(dy) + 'x' + str(dx)) ## print ' Adding time variable' vartdims = times.dimensions vartype = times.dtype varattr = times.ncattrs() newvar = ncfo.createVariable('time', vartype, vartdims, fill_value=varfil) newvar = ncfo.variables['time'] newvar[:] = times ncf.close() ncfo.sync() ncfo.close() fattradd('time', ncfile + ':time', ofile) fvaradd(ncfile + ',lon', ofile) fvaradd(ncfile + ',lat', ofile) ncfo = NetCDFFile(ofile,'a') newvar = ncfo.variables['time'] newvarattrs = newvar.ncattrs() if searchInlist(newvarattrs, 'bounds'): if newvar.getncattr('bounds') == 'time_bnds': ncf = NetCDFFile(ncfile,'r') tbnds = ncf.variables['time_bnds'] vardims = tbnds.dimensions vartype = tbnds.dtype varattr = tbnds.ncattrs() ncfo.createDimension('bnds', 2) newvar = ncfo.createVariable('time_bnds', vartype, vardims, fill_value=varfil) newvar[:] = tbnds[:] ncf.close() ncfo.sync() ncfo.close() fattradd('time_bnds', ncfile + ':time_bnds', ofile) else: ncfo.close() else: ncfo.close() fgaddattr(ncfile, ofile) print ' spacemean: File "' + ofile + '" as space mean of "' + varn + '" has been created' return def timemean(values, ncfile, varn): """ Function to retrieve a time mean series from a multidimensional variable of a file values = power of the polynomial fitting with time to be applied ncfile = netCDF file name varn = variable name """ import datetime as dt import calendar as cal powerv=int(values) ofile = 'timemean_' + varn + '.nc' varfil=1.e20 statsn = ['min', 'max', 'mean', 'mean2', 'stdv', 'quant','linregress','polregress'] statslongn = ['minimum', 'maximum', 'mean', 'quadratic mean', 'standard deviation', 'quantiles', \ 'linear regression', 'polynomial regression'] ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' timemean: File "' + ncfile + '" does not have variable ' + varn + ' !!!!' print errormsg ncf.close() quit(-1) times = ncf.variables['time'] timevals = times[:] attvar = times.ncattrs() if not searchInlist(attvar, 'units'): print errormsg print ' timemean: "time" does not have attribute: "units"' ncf.close() quit(-1) else: units = times.getncattr('units') txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: print ' timemean: refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) dimt = len(timevals) realdates = np.zeros((dimt, 6), dtype=int) print realdates.shape ## Not in timedelta # if tunits == 'years': # for it in range(dimt): # realdate = refdate + dt.timedelta(years=float(times[it])) # realdates[it] = int(realdate.year) # elif tunits == 'months': # for it in range(dimt): # realdate = refdate + dt.timedelta(months=float(times[it])) # realdates[it] = int(realdate.year) if tunits == 'weeks': for it in range(dimt): realdate = refdate + dt.timedelta(weeks=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'days': for it in range(dimt): realdate = refdate + dt.timedelta(days=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'hours': for it in range(dimt): realdate = refdate + dt.timedelta(hours=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'minutes': for it in range(dimt): realdate = refdate + dt.timedelta(minutes=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'seconds': for it in range(dimt): realdate = refdate + dt.timedelta(seconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'milliseconds': for it in range(dimt): realdate = refdate + dt.timedelta(milliseconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) else: print errormsg print ' timemean: time units "' + tunits + '" not ready!!!!' ncf.close() quit(-1) timesv = (realdates[:,0] - realdates[0,0])*12 + realdates[:,1] # Variable values (assuming time as first dimension) ## var = ncf.variables[varn] varinf = variable_inf(var) # varvals = var[:] vardims = varinf.dims varshape = varinf.Ndims vardimns = var.dimensions ## print ' timemean: Shape of data: ',varshape, ':' , vardims dimt=vardims[0] vardnames = [] if varshape == 1: print errormsg print ' timemean: You can not compute a time mean for a ', varshape, 'D var!!!' ncf.close() quit(-1) elif varshape == 2: dx=vardims[1] varstats = np.ones((5, dx), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((21, dx), dtype=np.float64) varquant = varquant*varfil varlregress = np.ones((5, dx), dtype=np.float64) varlregress = varlregress*varfil varpregress = np.ones((powerv+1, dx), dtype=np.float64) varpregress = varpregress*varfil varpregressres = np.ones((dx), dtype=np.float64) varpregressres = varpregressres*varfil varpregresssingval = varpregress.copy() varpregresssingval = varpregresssingval*varfil vardnames.append(vardimns[1]) for ix in range(dx): if not varinf.FillValue is None: varvl = var[:,ix] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[:,ix] percendone(ix,dx,5,'computed time statistics') varst = statsVal(varval) var2st = stats2Val(timesv, varval, powerv) varstats[0,ix] = varst.minv varstats[1,ix] = varst.maxv varstats[2,ix] = varst.meanv varstats[3,ix] = varst.mean2v varstats[4,ix] = varst.stdv varquant[:,ix] = varst.quantilesv varlregress[:,ix] = var2st.linRegress varpregress[:,ix] = var2st.polRegress varpregressres[ix] = var2st.polRegressRes varpregresssingval[:,ix] = var2st.polRegressSingVal elif varshape == 3: dy=vardims[1] dx=vardims[2] varstats = np.ones((5,dy, dx), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((21,dy, dx), dtype=np.float64) varquant = varquant*varfil varlregress = np.ones((5,dy, dx), dtype=np.float64) varlregress = varlregress*varfil varpregress = np.ones((powerv+1,dy, dx), dtype=np.float64) varpregress = varpregress*varfil varpregressres = np.ones((dy,dx), dtype=np.float64) varpregressres = varpregressres*varfil varpregresssingval = varpregress.copy() varpregresssingval = varpregresssingval*varfil vardnames.append(vardimns[1]) vardnames.append(vardimns[2]) for iy in range(dy): for ix in range(dx): if not varinf.FillValue is None: varvl = var[:,iy,ix] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[:,iy,ix] percendone(iy*dx+ix,dy*dx,5,'computed time statistics') varst = statsVal(varval) var2st = stats2Val(timesv, varval, powerv) varstats[0,iy,ix] = varst.minv varstats[1,iy,ix] = varst.maxv varstats[2,iy,ix] = varst.meanv varstats[3,iy,ix] = varst.mean2v varstats[4,iy,ix] = varst.stdv varquant[:,iy,ix] = varst.quantilesv varlregress[:,iy,ix] = var2st.linRegress varpregress[:,iy,ix] = var2st.polRegress varpregressres[iy,ix] = var2st.polRegressRes varpregresssingval[:,iy,ix] = var2st.polRegressSingVal elif varshape == 4: dz=vardims[1] dy=vardims[2] dx=vardims[3] varstats = np.ones((5, dz, dy, dx), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((21, dz, dy, dx), dtype=np.float64) varquant = varquant*varfil varlregress = np.ones((5, dz, dy, dx), dtype=np.float64) varlregress = varlregress*varfil varpregress = np.ones((powerv+1, dz, dy, dx), dtype=np.float64) varpregress = varpregress*varfil varpregressres = np.ones((dz,dy,dx), dtype=np.float64) varpregressres = varpregressres*varfil varpregresssingval = varpregress.copy() varpregresssingval = varpregresssingval*varfil vardnames.append(vardimns[1]) vardnames.append(vardimns[2]) vardnames.append(vardimns[3]) for iz in range(dz): for iy in range(dy): for ix in range(dx): if not varinf.FillValue is None: varvl = var[:,iz,iy,ix] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[:,iz,iy,ix] percendone(iz*dy*dx+iy*dx+ix,dz*dx*dy,5,'computed time statistics') varst = statsVal(varval) var2st = stats2Val(timesv, varval, powerv) varstats[0,iz,iy,ix] = varst.minv varstats[1,iz,iy,ix] = varst.maxv varstats[2,iz,iy,ix] = varst.meanv varstats[3,iz,iy,ix] = varst.mean2v varstats[4,iz,iy,ix] = varst.stdv varquant[:,iz,iy,ix] = varst.quantilesv varlregress[:,iz,iy,ix] = var2st.linRegress varpregress[:,iz,iy,ix] = var2st.polRegress varpregressres[iz,iy,ix] = var2st.polRegressRes varpregresssingval[:,iz,iy,ix] = var2st.polRegressSingVal elif varshape == 5: dn=vardims[1] dz=vardims[2] dy=vardims[3] dx=vardims[4] varstats = np.ones((5, dn, dz, dy, dx), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((21, dn, dz, dy, dx), dtype=np.float64) varquant = varquant*varfil varlregress = np.ones((5, dn, dz, dy, dx), dtype=np.float64) varlregress = varlregress*varfil varpregress = np.ones((powerv+1, dn, dz, dy, dx), dtype=np.float64) varpregress = varpregress*varfil varpregressres = np.ones((dn,dz,dy,dx), dtype=np.float64) varpregressres = varpregressres*varfil varpregresssingval = varpregress.copy() varpregresssingval = varpregresssingval*varfil vardnames.append(vardimns[1]) vardnames.append(vardimns[2]) vardnames.append(vardimns[3]) vardnames.append(vardimns[4]) for iN in range(dn): for iz in range(dz): for iy in range(dy): for ix in range(dx): if not varinf.FillValue is None: varvl = var[:,iN,iz,iy,ix] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[:,iN,iz,iy,ix] percendone(iN*dy*dx*dz+iz*dy*dx+iy*dx+ix,dn*dz*dx*dy,5,'computed time statistics') varst = statsVal(varval) var2st = stats2Val(timesv, varval, powerv) varstats[0,iN,iz,iy,ix] = varst.minv varstats[1,iN,iz,iy,ix] = varst.maxv varstats[2,iN,iz,iy,ix] = varst.meanv varstats[3,iN,iz,iy,ix] = varst.mean2v varstats[4,iN,iz,iy,ix] = varst.stdv varquant[:,iN,iz,iy,iiN,x] = varst.quantilesv varlregress[:,iN,iz,iy,ix] = var2st.linRegress varpregress[:,iN,iz,iy,ix] = var2st.polRegress varpregressres[iN,iz,iy,ix] = var2st.polRegressRes varpregresssingval[:,iN,iz,iy,ix] = var2st.polRegressSingVal else: print errormsg print ' timemean: ', varshape, ' shape of matrix not prepared !' ncf.close() quit(-1) vardimnames = tuple(vardnames) # print ' shape of desired values: ', vardesiredvalues.shape # Creation of file ## ncfo = NetCDFFile( ofile, 'w') vartype = var.dtype varattr = var.ncattrs() # Checking dimensions ## newdims = ncfo.dimensions Nvardnames = len(vardimnames) for idim in range(Nvardnames): rdim = vardimnames[idim] if not searchInlist(newdims, rdim): if not rdim == 'time': ## print ' timemean: Adding dimension ' + rdim ncfo.sync() ncf.close() ncfo.close() fdimadd(ncfile + ',' + rdim, ofile) ncf = NetCDFFile(ncfile,'r') ncfo = NetCDFFile(ofile,'a') else: print ' timemean: No time dimension!' # Checking fill value ## if searchInlist(varattr, '_FillValue'): varfil = var._FillValue else: varfil = False Nstats = len(statsn) for ist in range(Nstats): newvardnames = [] if statsn[ist] == 'quant': print ist, statsn[ist]##, ': ', varquant[int(dimt/2),10] newdim = ncfo.createDimension('quant', 21) newvardnames.append('quant') newvardnames = newvardnames + list(vardnames) newvar = ncfo.createVariable(varn + statsn[ist], vartype, tuple(newvardnames), fill_value=varfil) newvar[:] = varquant elif statsn[ist] == 'linregress': print ist, statsn[ist]##, ': ', varquant[int(dimt/2),10] newdim = ncfo.createDimension('lregress', 5) newvar = ncfo.createVariable('lregressn', str, ('lregress',)) newvar[0] = 'slope' newvar[1] = 'intercept' newvar[2] = 'r_value' newvar[3] = 'p_value' newvar[4] = 'std_err' newvardnames.append('lregress') newvardnames = newvardnames + list(vardnames) newvar = ncfo.createVariable(varn + statsn[ist], 'f4', tuple(newvardnames), fill_value=varfil) newvar[:] = varlregress*1. elif statsn[ist] == 'polregress': print ist, statsn[ist]##, ': ', varquant[int(dimt/2),10] newdim = ncfo.createDimension('pregress', powerv+1) newvar = ncfo.createVariable('pregressn', str, ('pregress',)) for ip in range(powerv+1): newvar[ip]='coefficient**' + str(powerv-ip) newvardnames.append('pregress') newvardnames = newvardnames + list(vardnames) newvar = ncfo.createVariable(varn + statsn[ist], 'f4', tuple(newvardnames), fill_value=varfil) newvar[:] = varpregress*1. newvar.setncattr('power',powerv) newvar.setncattr('values','Polynomial coefficients, highest power first') newvar = ncfo.createVariable(varn + statsn[ist] + '_Residual', vartype, tuple(vardnames), fill_value=varfil) newvar[:] = varpregressres newvar.setncattr('power',powerv) newvar.setncattr('values','Polynomial residuals') newvar = ncfo.createVariable(varn + statsn[ist] + '_VandermondeSingularVector', vartype, tuple(newvardnames), fill_value=varfil) newvar[:] = varpregresssingval newvar.setncattr('power',powerv) newvar.setncattr('values','Polynomial coefficients, highest power first') else: print ist, statsn[ist]##, ': ', varstats[int(dimt/2),ist] if statsn[ist] == 'mean' or statsn[ist] == 'stdv' or statsn[ist] == 'mean2' or statsn[ist] == 'polregress_Residual' \ or statsn[ist] == 'polregress_VandermondeSingularVector' and searchInlist(varattr, 'scale_factor'): newvar = ncfo.createVariable(varn + statsn[ist], 'f4', vardimnames, fill_value=varfil) if varshape == 2: newvar[:] = varstats[ist,:]*1. elif varshape == 3: newvar[:] = varstats[ist,:,:]*1. elif varshape == 4: newvar[:] = varstats[ist,:,:,:]*1. elif varshape == 5: newvar[:] = varstats[ist,:,:,:,:]*1. else: newvar = ncfo.createVariable(varn + statsn[ist], vartype, vardimnames, fill_value=varfil) if varshape == 2: newvar[:] = varstats[ist,:] elif varshape == 3: newvar[:] = varstats[ist,:,:] elif varshape == 4: newvar[:] = varstats[ist,:,:,:] elif varshape == 5: newvar[:] = varstats[ist,:,:,:,:] newvar = ncfo.variables[varn + statsn[ist]] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = var.getncattr(attr) if not searchInlist(newvarattrs, attr): if attr == 'scale_factor' or attr == 'add_offset' or attr == 'valid_range' \ or attr == 'unpacked_valid_range' or attr == 'actual_range' : if not statsn[ist] == 'mean' and not statsn[ist] == 'stdv' and not statsn[ist] == 'mean2' and not \ statsn[ist] == 'linregress' and not statsn[ist] == 'polregress' \ and not statsn[ist] == 'polregress_Residual' and not \ statsn[ist] == 'polregress_VandermondeSingularVector': newvar.setncattr(attr, attrv) else: newvar.setncattr(attr, attrv) newvar.setncattr('cell_methods', 'time ' + statslongn[ist] + ' all period in file ' + str(dimt) + ' time steps') ncfo.sync() ncfo.close() fvaradd(ncfile + ',lon', ofile) fvaradd(ncfile + ',lat', ofile) fgaddattr(ncfile, ofile) print ' timemean: File "' + ofile + '" as time mean of "' + varn + '" has been created' return def flipdim(values, filename, varn): """ flips the value following one dimension [Nflipdim] values=[Nflipdim]:[flipdim] [Nflipdim]: number of the dimension to flip [flipdim]: has also the corresondant variable dimension be flipped? (yes/no) filename= netCDF file name varn= variable name """ fname = 'flipdim' if values == 'h': print fname + '_____________________________________________________________' print flipdim.__doc__ quit() arguments = '[Nflipdim]:[flipdim]' gen.check_arguments(fname, values, arguments, ':') ncf = NetCDFFile(filename,'a') Nflipdim=int(values.split(':')[0]) flipdim=values.split(':')[1] if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ': File "' + filename + '" does not have variable ' +\ varn + ' !!!!' print errormsg ncf.close() quit(-1) varnc = ncf.variables[varn] varinf = variable_inf(varnc) if varinf.Ndims < Nflipdim: print errormsg print ' ' + fname + ': variable "' + varn + '" has less dimensions ', \ varinf.Ndims, 'than the required to flip "', Nflipdim, '!!!!' quit(-1) if flipdim == 'yes': flipdimname = varinf.dimns[Nflipdim] print ' ' + fname + ': Flipping also associated dimension variable "' + \ flipdimname + '"' flipdim = ncf.variables[flipdimname] if len(flipdim.shape) == 1: newdim = flipdim[:] flipdim[:] = newdim[::-1] elif len(flipdim.shape) == 2: if Nflipdim == varinf.Ndims: for i in range(flipdim.shape[0]): newdim = flipdim[i,:] flipdim[i,:] = newdim[::-1] if Nflipdim == varinf.Ndims - 1: for i in range(flipdim.shape[1]): newdim = flipdim[:,i] flipdim[:,:] = newdim[::-1] else: print errormsg print ' ' + fname + ': dimension to flip has ', len(flipdim.shape), \ ' dimensions, not ready to be flipped !!!' quit(-1) newvar = np.ones(varinf.dims, dtype=varinf.dtype) if varinf.Ndims == 1: newvar = varnc[:] varnc[:] = newvar[::-1] elif varinf.Ndims == 2: if Nflipdim == 1: for i in range(varinf.dims[0]): newvar[i,:] = varnc[i,:] varnc[i,:] = newvar[i,::-1] elif Nflipdim == 0: for i in range(varinf.dims[1]): newvar[:,i] = varnc[:,i] varnc[:,i] = newvar[::-1,i] elif varinf.Ndims == 3: if Nflipdim == 2: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): newvar[i,j,:] = varnc[i,j,:] varnc[i,j,:] = newvar[i,j,::-1] elif Nflipdim == 1: for i in range(varinf.dims[0]): for j in range(varinf.dims[2]): newvar[i,:,j] = varnc[i,:,j] varnc[i,:,j] = newvar[i,::-1,j] elif Nflipdim == 0: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): newvar[:,i,j] = varnc[:,i,j] varnc[:,i,j] = newvar[::-1,i,j] elif varinf.Ndims == 4: if Nflipdim == 3: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[2]): newvar[i,j,k,:] = varnc[i,j,k,:] varnc[i,j,k,:] = newvar[i,j,k,::-1] if Nflipdim == 2: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[3]): newvar[i,j,:,k] = varnc[i,j,:,k] varnc[i,j,:,k] = newvar[i,j,::-1,k] if Nflipdim == 1: for i in range(varinf.dims[0]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): newvar[i,:,j,k] = varnc[i,:,j,k] varnc[i,:,j,k] = newvar[i,::-1,j,k] if Nflipdim == 0: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): newvar[:,i,j,k] = varnc[:,i,j,k] varnc[:,i,j,k] = newvar[::-1,i,j,k] elif varinf.Ndims == 5: if Nflipdim == 4: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[2]): for l in range(varinf.dims[3]): newvar[i,j,k,l,:] = varnc[i,j,k,l,:] varnc[i,j,k,l,:] = newvar[i,j,k,l,::-1] if Nflipdim == 3: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[2]): for l in range(varinf.dims[4]): newvar[i,j,k,:,l] = varnc[i,j,k,:,l] var[i,j,k,:,l] = newvar[i,j,k,::-1,l] if Nflipdim == 2: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): newvar[i,j,:,k,l] = varnc[i,j,:,k,l] varnc[i,j,:,k,l] = newvar[i,j,::-1,k,l] if Nflipdim == 1: for i in range(varinf.dims[0]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): newvar[i,:,j,k,l] = varnc[i,:,j,k,l] varnc[i,:,j,k,l] = newvar[i,::-1,j,k,l] if Nflipdim == 0: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): newvar[:,i,j,k,l] = varnc[:,i,j,k,l] varnc[:,i,j,k,l] = newvar[::-1,i,j,k,l] else: print errormsg print ' ' + fname + ': Number of dimensions ', varinf.Ndims, \ ' not ready!!!!' quit(-1) ncf.sync() ncf.close() return def load_ncvariable_lastdims(ncf, varn, prevdims, Nlastdims): """ Function to load the last [Nlastdims] dimensions of a variable [varn] from a netCDF object at the other dimensions at [prevdims] ncf= netCDF object varn= variable name prevdims= values at the previous dimensions Nlastims= number of last dimensions """ fname='load_ncvariable_lastdims' var=ncf.variables[varn] dims=var.shape Ndims=len(dims) Nprevdims = len(prevdims) if not Nprevdims + Nlastdims == Ndims: print erromsg print ' ' + fname + ': number of dimensions previous (',Nprevdim,') and last (',Nlastdims, ') does not match with variable size: ',Ndims,' !!!!' print errormsg quit(-1) if Nlastdims > Ndims-1: print errormsg print ' ' + fname + ': number of last dimensions ', Nlastdims,' >= than the number of dimensions of the variable ', Ndims,' !!!!' print errormsg quit(-1) if Ndims == 1: loadvar = var[:] elif Ndims == 2: loadvar = var[prevdims[0], :] elif Ndims == 3: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], :] else: loadvar = var[prevdims[0], :, :] elif Ndims == 4: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], :, :] else: loadvar = var[prevdims[0], :, :, :] elif Ndims == 5: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :, :] elif Nlastdims == 3: loadvar = var[prevdims[0], prevdims[1], :, :, :] else: loadvar = var[prevdims[0], :, :, :, :] elif Ndims == 6: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], prevdims[4], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :, :] elif Nlastdims == 3: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :, :, :] elif Nlastdims == 4: loadvar = var[prevdims[0], prevdims[1], :, :, :, :] else: loadvar = var[prevdims[0], :, :, :, :] else: print errormsg print ' ' + fname + ' variable size ', Ndims, ' not ready!!!' print errormsg quit(-1) return loadvar def fill_ncvariable_lastdims(ncf, varn, prevdims, Nlastdims, fillvals): """ Function to fill the last [Nlastdims] dimensions of a variable [varn] from a netCDF object at the other dimensions at [prevdims] with as given set of [fillvals] values ncf= netCDF object varn= variable name prevdims= values at the previous dimensions Nlastims= number of last dimensions fillvals= values to use to fill """ fname='fill_ncvariable_lastdims' var=ncf.variables[varn] dims=var.shape Ndims=len(dims) Nprevdims = len(prevdims) if not Nprevdims + Nlastdims == Ndims: print erromsg print ' ' + fname + ': number of dimensions previous (',Nprevdim,') and last (',Nlastdims, ') does not match with variable size: ',Ndims,' !!!!' print errormsg quit(-1) if Nlastdims > Ndims-1: print errormsg print ' ' + fname + ': number of last dimensions ', Nlastdims,' >= than the number of dimensions of the variable ', Ndims,' !!!!' print errormsg quit(-1) if Ndims == 1: var[:] = fillvals elif Ndims == 2: var[prevdims[0], :] = fillvals elif Ndims == 3: if Nlastdims == 1: var[prevdims[0], prevdims[1], :] = fillvals else: var[prevdims[0], :, :] = fillvals elif Ndims == 4: if Nlastdims == 1: var[prevdims[0], prevdims[1], prevdims[2], :] = fillvals elif Nlastdims == 2: var[prevdims[0], prevdims[1], :, :] = fillvals else: var[prevdims[0], :, :, :] = fillvals elif Ndims == 5: if Nlastdims == 1: var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :] = fillvals elif Nlastdims == 2: var[prevdims[0], prevdims[1], prevdims[2], :, :] = fillvals elif Nlastdims == 3: var[prevdims[0], prevdims[1], :, :, :] = fillvals else: var[prevdims[0], :, :, :, :] = fillvals elif Ndims == 6: if Nlastdims == 1: var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], prevdims[4], :] = fillvals elif Nlastdims == 2: var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :, :] = fillvals elif Nlastdims == 3: var[prevdims[0], prevdims[1], prevdims[2], :, :, :] = fillvals elif Nlastdims == 4: var[prevdims[0], prevdims[1], :, :, :, :] = fillvals else: var[prevdims[0], :, :, :, :] = fillvals else: print errormsg print ' ' + fname + ' variable size ', Ndims, ' not ready!!!' print errormsg quit(-1) return def maskvar(values, filename, varparameters): """ Function to mask a variable using another variable to mask it. Only need to share at least 1 dimension with the same size (no need same name) values= [maskfilename]:[maskvarn]:[dimsmask]:[maskvalue] [maskfilename]= file with the mask [maskvarn]= name of the variable with the mask [dimsmask]= ',' list of [dimn]|[sec] to use for slicing [maskvar] [sec]: section * [integer]: which value of the dimension * -1: all along the dimension * -9: last value of the dimension * [beg]@[end] slice from [beg] to [end] * NOTE: no value provided is for all dimension range [maskvalue]= value to use for maskking ('mask' value for using an already masked variable) filename= netCF file name varparameters= [varn]:[dimsmask] [varn]: variable name [samedimsmask]: ',' list of name of dimensions which fits [maskvarn] slice """ fname='maskvar' if values == 'h': print fname + '_____________________________________________________________' print maskvar.__doc__ quit() expectargs = '[maskfilename]:[maskvarn]:[dimsmask]:[maskvalue]' gen.check_arguments(fname,values,expectargs,':') maskfilename = values.split(':')[0] maskvarn = values.split(':')[1] dimsmask = values.split(':')[2] maskvalue = values.split(':')[3] varn = varparameters.split(':')[0] samedimsmask = varparameters.split(':')[1].split(',') ofile = 'mask_' + varn + '.nc' ncf = NetCDFFile(filename,'r') if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ": File '" + filename + "' does not have variable '"+\ varn + '" !!!!' print errormsg ncf.close() quit(-1) varobj = ncf.variables[varn] varinf = variable_inf(varobj) ncfmask = NetCDFFile(maskfilename,'r') if not ncfmask.variables.has_key(maskvarn): print errormsg print ' ' + fname + ": File '" + maskfilename + "' does not have " + \ " variable mask '" + maskvarn + "' !!" print errormsg ncf.close() ncfmask.close() quit(-1) varmaskobj = ncfmask.variables[maskvarn] varmaskinf = variable_inf(varmaskobj) maskvalues, maskdims = slice_variable(varmaskobj, dimsmask.replace('|',':'). \ replace(',','|')) ncfmask.close() # Looking for variable & mask compatibility runningsize = 1 rundims = [] testslicevar = [] runshape = [] for dimn in varobj.dimensions: Ldim = len(ncf.dimensions[dimn]) if gen.searchInlist(samedimsmask,dimn): testslicevar.append(slice(0,Ldim,None)) else: runningsize = runningsize*Ldim rundims.append(dimn) runshape.append(Ldim) testslicevar.append(0) testvar = varobj[tuple(testslicevar)] gen.same_shape(testvar,maskvalues) # Creation of the new file ## ncfnew = NetCDFFile(ofile,'w') for idim in range(varinf.Ndims): newdimv = ncf.dimensions[varinf.dimns[idim]] if newdimv.isunlimited(): ncfnew.createDimension(varinf.dimns[idim], None) else: ncfnew.createDimension(varinf.dimns[idim], varinf.dims[idim]) # FillVallue if varinf.dtype == type(np.int(1)): fillValue = gen.fillValueI elif varinf.dtype == type('c'): fillValue = gen.fillValueC elif varinf.dtype == type(np.int16(1)) or varinf.dtype == type(np.int32(1)): fillValue = gen.fillValueI16 elif varinf.dtype == type(np.float(1.)): fillValue = gen.fillValueF elif varinf.dtype == type(np.float32(1.)) or varinf.dtype == type(np.float64(1.)): fillValue = gen.fillValueF64 else: print errormsg print ' ' + fname + ': type of variable:', varinf.dtype, 'not ready!!' quit(-1) if maskvalue != 'mask': maskval = gen.retype(maskvalue,varinf.dtype) else: print infmsg print ' ' + fname + ': maskgin with already masked variable' varmaskv = maskvalues.mask newvar = ncfnew.createVariable(varn, varinf.dtype, varinf.dimns, \ fill_value=fillValue) for attrn in varinf.attributes: if not attrn == '_FillValue': attr = newvar.setncattr(attrn, varobj.getncattr(attrn)) attr = newvar.setncattr('mask', 'variable masked using ' + values.split(':')[2]+ \ ' of variable ' + maskvarn + ' from file ' + maskfilename) newvar[:] = varobj[:] ncfnew.sync() ncfnew.close() for varns in ncf.variables: if not varns == varn: try: with gen.Capturing() as output: fvaradd(filename + ',' + varns, ofile) except: print errmsg print 'fvaradd(' + filename + ', ' + varns + ',' + ofilen + ')' for s1out in output: print s1out quit(-1) ncfnew = NetCDFFile(ofile,'a') varobj = ncfnew.variables[varn] # Masking following that dimensions which are not coincident with the mask # Check if there are some coincident dimensional names between both variables vdimns = list(varinf.dimns) mdimns = maskdims for dimn in mdimns: if gen.searchInlist(vdimns,dimn): vdimns.remove(dimn) #slices = gen.provide_slices(varinf.dimns, varinf.dims, list(varmaskinf.dimns)) slices = gen.provide_slices(varinf.dimns, varinf.dims, vdimns) if maskvalue != 'mask': maskTOuse = ma.masked_equal(maskvalues, maskval) maskTOusev = maskTOuse.mask else: maskTOusev = maskvalues.mask for ir in range(runningsize): islice = slices[ir] varvals = varobj[tuple(islice)] maskvarvals = np.where(maskTOusev, fillValue, varvals) newvar[tuple(islice)] = maskvarvals ncfnew.sync() ncfnew.close() ncf.close() print 'masked file "' + ofile + '" generated !!!' return def maskvarVal(values, filename, varparameters): """ Function to mask a variable as combination of values from other variables from another file and using a list of resultant variables to mask a variable by inter-matching values values= [maskfilename]:[maskvarns]:[dimsmask] [maskfilename]= file with the mask [maskvarns]= ',' list of [maskvarn1]|maskval1]|[comp],[...[maskvarnN]|[maskvalN]|[compN]] triplets of variable name, value and comparison to use to mask [comp]: comparisons 'eq:' masking all values equal than [maskvali] 'ge:' masking all values bigger or equal than [maskvali] 'gt:' masking all values bigger than [maskvali] 'le:' masking all values smaller or equal than [maskvali] 'lt:' masking all values smaller than [maskvali] [dimsmask]= ',' list of [dimn]|[sec] to use for slicing all [maskvari] [sec]: section * [integer]: which value of the dimension * -1: all along the dimension * -9: last value of the dimension * [beg]@[end] slice from [beg] to [end] * NOTE: no value provided is for all dimension range [matchingmaskvars]: ',' of name of variables from masking file, the non-masked values of which will be used to mask the variable filename= netCF file name varparameters= [varn]:[dimsmask] [varns]: ',' list of variable names in the same order than [matchingmaskvars] to search for coincidencies [applydimsmask]: ',' list of name of dimensions along which apply the mask """ fname='maskvarVal' if values == 'h': print fname + '_____________________________________________________________' print maskvarVal.__doc__ quit() expectargs = '[maskfilename]:[maskvarns]:[dimsmask]' gen.check_arguments(fname,values,expectargs,':') maskfilename = values.split(':')[0] maskvarns = gen.str_list(values.split(':')[1], ',') dimsmask = values.split(':')[2] varns = gen.str_list(varparameters.split(':')[0], ',') applydimsmask = gen.str_list(varparameters.split(':')[1], ',') ofile = 'mask_' + '-'.join(varns) + '.nc' # Masking variables ncfmask = NetCDFFile(maskfilename,'r') applymasks = [] for mvarns in maskvarns: maskvarn = mvarns.split('|')[0] maskval = mvarns.split('|')[1] maskcomp = mvarns.split('|')[2] if not ncfmask.variables.has_key(maskvarn): print errormsg print ' ' + fname + ": File '" + maskfilename + "' does not have " + \ " variable mask '" + maskvarn + "' !!" print errormsg ncf.close() ncfmask.close() quit(-1) varmaskobj = ncfmask.variables[maskvarn] varmaskinf = variable_inf(varmaskobj) mval = gen.retype(maskval,varmaskinf.type) maskvalues, maskdims = slice_variable(varmaskobj, dimsmask.replace('|',':'). \ replace(',','|')) if maskcomp == 'eq': applymasks[ma.masked_equal(maskvalues, mval)] elif maskcomp == 'ge': applymasks[ma.masked_equal(maskvalues, mval)] print 'NOT finished !!!!' quit(-1) ncfmask.close() ncf = NetCDFFile(filename,'r') for varn in varns: if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ": File '" + filename + "' does not have " + \ "variable '" + varn + '" !!!!' print errormsg ncf.close() quit(-1) varobj = ncf.variables[varn] varinf = variable_inf(varobj) # Looking for variable & mask compatibility runningsize = 1 rundims = [] testslicevar = [] runshape = [] for dimn in varobj.dimensions: Ldim = len(ncf.dimensions[dimn]) if gen.searchInlist(samedimsmask,dimn): testslicevar.append(slice(0,Ldim,None)) else: runningsize = runningsize*Ldim rundims.append(dimn) runshape.append(Ldim) testslicevar.append(0) testvar = varobj[tuple(testslicevar)] gen.same_shape(testvar,maskvalues) # Creation of the new file ## ncfnew = NetCDFFile(ofile,'w') for idim in range(varinf.Ndims): newdimv = ncf.dimensions[varinf.dimns[idim]] if newdimv.isunlimited(): ncfnew.createDimension(varinf.dimns[idim], None) else: ncfnew.createDimension(varinf.dimns[idim], varinf.dims[idim]) # FillVallue if varinf.dtype == type(np.int(1)): fillValue = gen.fillValueI elif varinf.dtype == type('c'): fillValue = gen.fillValueC elif varinf.dtype == type(np.int16(1)) or varinf.dtype == type(np.int32(1)): fillValue = gen.fillValueI16 elif varinf.dtype == type(np.float(1.)): fillValue = gen.fillValueF elif varinf.dtype == type(np.float32(1.)) or varinf.dtype == type(np.float64(1.)): fillValue = gen.fillValueF64 else: print errormsg print ' ' + fname + ': type of variable:', varinf.dtype, 'not ready!!' quit(-1) if maskvalue != 'mask': maskval = gen.retype(maskvalue,varinf.dtype) else: print infmsg print ' ' + fname + ': maskgin with already masked variable' varmaskv = maskvalues.mask newvar = ncfnew.createVariable(varn, varinf.dtype, varinf.dimns, \ fill_value=fillValue) for attrn in varinf.attributes: if not attrn == '_FillValue': attr = newvar.setncattr(attrn, varobj.getncattr(attrn)) attr = newvar.setncattr('mask', 'variable masked using ' + values.split(':')[2]+ \ ' of variable ' + maskvarn + ' from file ' + maskfilename) newvar[:] = varobj[:] ncfnew.sync() ncfnew.close() for varns in ncf.variables: if not varns == varn: try: with gen.Capturing() as output: fvaradd(filename + ',' + varns, ofile) except: print errmsg print 'fvaradd(' + filename + ', ' + varns + ',' + ofilen + ')' for s1out in output: print s1out quit(-1) ncfnew = NetCDFFile(ofile,'a') varobj = ncfnew.variables[varn] # Masking following that dimensions which are not coincident with the mask # Check if there are some coincident dimensional names between both variables vdimns = list(varinf.dimns) mdimns = maskdims for dimn in mdimns: if gen.searchInlist(vdimns,dimn): vdimns.remove(dimn) #slices = gen.provide_slices(varinf.dimns, varinf.dims, list(varmaskinf.dimns)) slices = gen.provide_slices(varinf.dimns, varinf.dims, vdimns) if maskvalue != 'mask': maskTOuse = ma.masked_equal(maskvalues, maskval) maskTOusev = maskTOuse.mask else: maskTOusev = maskvalues.mask for ir in range(runningsize): islice = slices[ir] varvals = varobj[tuple(islice)] maskvarvals = np.where(maskTOusev, fillValue, varvals) newvar[tuple(islice)] = maskvarvals ncfnew.sync() ncfnew.close() ncf.close() print 'masked file "' + ofile + '" generated !!!' return def chgtimestep(values, origfile, varN): """ Function to change the values of a given time-step of a variable inside a netCDF for values from a nother file [values]=[origtime_step]:[newfile]:[newtime_step] [origtime_step]: desired original time step to change: 'first', 'last', 'middle', '[num]' [newfile]: new netCDF file name [newtime_step]: desired new time step to change: 'first', 'last', 'middle', '[num]' 'zeros', 'fillVal' 'first': first time-step 'last': last time-step 'middle': middle time-step '[num]': [num] time-step 'zeros': fill time-step with zeros 'fillVal': fill time-step with fill_Values [FillValue] [varN]= variable name [origfile]= original netCDF file name """ fname = 'chgtimestep' ermsg = fname + ': ERROR -- error -- ERROR -- error' vals = values.split(':') origtime_step = vals[0] newfile = vals[1] newtime_step = vals[2] if len(vals) == 4: FillValue = np.float(vals[3]) filexist(origfile, ermsg, 'original') filexist(newfile, ermsg, 'new') ncofile = NetCDFFile(origfile,'a') varinfile(ncofile, origfile, ermsg, 'change', varN) ncnfile = NetCDFFile(newfile,'r') varinfile(ncofile, newfile, ermsg, 'new', varN) ncnfile.close() if newtime_step == 'zeros': newvals = uploading_timestep(newfile, varN, 'first') newvals = 0. elif newtime_step == 'fillVal': newvals = uploading_timestep(newfile, varN, 'first') newvals = FillValue else: newvals = uploading_timestep(newfile, varN, newtime_step) varobj = ncofile.variables[varN] Ndims = len(varobj.shape) dimt = varobj.shape[0] if origtime_step == 'first': oval = 0 elif origtime_step == 'middle': oval = int(dimt/2) elif origtime_step == 'last': oval = dimt-1 else: oval = int(origtime_step) if Ndims == 1: varobj[oval] = newvals elif Ndims == 2: varobj[oval,:] = newvals elif Ndims == 3: varobj[oval,:,:] = newvals elif Ndims == 4: varobj[oval,:,:,:] = newvals elif Ndims == 5: varobj[oval,:,:,:,:] = newvals elif Ndims == 6: varobj[oval,:,:,:,:,:] = newvals else: print ermsg print ' variable size', Ndims, 'not ready !' print ermsg quit(-1) ncofile.sync() ncofile.close() return def uploading_timestep(ncfile, varN, time_step): """ Function to upload a given time-step of a variable inside a netCDF [ncfile]= netCDF file name [varN]= variable name [time_step]= desired time step: 'first', 'last', 'middle', '[num]' 'first': first time-step 'last': last time-step 'middle': middle time-step '[num]': [num] time-step """ fname = 'uploading_timestep' ermsg = fname + ': ERROR -- error -- ERROR -- error' filexist(ncfile, ermsg, 'uploading') ncufile = NetCDFFile(ncfile,'r') varinfile(ncufile, ncfile, ermsg, 'uploading', varN) varobj = ncufile.variables[varN] Ndims = len(varobj.shape) dimt = varobj.shape[0] if time_step == 'first': upval = 0 elif time_step == 'middle': upval = int(dimt/2) elif time_step == 'last': upval = dimt-1 else: upval = int(time_step) if Ndims == 1: uploadvar = varobj[upval] elif Ndims == 2: uploadvar = varobj[upval,:] elif Ndims == 3: uploadvar = varobj[upval,:,:] elif Ndims == 4: uploadvar = varobj[upval,:,:,:] elif Ndims == 5: uploadvar = varobj[upval,:,:,:,:] elif Ndims == 6: uploadvar = varobj[upval,:,:,:,:,:] else: print ermsg print ' variable size', Ndims, 'not ready !' print ermsg quit(-1) return uploadvar def timeshiftvar(values, fileN, varN, FillVal=1.e20): """ Function to temporaly shift a number of time-steps a given variable inside a netCDF file [values]=[nsteps]:[[FillValue]] [nsteps]: desired number of time step to move [FillValue]: value to fill the empty time-steps [fileobj]= original netCDF object [varN]= variable name """ from sys import stdout fname = 'timesfhitvar' ermsg = fname + ': ERROR -- error -- ERROR -- error' nsteps = int(values.split(':')[0]) if len(values.split(':')) == 2: FillVal = np.float(values.split(':')[1]) fileobj = NetCDFFile(fileN,'a') varobj = fileobj.variables[varN] vardims = varobj.shape vartype = varobj.dtype Ndims = len(vardims) dimt = vardims[0] if nsteps > 0: inimov = nsteps endmov = dimt iniplace = 0 endplace = dimt - nsteps else: inimov = 0 endmov = dimt + nsteps iniplace = -nsteps endplace = dimt print ' ' + fname + ': moving', nsteps, '___ __ _' print ' from (', inimov, ',', endmov, ') to (',iniplace,',',endplace,')' if Ndims == 1: vals = np.ones(vardims, dtype=vartype)*FillVal vals[iniplace:endplace] = varobj[inimov:endmov] varobj[:] = vals elif Ndims == 2: vals = np.ones(vardims, dtype=vartype)*FillVal vals[iniplace:endplace,:] = varobj[inimov:endmov,:] varobj[:] = vals elif Ndims == 3: vals = np.ones((vardims[0], vardims[1]), dtype=vartype)*FillVal for i in range(vardims[2]): if (i%10 == 0): print '\r shifted ' + '{0:5.2g}'.format(i*100./vardims[2]) + ' %', stdout.flush() vals[iniplace:endplace,:] = varobj[inimov:endmov,:,i] varobj[:,:,i] = vals elif Ndims == 4: vals = np.ones((vardims[0], vardims[1]), dtype=vartype)*FillVal for i in range(vardims[2]): for j in range(vardims[3]): nvals=i*vardims[3] + j if (nvals%100 == 0): print '\r shifted ' + '{0:5.2g}'.format(nvals*100./(vardims[2]*vardims[3])) + ' %', stdout.flush() vals[iniplace:endplace,:] = varobj[inimov:endmov,:,i,j] varobj[:,:,i,j] = vals elif Ndims == 5: vals = np.ones((vardims[0], vardims[1]), dtype=vartype)*FillVal for i in range(vardims[2]): for j in range(vardims[3]): for k in range(vardims[4]): nvals=i*vardims[3]*vardims[4] + j*vardims[4] + k if (nvals%1000 == 0): print '\r shifted ' + '{0:5.2g}'.format(nvals*100./(vardims[2]*vardims[3]*vardims[4])) + ' %', stdout.flush() vals[iniplace:endplace,:] = varobj[inimov:endmov,:,i,j,k] varobj[:,:,i,j,k] = vals elif Ndims == 6: vals = np.ones((vardims[0], vardims[1]), dtype=vartype)*FillVal for i in range(vardims[2]): for j in range(vardims[3]): for k in range(vardims[4]): for l in range(vardims[5]): nvals=i*vardims[3]*vardims[4]*vardims[5] + j*vardims[4]*vardims[5] + k*vardims[5] + l if (nvals%10000 == 0): print '\r shifted ' + \ '{0:5.2g}'.format(nvals*100./(vardims[2]*vardims[3]*vardims[4]*vardims[5])) + ' %', stdout.flush() vals[iniplace:endplace,:] = varobj[inimov:endmov,:,i,j,k,l] varobj[:,:,i,j,k,l] = vals else: print ermsg print ' variable size', Ndims, 'not ready !' print ermsg quit(-1) print stdout.write("\n") fileobj.sync() fileobj.close() return def addvals(values, filename, varN): """ Function to add values to a given variable at a given dimension values = [dimension]:[position]:[Nvals]:[addval] [dimension]: dimension at which the variable wants to be increased [position]: position within the dimension at which the variable wants to be increased 'first': first position of the dimension 'middle': middle position of the dimension 'last': last position of the dimension 'num': before given value [Nvals]: number of values [addval]: value to give to the added values 'val': a given value 'missing': as a missing value filename = name of the netCDF file varN = name of the variable """ import subprocess as sub fname = 'addvals' if values == 'h': print fname + '_____________________________________________________________' print addvals.__doc__ quit() newf = 'addvals.nc' dimension=int(values.split(':')[0]) position=values.split(':')[1] Nvals=int(values.split(':')[2]) addval=values.split(':')[3] gen.filexist(filename, errormsg, 'add values') ncfa = NetCDFFile(filename,'r') varinfile(ncfa, filename, errormsg, 'adding', varN) varobj=ncfa.variables[varN] varinf=variable_inf(varobj) newdims=list(varinf.dims) newdims[dimension] = newdims[dimension] + Nvals newdims=tuple(newdims) dmodname = varinf.dimns[dimension] print ' ' + fname + ' increassed dimension "' + dmodname + '"' if position == 'first': pos=0 elif position == 'middle': pos=(varinf.dims[dimension])/2 elif position == 'last': pos=varinf.dims[dimension] else: pos=int(position) if pos > varinf.dims[dimension]: print errormsg print fname + ': position given ', pos,' is too big for the size of the dimension ',varinf.dims[dimension], '!!!!' print errormsg quit(-1) ## print 'starting position: ', pos if addval == 'missing': addvaln = 1.e20 else: addvaln = addval addvalv = gen.typemod(addvaln, str(varinf.dtype)) newnc = NetCDFFile(newf,'w') # Dimensions ## for dim in ncfa.dimensions.values(): dname = dim._name dsize = dim.__len__() if dim.isunlimited(): dsize=None else: if dname == varinf.dimns[dimension]: dsize=newdims[dimension] newnc.createDimension(dname, size=dsize) # Variables ## for var in ncfa.variables.values(): vname = var._name vkind = var.dtype vdims = var.dimensions varattrs = var.ncattrs() if gen.searchInlist(varattrs, '_FillValue'): fillval = var.getncattr('_FillValue') newvarobj = newnc.createVariable(vname, vkind, vdims, fill_value=fillval) elif vname == varN and addval == 'missing': newvarobj = newnc.createVariable(vname, vkind, vdims, \ fill_value=gen.typemod(addvaln, str(varinf.dtype))) else: newvarobj = newnc.createVariable(vname, vkind, vdims) varobj=ncfa.variables[vname] vals = varobj[:] if vname == varN: for ival in range(Nvals): newvar = np.insert(vals, pos, addvalv, axis=dimension) vals=newvar.copy() newvarobj[:] = newvar else: if vname == dmodname: # Increassing dimension variable values ## dvals=ncfa.variables[dmodname][:] if position == 'first': valB = -(dvals[1] - dvals[0]) valA=dvals[pos] + valB elif position == 'last': valB=-(dvals[1] - dvals[0]) valA=dvals[pos-1] - valB*Nvals else: valB=-(dvals[pos] - dvals[pos-1])/(Nvals+1) valA=dvals[pos-1] - valB*Nvals for ival in range(Nvals): addvalvar=valA + valB*ival ## print ' ' + fname + ' adding to dimension variable "', dmodname ,'" value ',addvalvar newvar = np.insert(vals, pos, addvalvar, axis=0) vals=newvar.copy() newvarobj[:] = newvar else: vardims=list(var.dimensions) if gen.searchInlist(vardims,varinf.dimns[dimension]): ## print ' ' + fname + ' variable ' + vname + ' has also the increased dimension!' vardim=vardims.index(varinf.dimns[dimension]) for ival in range(Nvals): addvalvar = addval newvar = np.insert(vals, pos, addvalvar, axis=vardim) vals=newvar.copy() newvarobj[:] = newvar else: newvarobj[:] = vals for vattr in varobj.ncattrs(): if not vattr == '_FillValue': vattrval = varobj.getncattr(vattr) newattr = set_attribute(newvarobj, vattr, vattrval) # Global attributes ## for gattr in ncfa.ncattrs(): gattrval = ncfa.getncattr(gattr) newattr = set_attribute(newnc, gattr, gattrval) newnc.sync() newnc.close() ncfa.close() try: ins = 'cp ' + newf + ' ' + filename syscode = sub.call(ins, shell=True) if not syscode == 0: print errmsg print " Copy to the new file failed!" print ins print syscode quit() else: sub.call('rm ' + newf, shell=True) except OSError, e: print errmsg print " Copy failed!" print e quit(-1) return def TimeInf(filename, tname): """ Function to print all the information from the variable time filename= netCDF file name tname= name of the variable time """ ncfobj = NetCDFFile(filename,'r') timeinf = cls_time_information(ncfobj, tname) fmtprinting_class(timeinf) ncfobj.close() return def sorttimesmat(filename, tname): """ Function to sort the time values of a given file tname, name of the variable time filename= netCDF file name """ fname = 'sorttimesmat' filexist(filename, errormsg, 'to sort time') ncft = NetCDFFile(filename,'a') varinfile(ncft, filename, errormsg, 'time', tname) timeobj = ncft.variables[tname] timeinf = cls_time_information(ncft, tname) tvals = timeobj[:] Ltvals = list(tvals) sortedtvals = sorted(Ltvals) tinds = np.zeros((timeinf.dimt), dtype=int) for it in range(timeinf.dimt): tinds[it] = Ltvals.index(sortedtvals[it]) for var in ncft.variables: varobj = ncft.variables[var] if searchInlist(varobj.dimensions, tname): print fname + ': time-sorting variable: "' + var + '" ' varvals = varobj[:] newvarvals = varvals.copy() for it in range(timeinf.dimt): if var == 'time': print it, tinds[it], tvals[tinds[it]] newvarvals[it,] = varvals[tinds[it],] varobj[:] = newvarvals ncft.sync() ncft.close() return def get_namelist_vars(values, namelist): """ Function to get namelist-like values ([varname] = [value]) get_namelist_vars(namelist) values= [sectionname],[kout] [sectionname]: name of the section from which one want the values ('all', for all) [kout]: kind of output 'tex3': printed output as LaTeX table of three columns \verb+namelist_name+ & value 'column': printed output as namelist_name value 'dict': as two python dictionary object (namelistname, value and namelistname, sectionname) namelist= namelist_like file to retrieve values >>> get_namelist_vars('geogrid,dic','/home/lluis/etudes/domains/medic950116/namelist.wps') {'e_sn': '32, 97,', 'stand_lon': '0.', 'opt_geogrid_tbl_path': "'./'", 'geog_data_path': "'/home/lluis/DATA/WRF/geog'", 'pole_lat': '90.0', 'ref_lat': '35.0,', 'map_proj': "'lat-lon',", 'parent_id': '1, 1,', 'geog_data_res': "'2m','2m',", 'e_we': '32, 112,', 'dx': '0.35,', 'dy': '0.35,', 'parent_grid_ratio': '1, 3,', 'pole_lon': '0.0', 'ref_lon': '20,', 'j_parent_start': '1, 17,', 'i_parent_start': '1, 31,'} """ fname = 'get_namelist_vars' # List of characters which split pairs name/value valuessep = ['='] # List of characters which indicate a comment commentchars = ['#'] if values == 'h': print fname + '_____________________________________________________________' print get_namelist_vars.__doc__ quit() expectargs = '[sectionname],[kout]' gen.check_arguments(fname,values,expectargs,',') secname = values.split(',')[0] kout = values.split(',')[1] if not os.path.isfile(namelist): print errormsg print ' ' + fname + ": namelist file '" + namelist + "' does not exist !!" quit(-1) ncml = open(namelist, 'r') sections = {} namelistvals = {} namelistsecs = {} sectionnames = [] namessec = [] allnames = [] namelistvalssec = {} namelistsecssec = {} nmlname = '' sectionname = '' for line in ncml: linevals = gen.reduce_spaces(line) Nvals = len(linevals) if Nvals >= 1 and linevals[0][0:1] == '&': if len(sectionnames) > 1: sections[sectionname] = namessec sectionname = linevals[0][1:len(linevals[0])+1] # print ' ' + fname + ": new section '" + sectionname + "' !!!" sectionnames.append(sectionname) namessec = [] nmlname = '' elif Nvals >= 1 and not gen.searchInlist(commentchars,linevals[0][0:1]): if Nvals >= 3 and gen.searchInlist(valuessep,linevals[1]): nmlname = linevals[0] nmlval = gen.numVector_String(linevals[2:Nvals],' ') elif Nvals == 1: for valsep in valuessep: if linevals[0].find(valsep) != -1: nmlname = linevals[0].split(valsep)[0] nmlval = linevals[0].split(valsep)[1] break elif Nvals == 2: for valsep in valuessep: if linevals[0].find(valsep) != -1: nmlname = linevals[0].split(valsep)[0] nmlval = linevals[1] break elif linevals[1].find(valsep) != -1: nmlname = linevals[0] nmlval = linevals[1].split(valsep)[0] break else: print warnmsg print ' ' + fname + ': wrong number of values', Nvals, \ 'in the namelist line!' print ' line: ',line print ' line values:',linevals # quit(-1) namelistvals[nmlname] = nmlval namelistsecs[nmlname] = sectionname namessec.append(nmlname) allnames.append(nmlname) if len(sectionname) > 1: sections[sectionname] = namessec if secname != 'all': if not gen.searchInlist(sections.keys(),secname): print errormsg print ' ' + fname + ": section '" + values + "' does not exist !!" print ' only found:',sectionnames quit(-1) namestouse = [] for nml in allnames: for nnml in sections[secname]: namelistvalssec[nnml] = namelistvals[nnml] namelistsecssec[nnml] = secname if nml == nnml: namestouse.append(nml) else: namestouse = allnames namelistvalssec = namelistvals namelistsecssec = namelistsecs if kout == 'tex3': ofile='get_namelist_vars_3col.tex' fobj = open(ofile, 'w') vals = namestouse Nvals = len(vals) Nvals3 = int(Nvals/3) latextab = '\\begin{center}\n\\begin{tabular}{lclclc}\n' for il in range(2): latextab = latextab + '{\\bfseries{name}} & {\\bfseries{value}} & ' latextab= latextab+ '{\\bfseries{name}} & {\\bfseries{value}} \\\\ \\hline\n' if np.mod(Nvals,3) != 0: Nvals0 = Nvals - np.mod(Nvals,3) else: Nvals0 = Nvals for ival in range(0,Nvals0,3): line = '' print ' ival:',ival for il in range(2): line = line + '\\verb+' + vals[ival+il] + '+ & ' + \ namelistvalssec[vals[ival+il]].replace('_','\\_') +' & ' line = line + '\\verb+' + vals[ival+2] + '+ & ' + \ namelistvalssec[vals[ival+2]].replace('_','\\_') + ' \\\\\n' latextab = latextab + line latextab = latextab + '%not multiple of three!!!!\n' print 'mod:', np.mod(Nvals,3),Nvals0,Nvals if np.mod(Nvals,3) != 0: ival = Nvals0 line = '' for il in range(np.mod(Nvals,3)): print 'ival:',ival + il line = line + '\\verb+' + vals[ival+il] + '+ & ' + \ namelistvalssec[vals[ival+il]].replace('_','\\_') + ' & ' for il in range(2-np.mod(Nvals,3)): line = line + ' & & ' latextab = latextab + line + ' & \\\\\n' latextab = latextab + '\\end{tabular}\n\\end{center}\n' # print latextab fobj.write(latextab) fobj.close() print fname + ": successful writen '" + ofile + "' LaTeX tale file !!" return elif kout == 'column': for dictv in namestouse: print dictv + ' = ' + namelistvalssec[dictv] return elif kout == 'dict': return namelistvalssec, namelistsecssec else: print errormsg print ' ' + fname + ": kind of output '" + kout + "' not ready!!!" quit(-1) return def check_times_file(values, filename, timen): """ Function to check time-steps of a given file values=[FirstDate]:[LastDate][freq]:[auxmonth] [FirstDate] = first date within file (in [YYYY][MM][DD][HH][MI][SS] format) [LastDate] = last date within file (in [YYYY][MM][DD][HH][MI][SS] format) [freq] = frequency of the data within the file (in time units) or 'month' (monthly values) [auxmonth] = in case of 'month' midmon: dates are given according to the mid time value of the month real: dates are given increassing a month from the previous date filename: name of the file timen: name of the variable time """ import datetime as dt fname = 'check_times_file' FirstDate = values.split(':')[0] LastDate = values.split(':')[1] freq = values.split(':')[2] ncft = NetCDFFile(filename, 'r') varinfile(ncft, filename, errormsg, '', timen) timeobj=ncft.variables[timen] timeinf = cls_time_information(ncft, timen) timevals = timeobj[:] ncft.close() yrf=int(FirstDate[0:4]) monf=int(FirstDate[4:6]) dayf=int(FirstDate[6:8]) horf=int(FirstDate[8:10]) minf=int(FirstDate[10:12]) secf=int(FirstDate[12:14]) firstdate = dt.datetime(yrf, monf, dayf, horf, minf, secf) yrl=int(LastDate[0:4]) monl=int(LastDate[4:6]) dayl=int(LastDate[6:8]) horl=int(LastDate[8:10]) minl=int(LastDate[10:12]) secl=int(LastDate[12:14]) lastdate = dt.datetime(yrl, monl, dayl, horl, minl, secl) firstT = firstdate - timeinf.refdate t0 = diffdate_units(firstT, timeinf.units) period = lastdate - firstdate per = diffdate_units(period, timeinf.units) if freq != 'month': theordimt = np.int(per / np.float64(freq) + 1) else: theordimt = np.int((yrl - yrf + 1) * 12) theordates = np.zeros((theordimt), dtype=np.float64) theordates[0] = t0 if freq != 'month': theordates = t0 + np.arange(1,theordimt) * np.float64(freq) else: auxmonth = values.split(':')[3] if auxmonth == 'midmon': t01 = np.array([yrf, monf, 1, 0, 0, 0]) else: t01 = np.array([yrf, monf, dayf, horf, minf, secf]) for it in range(1,theordimt): t02 = t01.copy() t02[1] = t02[1] + 1 if t02[1] > 12: t02[1] = 1 t02[0] = t02[0] + 1 date01 = dt.datetime(t01[0], t01[1], t01[2], t01[3], t01[4], t01[5]) if auxmonth == 'midmon': date02 = dt.datetime(t02[0], t02[1], t02[2], t02[3], t02[4], t02[5]) diff012 = date02 - date01 t01 = diffdate_units(date01 - timeinf.refdate, timeinf.units) theordates[it] = t01 + diffdate_units(diff012, timeinf.units) / 2. elif auxmonth == 'real': date02 = dt.datetime(t02[0], t02[1], t01[2], t01[3], t01[4], t01[5]) diff012 = date02 - date01 theordates[it] = theordates[it-1] + diffdate_units(diff012, timeinf.units) else: print errormsg print fname + ': frequency "month" required third argument "' + auxmonth + '" not ready!!!!' print errormsg quit(-1) t01 = t02.copy() equaldates = sorted(list(set(theordates) & set(timevals))) missdates = sorted(list(set(theordates) - set(timevals))) Nequal = len(equaldates) Ndiff = len(missdates) if theordimt != timeinf.dimt: print warnmsg print fname + ': there are not the correspondant times (', theordimt ,') in file "' + filename + '" with ', timeinf.dimt,' time-steps !!!!!!!' print ' ' + fname + ': Number equal times-steps: ', Nequal, 'Number different time-steps: ', Ndiff if Nequal >= Ndiff or Nequal == 0: matdates = netCDFdatetime_realdatetime(timeinf.unitsval, timeinf.calendar, missdates) print ' file does not have dates: ' for it in range(Ndiff): print matdates[it,:] else: matdates = netCDFdatetime_realdatetime(timeinf.unitsval, timeinf.calendar, equaldates) print ' file only has: ' for it in range(Nequal): print matdates[it,:] return def writing_str_nc(varo, values, Lchar): """ Function to write string values in a netCDF variable as a chain of 1char values varo= netCDF variable object values = list of values to introduce Lchar = length of the string in the netCDF file """ fname = 'writing_str_nc' Nvals = len(values) for iv in range(Nvals): if values[iv] is None: stringv = 'None' else: stringv = values[iv] charvals = np.zeros((Lchar), dtype='string_') Lstr = len(stringv) charvals[Lstr:Lchar] = '' for ich in range(Lstr): chrv = stringv[ich:ich+1] charvals[ich] = stringv[ich:ich+1] varo[iv,:] = charvals return def writing_1str_nc(varo, values, Lchar): """ Function to write 1 string value in a netCDF variable as a chain of 1char values varo= netCDF variable object values = list of values to introduce Lchar = length of the string in the netCDF file """ stringv=values charvals = np.chararray(Lchar) Lstr = len(stringv) charvals[Lstr:Lchar] = '' for ich in range(Lstr): charvals[ich] = stringv[ich:ich+1] varo[:] = charvals return def get_1str_nc(varo, Lchar): """ Function to get 1 string value in a netCDF variable as a chain of 1char values varo= netCDF variable object Lchar = length of the string in the netCDF file """ string = '' for ich in range(Lchar): charval = str(stringv[ich:ich+1]).replace('[','').replace(']','') if charval == '': break else: if ich == 0: string = charval else: string = string + charval return string def get_str_nc(varo, Lchar): """ Function to get string values in a netCDF variable as a chains of 1char values varo= netCDF variable object Lchar = length of the string in the netCDF file """ fname = 'get_str_nc' Nvalues = varo.shape[0] strings = [] for ival in range(Nvalues): stringv = varo[ival,:] string = str('') for ich in range(Lchar): charval = str(stringv[ich:ich+1]).replace('[','').replace(']','').replace("'","") if charval == '--' or len(charval) == 0: break else: if ich == 0: string = charval else: string = string + charval strings.append(string.strip()) return strings def get_str_nc_word(varo, Lchar): """ Function to get string values in a netCDF variable as a chains of 1char values varo= netCDF variable object Lchar = length of the string in the netCDF file """ Nvalues = varo.shape[0] strings = [] for ival in range(Nvalues): stringv = varo[ival] string = str('') for ich in range(Lchar): charval = str(stringv[ich:ich+1]).replace('[','').replace(']','') if charval == '--': break else: if ich == 0: string = charval else: string = string + charval strings.append(string) return strings def remapnn(values, filename, varname): """ Function to remap to the nearest neightbor a variable using projection from another file values=[newprojectionfile]|[newlonname,newlatname]|[newlondim,newlatdim]|[oldlonname,oldlatname]| [oldlondim,oldlatdim] [newprojectionfile]: name of the file with the new projection [newlonname,newlatname]: name of the longitude and the latitude variables in the new file [newlondim,newlatdim]: name of the dimensions for the longitude and the latitude in the new file [oldlonname,oldlatname]: name of the longitude and the latitude variables in the old file [oldlondim,oldlatdim]: name of the dimensions for the longitude and the latitude in the old file filename= netCDF file name varn= variable name ('all', for all variables) """ fname = 'remapnn' if values == 'h': print fname + '_____________________________________________________________' print remapnn.__doc__ quit() expectargs = '[newprojectionfile]|[newlonname,newlatname]|[newlondim,' + \ 'newlatdim]|[oldlonname,oldlatname]|[oldlondim,oldlatdim]' gen.check_arguments(fname,values,expectargs,'|') newprojectionfile = values.split('|')[0] newlonlatname = values.split('|')[1] newlonlatdim = values.split('|')[2].split(',') oldlonlatname = values.split('|')[3] oldlonlatdim = values.split('|')[4].split(',') if varname != 'all': ofile = 'remapnn_' + varname + '.nc' else: ofile = 'remapnn_allvars.nc' gen.filexist(filename, errormsg, 'old file') gen.filexist(newprojectionfile, errormsg, 'new file') oldlon=oldlonlatname.split(',')[0] oldlat=oldlonlatname.split(',')[1] oldncfile = NetCDFFile(filename,'r') varinfile(oldncfile, filename, errormsg, 'old', oldlon) varinfile(oldncfile, filename, errormsg, 'old', oldlat) newlon=newlonlatname.split(',')[0] newlat=newlonlatname.split(',')[1] newncfile = NetCDFFile(newprojectionfile,'r') varinfile(newncfile, newprojectionfile, errormsg, 'new', newlon) varinfile(newncfile, newprojectionfile, errormsg, 'new', newlat) oldlon2d, oldlat2d = gen.lonlat2D(oldncfile.variables[oldlon], \ oldncfile.variables[oldlat]) newlon2d, newlat2d = gen.lonlat2D(newncfile.variables[newlon], \ newncfile.variables[newlat]) newdimx = newlon2d.shape[1] newdimy = newlon2d.shape[0] olddimx = oldlon2d.shape[1] olddimy = oldlon2d.shape[0] # We are going to do the search in two phases # 1.- Searching on grid points every 10% of the size of the origina domain # 2.- Within the correspondant box of 20%x20% percent # 1.- first pairs old-->new ## dx10 = int(olddimx/10) dy10 = int(olddimy/10) oldlon2d10 = oldlon2d[slice(dy10,olddimy,dy10), slice(dx10,olddimx,dx10)] oldlat2d10 = oldlat2d[slice(dy10,olddimy,dy10), slice(dx10,olddimx,dx10)] firstmappairs = np.ones((2, newdimy, newdimx), dtype=int) print ' ' + fname + ' first searching nearest neighbor by',dx10,',',dy10,'...' for jnew in range(newdimy): for inew in range(newdimx): dist=np.sqrt((newlon2d[jnew, inew] - oldlon2d10)**2. + \ (newlat2d[jnew, inew] - oldlat2d10)**2.) mindist = np.min(dist) minydist10, minxdist10 = gen.index_mat(dist,mindist) firstmappairs[:,jnew,inew] = [int(dy10*(1+minydist10)), \ int(dx10*(1+minxdist10))] # 2.- Looking around the closest coarse point ## mappairs = np.ones((2, newdimy, newdimx), dtype=int) dx2 = dx10/2 dy2 = dy10/2 print ' ' + fname + ' searching nearest neighbor...' for jnew in range(newdimy): for inew in range(newdimx): i10 = firstmappairs[1,jnew,inew] j10 = firstmappairs[0,jnew,inew] imin = np.max([i10-dx2, 0]) imax = np.min([i10+dx2, olddimx+1]) jmin = np.max([j10-dy2, 0]) jmax = np.min([j10+dy2, olddimy+1]) oldlon2d2 = oldlon2d[slice(jmin,jmax), slice(imin,imax)] oldlat2d2 = oldlat2d[slice(jmin,jmax), slice(imin,imax)] dist=np.sqrt((newlon2d[jnew, inew] - oldlon2d2)**2. + \ (newlat2d[jnew, inew] - oldlat2d2)**2.) # print 'oldlon2d2________________',i10,j10 # print oldlon2d2 # print 'oldlat2d2________________' # print oldlat2d2 # print 'dist________________' # print dist mindist = np.min(dist) minydist, minxdist = index_mat(dist,mindist) mappairs[:,jnew,inew] = [int(j10+minydist), int(i10+minxdist)] # print jnew,inew,'|',firstmappairs[:,jnew,inew],':',minxdist,minydist, \ # 'final:',mappairs[:,jnew,inew],'lon,lat old:',oldlon2d[minydist,minxdist],\ # oldlat2d[minydist,minxdist], 'new:', newlon2d[jnew, inew], \ # newlat2d[jnew, inew],'mindist:',mindist # quit() # Creation of the new file ## ncfnew = NetCDFFile(ofile,'w') for dn in oldncfile.dimensions: if dn == oldlon: ncfnew.createDimension(newlonlatdim[0], newdimx) elif dn == oldlat: ncfnew.createDimension(newlonlatdim[1], newdimy) else: if oldncfile.dimensions[dn].isunlimited(): ncfnew.createDimension(dn, None) else: ncfnew.createDimension(dn, len(oldncfile.dimensions[dn])) ncfnew.sync() # looping along all the variables ## if varname == 'all': oldvarnames = oldncfile.variables else: oldvarnames = [varname] for varn in oldvarnames: print ' adding:',varn,'...' ovar = oldncfile.variables[varn] vardims = ovar.dimensions vartype = ovar.dtype varattrs = ovar.ncattrs() # Looking if variable has lon and/or lat: varlonlat = False if searchInlist(vardims, oldlonlatdim[0]): varlonlat = True if searchInlist(vardims, oldlonlatdim[1]): varlonlat = True if varn != oldlon and varn != oldlat and varlonlat: newvarshape = [] newvardimn = [] newslice = [] oldslice = [] for dn in vardims: if dn == oldlat: newvarshape.append(newdimy) newvardimn.append(newlonlatdim[1]) elif dn == oldlon: newvarshape.append(newdimx) newvardimn.append(newlonlatdim[0]) else: newvarshape.append(len(oldncfile.dimensions[dn])) newvardimn.append(dn) newslice.append(slice(0,len(oldncfile.dimensions[dn]))) oldslice.append(slice(0,len(oldncfile.dimensions[dn]))) newvarval = np.zeros(tuple(newvarshape), dtype=vartype) for jnew in range(newdimy): for inew in range(newdimx): oldslice0 = oldslice[:] newslice0 = newslice[:] if searchInlist(vardims,oldlat): oldslice0.append(mappairs[0,jnew,inew]) newslice0.append(jnew) if searchInlist(vardims,oldlon): oldslice0.append(mappairs[1,jnew,inew]) newslice0.append(inew) # if ((jnew*newdimx + inew)%100 == 0): print ' ' + fname + ': ', \ # '{0:5.2f}'.format(float((jnew*newdimx + inew)*100./(newdimx*newdimy))),\ # '% done' # print 'slices new:',newslice0,'old:',oldslice0 newvarval[tuple(newslice0)] = ovar[tuple(oldslice0)] elif varn == oldlon: ovar = newncfile.variables[newlon] vardims = ovar.dimensions vartype = ovar.dtype varattrs = ovar.ncattrs() newvardimn = (newlonlatdim[1], newlonlatdim[0]) newvarval = newlon2d elif varn == oldlat: ovar = newncfile.variables[newlat] vardims = ovar.dimensions vartype = ovar.dtype varattrs = ovar.ncattrs() newvardimn = (newlonlatdim[1], newlonlatdim[0]) newvarval = newlat2d else: newvardimn = vardims newvarval = ovar[:] if searchInlist(varattrs, '_FillValue'): fvalv = ovar.getncattr('_FillValue') newvar = ncfnew.createVariable(varn, vartype, tuple(newvardimn), \ fill_value=fvalv) else: newvar = ncfnew.createVariable(varn, vartype, tuple(newvardimn)) for ncattr in varattrs: if ncattr != '_FillValue': attrv = ovar.getncattr(ncattr) newattr = set_attribute(newvar,ncattr,attrv) attr = newvar.setncattr('mask', 'variable remapped at nearest neighbors ' + \ 'using ' + values.split(':')[0] + ' using as longitude and latitude values ') newvar[:] = newvarval ncfnew.sync() ncfnew.close() newncfile.close() oldncfile.close() # Adding attributes fgaddattr(filename, ofile) print 'remapnn: new file "' + ofile + '" with remapped variable written !!' return #remapnn('met_em.d01.1979-01-01_00:00:00.nc|XLONG_M,XLAT_M|east_weast,south_north|longitude,latitude|longitude,latitude', \ # 'Albedo.nc', 'all') #quit() def remapnn_old(values, filename, varn, ddegL, ddegl): """ Function to remap to the nearest neightbor a variable using projection from another file values=[newprojectionfile]:[newlonname,newlatname]:[oldlonname,oldlatname] [newprojectionfile]: name of the file with the new projection [newlonname,newlatname]: name of the longitude and the latitude variables in the new file [oldlonname,oldlatname]: name of the longitude and the latitude variables in the old file filename= netCDF file name varn= variable name ddegL= Longitude range around the old grid point to speed up the search ddegl= latitude range around the old grid point to speed up the search """ fname = 'remapnn' ofile = 'remapnn_' + varn + '.nc' fillValue = 1.e20 newprojectionfile = values.split(':')[0] newlonlatname = values.split(':')[1] oldlonlatname = values.split(':')[2] filexist(filename, errormsg, 'old file') filexist(newprojectionfile, errormsg, 'new file') oldlon=oldlonlatname.split(',')[0] oldlat=oldlonlatname.split(',')[1] oldncfile = NetCDFFile(filename,'r') varinfile(oldncfile, filename, errormsg, 'old', oldlon) varinfile(oldncfile, filename, errormsg, 'old', oldlat) newlon=newlonlatname.split(',')[0] newlat=newlonlatname.split(',')[1] newncfile = NetCDFFile(newprojectionfile,'r') varinfile(newncfile, newprojectionfile, errormsg, 'old', newlon) varinfile(newncfile, newprojectionfile, errormsg, 'old', newlat) oldlon2d, oldlat2d = lonlat2D(oldncfile, oldlon, oldlat) newlon2d, newlat2d = lonlat2D(newncfile, newlon, newlat) newdimx = newlon2d.shape[1] newdimy = newlon2d.shape[0] olddimx = oldlon2d.shape[1] olddimy = oldlon2d.shape[0] # pairs old-->new ## mappairs = np.ones((2, newdimy, newdimx), dtype=int) mappairs = mappairs*fillValue print ' ' + fname + ' searching nearest neighbor...' for jnew in range(newdimy): for inew in range(newdimx): if ((jnew*newdimx + inew)%100 == 0): print ' ' + fname + ' found: ', '{0:5.2f}'.format(float(jnew*newdimx + inew)*100/(newdimx*newdimy*1)), '%' masklonoldpt1 = oldlon2d >= newlon2d[jnew, inew] - float(ddegL) masklonoldpt2 = oldlon2d <= newlon2d[jnew, inew] + float(ddegL) masklonoldpt = masklonoldpt1 * masklonoldpt2 masklatoldpt1 = oldlat2d >= newlat2d[jnew, inew] - float(ddegl) masklatoldpt2 = oldlat2d <= newlat2d[jnew, inew] + float(ddegl) masklatoldpt = masklatoldpt1 * masklatoldpt2 Nmasklon = np.sum(masklonoldpt) Nmasklat = np.sum(masklatoldpt) if Nmasklon > 0 and Nmasklat > 0: ## print newlon2d[jnew, inew], newlat2d[jnew, inew], ' ### ', np.min(oldlon2d), np.max(oldlon2d), ':', np.min(oldlat2d), np.max(oldlat2d) ## print masklonoldpt.shape, masklatoldpt.shape, Nmasklon, Nmasklat maskindlon = indices_mask(masklonoldpt) maskindlat = indices_mask(masklatoldpt) mindist=1000000. # for jold in range(olddimy): # for iold in range(olddimx): # dist=np.sqrt((oldlon2d[jold, iold] - newlon2d[jnew, inew])**2. + \ # (oldlat2d[jold, iold] - newlat2d[jnew, inew])**2.) for jold in range(Nmasklat): for iold in range(Nmasklon): if maskindlon[iold,0] == maskindlat[jold,0] and maskindlon[iold,1] == maskindlat[jold,1]: dist=np.sqrt((newlon2d[jnew, inew] - oldlon2d[maskindlon[iold,0], maskindlon[iold,1]])**2. + \ (newlat2d[jnew, inew] - oldlat2d[maskindlat[jold,0], maskindlat[jold,1]])**2.) ## print jold, iold, ':' , maskindlon[iold,:], '-->', maskindlat[jold,:], '**** L:', \ ## oldlon2d[maskindlon[iold,0], maskindlon[iold,1]], 'l', oldlat2d[maskindlat[jold,0], maskindlat[jold,1]], \ ## 'dist:',dist, 'mindist: ', mindist if dist < mindist: mindist = dist ## print jold, iold, 'dist: ', dist, 'L', maskindlon[iold,:], 'l', maskindlat[jold,:] mappairs[0, jnew, inew] = maskindlon[iold,1] mappairs[1, jnew, inew] = maskindlat[jold,0] ## quit() varobj = oldncfile.variables[varn] varinf = variable_inf(varobj) # Creation of the new file ## ncfnew = NetCDFFile(ofile,'w') for idim in range(varinf.Ndims): newdimv = oldncfile.dimensions[varinf.dimns[idim]] if dimensions[varinf.dimns[idim]] == oldlon or dimensions[varinf.dimns[idim]] == oldlat: if dimensions[varinf.dimns[idim]] == oldlon: ncfnew.createDimension(varinf.dimns[idim], newdimx) else: ncfnew.createDimension(varinf.dimns[idim], newdimy) else: if newdimv.isunlimited(): ncfnew.createDimension(varinf.dimns[idim], None) else: ncfnew.createDimension(varinf.dimns[idim], varinf.dims[idim]) newvar = ncfnew.createVariable(varn, varinf.dtype, varinf.dimns, fill_value=fillValue) for attrn in varinf.attributes: if not attrn == '_FillValue': attr = newvar.setncattr(attrn, varobj.getncattr(attrn)) attr = newvar.setncattr('mask', 'variable remapped at nearest neighbors using ' + values.split(':')[0] + ' using ' + \ ' as longitude and latitude valules ') ncfnew.sync() ncfnew.close() for varns in ncf.variables: if not varns == varn: fvaradd(filename + ',' + varns, ofile) elif varns == oldlon: fvaradd(newprojectionfile + ',' + newlon, ofile) elif varns == oldlat: fvaradd(newprojectionfile + ',' + newlat, ofile) chvarname(oldlon, ofile, newlon) chvarname(oldlat, ofile, newlat) ncfnew = NetCDFFile(ofile,'a') # looping on all the variable ## varsecdims = [] Nvarloop=1 for insdim in range(varinf.Ndims - 2): Nvarloop = Nvarloop * varinf.dims[insdim] varsecdims.append(0) Nsecdims = len(varsecdims) for iloop in range(Nvarloop): varval = load_ncvariable_lastdims(oldncfile, varn, varsecdims, 2) for jnew in range(newdimy): for inew in range(newdimx): if ((jnew*newdimx + inew)%100 == 0): print ' ' + fname + ': ', '{0:5.2f}'.format(float((jnew*newdimx + inew)*100./(newdimx*newdimy))), '% done' newvarval[jnew, inew] = varval[mappairs[jnew, inew, 0], mappairs[jnew, inew, 1]] fill_ncvariable_lastdims(ncfnew,varn,varsecdims,2,newvarval) varsecdims[Nsecdims -1] = varsecdims[Nsecdims - 1] + 1 if varsecdims[Nsecdims - 1] > varinf.dims[Nsecdims - 1] - 1: varsecdims[Nsecdims - 1] = 0 for jdim in range(Nsecdims - 1): varsecdims[Nsecdims - 2 - jdim] = varsecdims[Nsecdims - 2 - jdim] + 1 if varsecdims[Nsecdims - 2 - jdim] <= varinf.dims[Nsecdims - 2 - jdim] - 1: break else: varsecdims[Nsecdims - 2 - jdim] = 0 ncfnew.sync() ncfnew.close() print 'remapnn: new file "' + ofile + '" with remapped variable written !!' return #remapnn('data/R1_CCRC_NARCliM_MOM_1950-2009_pracc.nc:lon,lat:lon,lat', 'data/AWAP_pre_1950-2009_pre_mask.nc', 'pre', 1. , 1.) #quit() def checkallvars(values, filen): """ Function to check all variables of a file values=[dimn1],[[dimn2],...,[dimnN]]:[dim1],[[dim2],...,[dimN]] [dimn1/N]: name of dimension [dim1/N]: value for the given dimension [0-N]: dimension value -1: last value -2: half of the dimension size -3: all values filen=name of the netCDF file to check """ fname = "'checkallvars'" if values == 'h': print fname + '_____________________________________________________________' print checkallvars.__doc__ quit() dimns = values.split(':')[0].split(',') dims0 = np.array(values.split(':')[1].split(',')) Ndims = len(dimns) dims = np.zeros( (Ndims), dtype=np.int) dimvals = np.zeros( (Ndims), dtype=np.int) ncobj = NetCDFFile(filen,'r') for idim in range(Ndims): dims[idim] = np.int(dims0[idim]) if not ncobj.dimensions.has_key(dimns[idim]): print errormsg print ' ' + fname + " file '" + filen + "' does not have dimension '" + dimns[idim] + "' !!!!" quit(-1) sizedim = len(ncobj.dimensions[dimns[idim]]) if dims[idim] == -3: dimvals[idim] = -1 elif dims[idim] == -2: dimvals[idim] = sizedim/2 elif dims[idim] == -1: dimvals[idim] = sizedim-1 else: dimvals[idim] = dims[idim] print " dimension '" + dimns[idim] + "' of size: ",sizedim,' shown at :',\ dimvals[idim] ivar = 1 for varn in ncobj.variables: varobj = ncobj.variables[varn] varinf = variable_inf(varobj) dimsvar = np.zeros( (varinf.Ndims), dtype=np.int) dimsvar[:] = -9 for ivardim in range(varinf.Ndims): vardimn = varinf.dimns[ivardim] for idim in range(Ndims): if vardimn == dimns[idim]: dimsvar[ivardim] = dimvals[idim] if dimsvar[ivardim] == -9: print errormsg print ' ' + fname + " No value given to the dimension '" + vardimn +\ "' !!" quit(-1) print ivar, ':', varn + ' with: ', varinf.dimns,' outputted as: ',dimsvar,' ___________________' if varinf.Ndims == 1: if dimsvar == -1: varvals = varobj[:] else: varvals = varobj[dimsvar] elif varinf.Ndims == 2: if dimsvar[0] == -1 and dimsvar[1] == -1: varvals = varobj[:] elif dimsvar[0] == -1 and dimsvar[1] != -1: varvals = varobj[:,dimsvar[1]] elif dimsvar[0] != -1 and dimsvar[1] == -1: varvals = varobj[dimsvar[0],:] else: varvals = varobj[dimsvar[0],dimsvar[1]] elif varinf.Ndims == 3: if dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] == -1: varvals = varobj[:] elif dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] != -1: varvals = varobj[:,:,dimsvar[2]] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] == -1: varvals = varobj[:,dimsvar[1],:] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] == -1: varvals = varobj[dimsvar[0],:,:] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] != -1: varvals = varobj[:,dimsvar[1],dimsvar[2]] elif dimsvar[0] != -1 and dimsvar[1] != -1 and dimsvar[2] == -1: varvals = varobj[dimsvar[0],dimsvar[1],:] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] != -1: varvals = varobj[dimsvar[0],:,dimsvar[2]] else: varvals = varobj[dimsvar[0],dimsvar[1],dimsvar[2]] elif varinf.Ndims == 4: if dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] == -1 and dimsvar[3] == -1: varvals = varobj[:] elif dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] == -1 and dimsvar[3] != -1: varvals = varobj[:,:,:,dimsvar[2]] elif dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] != -1 and dimsvar[3] == -1: varvals = varobj[:,:,dimsvar[2],:] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] == -1 and dimsvar[3] == -1: varvals = varobj[:,dimsvar[1],:,:] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] == -1 and dimsvar[3] == -1: varvals = varobj[dimsvar[0],:,:,:] elif dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] != -1 and dimsvar[3] != -1: varvals = varobj[:,:,dimsvar[2],dimsvar[3]] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] == -1 and dimsvar[3] != -1: varvals = varobj[:,dimsvar[1],:,dimsvar[3]] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] == -1 and dimsvar[3] != -1: varvals = varobj[dimsvar[1],:,:,dimsvar[3]] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] != -1 and dimsvar[3] == -1: varvals = varobj[:,dimsvar[1],dimsvar[2],:] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] != -1 and dimsvar[3] == -1: varvals = varobj[dimsvar[0],:,dimsvar[2],:] elif dimsvar[0] != -1 and dimsvar[1] != -1 and dimsvar[2] == -1 and dimsvar[3] == -1: varvals = varobj[dimsvar[0],dimsvar[1],:,:] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] != -1 and dimsvar[3] != -1: varvals = varobj[:,dimsvar[1],dimsvar[2],dimsvar[3]] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] != -1 and dimsvar[3] != -1: varvals = varobj[dimsvar[0],:,dimsvar[2],dimsvar[3]] elif dimsvar[0] != -1 and dimsvar[1] != -1 and dimsvar[2] == -1 and dimsvar[3] != -1: varvals = varobj[dimsvar[0],dimsvar[1],:,dimsvar[3]] elif dimsvar[0] != -1 and dimsvar[1] != -1 and dimsvar[2] != -1 and dimsvar[3] == -1: varvals = varobj[dimsvar[0],dimsvar[1],dimsvar[2],:] else: varvals = varobj[dimsvar[0],dimsvar[1],dimsvar[2],dimvals[3]] else: print errormsg print ' ' + fname + ' number of dimensions ',varinf.Ndims,' not ready!!' quit(-1) print varvals ivar = ivar + 1 ncobj.close() return def valmod_dim(values, ncfile, varn): """ Function to modify the value of a variable at a given dimension and value values = dimn,dimval,modins,modval1,[modval2,...] dimn = name of the dimension dimval = value of the dimension at which change the values modins = instruction: 'sumc', add [modval1] 'subc', substraction [modval1] 'mulc', multiply by [modval1] 'divc', divide by [modval1] 'lowthres': modify all values below [modval1] to [modval2] 'upthres': modify all values above [modval1] to [modval2] ncfile = netCDF file name varn = name of the variable >>> valmod_dim('num_metgrid_levels,10,mulc,-1','aqua_met-em_control.nc','PRES') """ fname = 'valmod_dim' if values == '-h': print """ Function to modify the value of a variable at a given dimension and value values = dimn,dimval,modins,modval1,[modval2,...] dimn = name of the dimension dimval = value of the dimension at which change the values modins = instruction: 'sumc', add [modval1] 'subc', substraction [modval1] 'mulc', multiply by [modval1] 'divc', divide by [modval1] 'lowthres': modify all values below [modval1] to [modval2] 'upthres': modify all values above [modval1] to [modval2] ncfile = netCDF file name varn = name of the variable >>> valmod_dim('num_metgrid_levels,10,mulc,-1','aqua_met-em_control.nc','PRES') """ quit() ## mat = np.array( (range(27)), dtype=np.float).reshape(3,3,3) vals = values.split(',') Lvals = len(vals) dimn = vals[0] dimval = np.int(vals[1]) modins = vals[2] modval = np.float(vals[3]) newvalues = ','.join(vals[2:Lvals]) if not os.path.isfile(ncfile): print errormsg print ' valmod: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.dimensions.has_key(dimn): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not have dimension "' + \ dimn + '" !!!!' ncf.close() quit(-1) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not have variable "' + \ varn + '" !!!!' ncf.close() quit(-1) varobj = ncf.variables[varn] varinf = variable_inf(varobj) varvals = varobj[:] # Dimension id iddim=varinf.Ndims - 1 for idimn in varinf.dimns: # print iddim, idimn if dimn == idimn: break iddim = iddim - 1 matshape = varvals.shape mattype = varvals.dtype matmask = np.zeros( tuple(matshape), dtype=bool) zerodims=np.ones( (6), dtype=int) Ndims = len(matshape) if Ndims > 6: print errmsg print ' ' + fname + ' number of dimensions: ',Ndims,' not ready!!!!!' quit(-1) zerodims[5-Ndims+1:6] = matshape newdims = [] for idim in range(Ndims): # print idim,'matshape[idim]:',matshape[Ndims-1-idim],'iddim:',iddim if idim != iddim: newdims.append(matshape[Ndims-1-idim]) newdims.reverse() mask = np.zeros(tuple(zerodims), dtype=bool) vals = np.zeros(tuple(zerodims), dtype=mattype) newvals = np.zeros(tuple(zerodims), dtype=mattype) vals[0:zerodims[0],0:zerodims[1],0:zerodims[2],0:zerodims[3],0:zerodims[4], \ 0:zerodims[5]] = varvals # print 'vals shape:',vals.shape,':',newdims for i5 in range(zerodims[5]): for i4 in range(zerodims[4]): for i3 in range(zerodims[3]): for i2 in range(zerodims[2]): for i1 in range(zerodims[1]): for i0 in range(zerodims[0]): # print 'iddim:',iddim,'dimval:',dimval,'-->',i0,i1,i2,i3,i4,i5 if iddim == 5 and dimval == i0: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 4 and dimval == i1: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 3 and dimval == i2: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 2 and dimval == i3: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 1 and dimval == i4: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 0 and dimval == i5: mask[i0,i1,i2,i3,i4,i5] = True # print 'newmat dim:',iddim,'val:',dimval,':',newdims,'______' newmat = vals[mask].reshape(tuple(newdims)) varmod = np.zeros(tuple(newdims), dtype=mattype) varshape = newmat.shape vartype = newmat.dtype Ndims = len(varshape) if Ndims <= 2: varmod = gen.valmodoper(newmat, newvalues) elif Ndims == 3: for i in range(varshape[0]): varmod[i,:,:] = gen.valmodoper(newmat[i,:,:], newvalues) elif Ndims == 4: for i in range(varshape[0]): for j in range(varshape[1]): varmod[i,j,:,:] = gen.valmodoper(newmat[i,j,:,:], newvalues) elif Ndims == 5: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): varmod[i,j,k,:,:] = gen.valmodoper(newmat[i,j,k,:,:], newvalues) newvals = np.where( mask, varmod, vals) zerdims = np.where( zerodims == 1, 0, zerodims ) finalvals = newvals[0:zerodims[0],0:zerodims[1],0:zerodims[2],0:zerodims[3],\ 0:zerodims[4],0:zerodims[5]] finalvals = np.squeeze(finalvals) if len(varobj.shape) != len(finalvals.shape): print ' Modified values have a different shape:', finalvals.shape, \ 'than variable:', varobj.shape print ' Assuming that extra dimensions are at the beggining' newfinalvals = finalvals.reshape(varobj.shape) varobj[:] = newfinalvals[:] else: varobj[:] = finalvals[:] ncf.sync() ncf.close() return def checkNaNs(values, filen): """ Function to check for NaN values over all variables in a file values = absolute lower threshold for NaN value filen=name of the netCDF file to check """ fname = "'checkNaNs'" valthres = np.float(values) ncobj = NetCDFFile(filen,'r') S1val = np.dtype('a1') Sval = np.dtype(str) Uval = np.dtype(unicode) ivar = 1 print fname + ' checking with: ',valthres,' in _______' for varn in ncobj.variables: varobj = ncobj.variables[varn] varinf = variable_inf(varobj) varvals = varobj[:] var1 = varvals.flatten()[0] if varinf.dtype != type('c') and varinf.dtype != type('Str') and \ varinf.dtype != Sval.type and varinf.dtype != Uval.type and \ varinf.dtype != S1val.type and type(var1) != S1val.type and \ type(var1) != Sval.type and type(var1) != Uval.type: print ' ' + varn + ':',varinf.dims found=0 if varinf.Ndims == 1: for i0 in range(varinf.dims[0]): if abs(varvals[i0]) > valthres or np.isnan(varvals[i0]): print ' ' + fname + ': NaN value found on variable "' + \ varn + '" at: ',i0,' value: ',varvals[i0] break elif varinf.Ndims == 2: for i1 in range(varinf.dims[0]): for i0 in range(varinf.dims[1]): if abs(varvals[i1,i0]) > valthres or np.isnan(varvals[i1,i0]): print ' ' + fname + ': NaN value found on variable "' + \ varn + '" at: ',i1,', ',i0,' value: ',varvals[i1,i0] found=1 break if found == 1: break elif varinf.Ndims == 3: for i2 in range(varinf.dims[0]): for i1 in range(varinf.dims[1]): for i0 in range(varinf.dims[2]): if abs(varvals[i2,i1,i0]) > valthres or \ np.isnan(varvals[i2,i1,i0]): print ' ' + fname + \ ': NaN value found on variable "' + varn + \ '" at: ', i2,', ',i1,', ',i0,' value: ', \ varvals[i2,i1,i0] found=1 break if found == 1: break if found == 1: break elif varinf.Ndims == 4: for i3 in range(varinf.dims[0]): for i2 in range(varinf.dims[1]): for i1 in range(varinf.dims[2]): for i0 in range(varinf.dims[3]): if abs(varvals[i3,i2,i1,i0]) > valthres or \ np.isnan(varvals[i3,i2,i1,i0]): print ' ' + fname + \ ': NaN value found on variable "' + varn + \ '" at: ', i3,', ', i2,', ',i1,', ',i0, \ ' value: ',varvals[i3,i2,i1,i0] found=1 break if found == 1: break if found == 1: break if found == 1: break elif varinf.Ndims == 5: for i4 in range(varinf.dims[0]): for i3 in range(varinf.dims[1]): for i2 in range(varinf.dims[2]): for i1 in range(varinf.dims[3]): for i0 in range(varinf.dims[4]): if abs(varvals[i4,i3,i2,i1,i0]) > valthres or \ np.isnan(varvals[i4,i3,i2,i1,i0]): print ' ' + fname + \ ': NaN value found on variable "' + varn + \ '" at: ', i4,', ', i3,', ', i2,', ',i1, \ ', ',i0,' value: ',varvals[i4,i3,i2,i1,i0] found=1 break if found == 1: break if found == 1: break if found == 1: break if found == 1: break else: print errormsg print ' ' + fname + ' number of dimensions ',varinf.Ndims, \ ' not ready!!' quit(-1) ivar = ivar + 1 ncobj.close() return ##checkNaNs(10.e10, '/d4/lflmd/etudes/WRF_LMDZ/WaquaL/WRF/control/wrfout_d01_1979-12-01_00:00:00') ##checkNaNs(10.e10, '/d4/lflmd/etudes/FF/tests/em_quarter_ss/run/wrfout_d01_0001-01-01_00:00:00') def checkAllValues(values, filen): """ Function to check for variables with along all their dimensions with the same value in a file values = value to check filen=name of the netCDF file to check """ fname = "'checkAllValues'" valcheck = np.float(values) ncobj = NetCDFFile(filen,'r') S1val = np.dtype('a1') Sval = np.dtype(str) Uval = np.dtype(unicode) ivar = 1 print fname + ' checking with: ',valcheck,' in _______' for varn in ncobj.variables: varobj = ncobj.variables[varn] varinf = variable_inf(varobj) varvals = varobj[:] var1 = varvals.flatten()[0] if varinf.dtype != type('c') and varinf.dtype != type('Str') and \ varinf.dtype != Sval.type and varinf.dtype != Uval.type and \ varinf.dtype != S1val.type and type(var1) != S1val.type and \ type(var1) != Sval.type and type(var1) != Uval.type: print ' ' + varn + ':',varinf.dims if np.all(varvals == valcheck): print ' ' + fname + ': variable "' + varn + \ '" with all its values == ',valcheck ivar = ivar + 1 ncobj.close() return ##checkAllValues(0., '/d4/lflmd/etudes/WRF_LMDZ/WaquaL/WRF/control/wrfout_d01_1979-12-01_00:00:00') ##checkAllValues(0., '/d4/lflmd/etudes/FF/tests/em_quarter_ss/run/wrfout_d01_0001-01-01_00:00:00') ##checkAllValues(0., '/home/lluis/etudes/WRF_LMDZ/tests/wrf_input/AR40.0/wrfout_d01_1979-01-01_00:00:00') def DataSetSection(values, filen): """ Function to get a section (values along a dimension) of a given data-set values= [dimn],[beg],[end],[int] [dimn]: name of the dimension [beg],[end],[int]: beginning, end and interval along the dimension-axis [end] = -1, values until the end [beg] = -9, last value filen= netCDF with the data-set DataSetSection('time,3,15,1','test.nc') """ fname = 'DataSetSection' if values == 'h': print fname + '_____________________________________________________________' print DataSetSection.__doc__ quit() dimn = values.split(',')[0] begv = np.int(values.split(',')[1]) endv = np.int(values.split(',')[2]) intv = np.int(values.split(',')[3]) ofile=filen.split('.')[0] + '_' + dimn + '_B' + str(begv) + '-E' + str(endv) + \ '-I' + str(intv) + '.nc' nciobj = NetCDFFile(filen,'r') ncoobj = NetCDFFile(ofile,'w') # Creating dimensions ## dimsecslice = {} for dims in nciobj.dimensions: ibegv = begv iendv = endv iintv = intv objdim = nciobj.dimensions[dims] if objdim.isunlimited(): dimsize = None if dims == dimn: if begv != -9: if begv > len(objdim): print errormsg print ' ' + fname + ': beginning value' ,begv, \ 'is larger than dimension (', len(objdim), ')!!!!' quit(-1) if endv > len(objdim): print errormsg print ' ' + fname + ': endining value' ,endv, \ 'is larger than dimension (', len(objdim), ')!!!!' quit(-1) if endv == -1: iendv = len(objdim) dimsize = (iendv - ibegv + 1)/iintv - 1 else: dimsize = 0 if begv == -9: ibegv = len(objdim) - 1 iendv = len(objdim) - 1 iintv = 0 else: if len(objdim) > 1: dimsize = len(objdim) ibegv = 0 iendv = dimsize - 1 iintv = 1 else: dimsize = 0 ibegv = 0 iendv = 0 iintv = 0 if ibegv != iendv: dimsecslice[dims] = slice(ibegv,iendv,iintv) else: dimsecslice[dims] = ibegv if dimsize != 0: if begv != -9: dim = ncoobj.createDimension(dims, dimsize) print ' ' + fname + ': adding dimension: ' + dims + ' size:',dimsize else: if dims != dimn: dim = ncoobj.createDimension(dims, dimsize) print ' ' + fname + ': adding dimension: ' + dims + ' size:',dimsize ncoobj.sync() # Creating variables ## filedims = ncoobj.dimensions for varns in nciobj.variables: print ' ' + fname + ': adding variable "' + varns + '"...' varobj = nciobj.variables[varns] vardims = varobj.dimensions if not gen.searchInlist(list(vardims),dimn): # removing monotones varslice = [] filevardims = [] for dimname in varobj.dimensions: lendim = len(nciobj.dimensions[dimname]) if lendim > 1: varslice.append(slice(0, lendim)) filevardims.append(dimname) else: varslice.append(0) varvals = varobj[tuple(varslice)] else: varslice = [] filevardims = [] for dimname in varobj.dimensions: if dimname == dimn: varslice.append(dimsecslice[dimn]) if type(dimsecslice[dimn]) == type(slice(1)): filevardims.append(dimname) else: Ldim = len(nciobj.dimensions[dimname]) if Ldim > 1: varslice.append(slice(0,Ldim)) filevardims.append(dimname) else: varslice.append(0) varvals = varobj[tuple(varslice)] # Adding fill value attribute ## varattrs = varobj.ncattrs() if gen.searchInlist(varattrs, '_FillValue'): varfil = varobj._FillValue else: varfil = None vartype = varobj.dtype if varfil is not None: newvar = ncoobj.createVariable(varns, vartype, tuple(filevardims), \ fill_value=varfil) else: newvar = ncoobj.createVariable(varns, vartype, tuple(filevardims)) ncoobj.sync() newvar[:] = varvals ncoobj.sync() for attrs in varattrs: if not attrs == '_FillValue': attrv = varobj.getncattr(attrs) attr = set_attribute(newvar, attrs, attrv) ncoobj.sync() # Global attributes ## for attrs in nciobj.ncattrs(): attrv = nciobj.getncattr(attrs) attr = set_attribute(ncoobj, attrs, attrv) nciobj.close() ncoobj.sync() ncoobj.close() onc = NetCDFFile(ofile, 'r') onc.close() print ' ' + fname + ' succesfull creation of file "' + ofile + '" !!!' return def DataSetSection_multidims(values, filen, varn): """ Function to get a section (values along multiple dimensions) of a given data-set values= [dimn1],[beg1],[end1],[int1]@[...[[dimnM],[begM],[endM],[intM]]] [dimni]: name of the dimension [begi],[endi],[inti]: beginning, end and interval along the dimension-axis [endi] = -1, last value NOTE: dimensions without values are taken allover their size filen= netCDF with the data-set varn= ',' list of variables, 'all' for all variables in data-set DataSetSection_multidims('Time,-1,-1,1@bottom_top,6,6,1','wrfout_d01_1979-12-01_00:00:00','all') """ fname = 'DataSetSection_multidims' if values == 'h': print fname + '_____________________________________________________________' print DataSetSection_multidims.__doc__ quit() Ndims = len(values.split('@')) dimns = [] ofiletile = '' begvs = np.zeros((Ndims), dtype=int) endvs = np.zeros((Ndims), dtype=int) intvs = np.zeros((Ndims), dtype=int) for idim in range(Ndims): val = values.split('@')[idim] dimns.append(val.split(',')[0]) begvs[idim] = int(val.split(',')[1]) endvs[idim] = int(val.split(',')[2]) intvs[idim] = int(val.split(',')[3]) ofiletile = ofiletile + '_' + dimns[idim] + '_B' + str(begvs[idim]) + '-E' + \ str(endvs[idim]) + '-I' + str(intvs[idim]) ofile = ofile=filen.split('.')[0] + ofiletile + '.nc' nciobj = NetCDFFile(filen,'r') ncoobj = NetCDFFile(ofile,'w') # Creating dimensions ## print ' ' + fname + ': adding dimensions...' newdimsizes = {} dimslices = {} varslice = [] for dims in nciobj.dimensions: objdim = nciobj.dimensions[dims] dimsize = -1 if not gen.searchInlist(dimns, dims): if objdim.isunlimited(): dimsize = None else: dimsize = len(objdim) varslice.append(slice(0,len(objdim))) dimslices[dims] = [0,len(objdim)] else: dimid = dimns.index(dims) if begvs[dimid] > len(objdim): print errormsg print ' ' + fname + ': beginning value' ,begvs[dimid], \ 'is larger than dimension (', len(objdim), ')!!!!' quit(-1) if endvs[dimid] > len(objdim): print errormsg print ' ' + fname + ': endining value' ,endvs[dimid], \ 'is larger than dimension (', len(objdim), ')!!!!' quit(-1) if begvs[dimid] != endvs[dimid]: if endvs[dimid] != -1: dimsize = (endvs[dimid]-begvs[dimid]+1)/intvs[dimid] dimslices[dims] = [begvs[dimid],endvs[dimid]+1,intvs[dimid]] else: dimsize = len(objdim)-begvs[dimid] dimslices[dims] = [begvs[dimid],len(objdim),1] else: dimsize = 1 if begvs[dimid] == -1: dimslices[dims] = [len(objdim)-1,len(objdim)-1,1] else: dimslices[dims] = [begvs[dimid],begvs[dimid],1] newdimsizes[dims] = dimsize if dimsize != -1: if len(objdim) == 1 or dimsize != 1: print ' ' + fname + ': adding dimension: '+ dims +' size:',dimsize if objdim.isunlimited(): dimsize=None dim = ncoobj.createDimension(dims, dimsize) ncoobj.sync() print ' ' + fname + ': Slice according to dimension selection: ',dimslices # Creating variables ## print ' ' + fname + ': adding variables...' if varn == 'all': getvarns = nciobj.variables else: getvarns = varn.split(',') for varns in getvarns: print ' ' + fname + ': adding variable "' + varns + '"...' varobj = nciobj.variables[varns] varorigdims = varobj.dimensions vardims = [] # From: http://stackoverflow.com/questions/1388818/how-can-i-compare-two-lists-in-python-and-return-matches coinc = list(set(varorigdims) & set(dimns)) if len(coinc) == 0: varvals = varobj[:] vardims = varorigdims else: varvals = None varslice = [] for dimname in varorigdims: if gen.searchInlist(dimns, dimname): if dimslices[dimname][0] != dimslices[dimname][1]: varslice.append(slice(dimslices[dimname][0], \ dimslices[dimname][1], dimslices[dimname][2])) else: varslice.append(dimslices[dimname][0]) if newdimsizes[dimname] != 1: vardims.append(dimname) else: Ldim = len(nciobj.dimensions[dimname]) varslice.append(slice(0,Ldim)) vardims.append(dimname) varvals = varobj[tuple(varslice)] # Adding fill value attribute ## # print ' final variable dimensions:',vardims varattrs = varobj.ncattrs() if gen.searchInlist(varattrs, '_FillValue'): varfil = varobj._FillValue else: varfil = False vartype = varobj.dtype newvar = ncoobj.createVariable(varns, vartype, tuple(vardims), \ fill_value=varfil) newvar[:] = varvals for attrs in varattrs: if not attrs == '_FillValue': attrv = varobj.getncattr(attrs) attr = set_attribute(newvar, attrs, attrv) # Global attributes ## for attrs in nciobj.ncattrs(): attrv = nciobj.getncattr(attrs) attr = set_attribute(ncoobj, attrs, attrv) slicevalS = '' for dimid in range(Ndims): dn = dimns[dimid] if not dimslices.has_key(dn): print errormsg print ' ' + fname + ": dimension name '" + dn + "' not present !!" print ' available ones:', dimslices quit(-1) if dimslices[dn][2] is not None: slicevalS = slicevalS + dn + ' (' + str(dimslices[dn][0]) + ',' + \ str(dimslices[dn][1]) + ',' + str(dimslices[dn][2]) + '); ' else: slicevalS = slicevalS + dn + ' (' + str(dimslices[dn][0]) + ',' + \ str(dimslices[dn][1]) + ',1); ' attr = set_attribute(ncoobj, 'sliced_dimensions', slicevalS) nciobj.close() ncoobj.sync() ncoobj.close() print ' ' + fname + ' succesfull creation of file "' + ofile + '" !!!' return #DataSetSection_multidims('Time,-1,-1,1@bottom_top,6,6,1','wrfout_d01_1979-12-01_00:00:00') #DataSetSection_multidims('bottom_top,6,6,1@south_north,3,3,1@west_east,26,26,1','wrfout_d01_1979-12-01_00:00:00') def sellonlatbox(values, ncfile, varn): """ Function to select a lotlan box from a data-set values= [lonName],[latName],[lonSW],[latSW],[lonNE],[latNE] [lonName]: name of the variable with the longitudes [latName]: name of the variable with the latitudes [lonSW],[latSW]: longitude and latitude of the SW vertex of the box [lonNE],[latNE]: longitude and latitude of the NE vertex of the box ncfile= netCDF file varn= ',' list of names of the variables ('all', for all variables) """ import numpy.ma as ma fname = 'sellonlatbox' if values == 'h': print fname + '_____________________________________________________________' print sellonlatbox.__doc__ quit() arguments = '[lonName],[latName],[lonSW],[latSW],[lonNE],[latNE]' gen.check_arguments(fname,values,arguments,',') lonn = values.split(',')[0] latn = values.split(',')[1] lonSW = np.float(values.split(',')[2]) latSW = np.float(values.split(',')[3]) lonNE = np.float(values.split(',')[4]) latNE = np.float(values.split(',')[5]) objfile = NetCDFFile(ncfile, 'r') if not objfile.variables.has_key(lonn): print errormsg print ' ' + fname + ": file '" + ncfile + "' des not have longitude " + \ "variable '" + lonn + "' !!" quit(-1) if not objfile.variables.has_key(latn): print errormsg print ' ' + fname + ": file '" + ncfile + "' des not have latitude " + \ "variable '" + lonn + "' !!" quit(-1) lonobj = objfile.variables[lonn] latobj = objfile.variables[latn] loninf = variable_inf(lonobj) latinf = variable_inf(latobj) if len(lonobj.shape) <= 2: lonv = lonobj[:] latv = latobj[:] else: print warnmsg print ' ' + fname + ': assuming that lon/lat is constant to the first dimension!!' print ' shape lon:', lonobj.shape, 'lat:', latobj.shape lonv = lonobj[0,] latv = latobj[0,] if varn == 'all': varns = objfile.variables filevarn = varn elif varn.find(',') != -1: varns = varn.split(',') filevarn = varn.replace(',','-') else: varns = [varn] filevarn = varn ofile = 'sellonlatbox_' + filevarn + '.nc' # Lon/lat box nalon = lonv.min() nalat = latv.min() xalon = lonv.max() xalat = latv.max() if lonSW < nalon: print errormsg print ' ' + fname + ': longitude SW corner:', lonSW,' too small!!' print ' min lon data:', nalon, 'increase SW corner longitude' quit(-1) if latSW < nalat: print errormsg print ' ' + fname + ': latitude SW corner:', latSW,' too small!!' print ' min lat data:', nalat, 'increase SW corner latitude' quit(-1) if lonNE > xalon: print errormsg print ' ' + fname + ': longitude NE corner:', lonNE,' too large!!' print ' max lon data:', xalon, 'decrease NE corner longitude' quit(-1) if latNE > xalat: print errormsg print ' ' + fname + ': latitude NE corner:', latNE,' too large!!' print ' max lat data:', xalat, 'decrease NE corner latitude' quit(-1) if lonSW < lonNE: malon = ma.masked_outside(lonv,lonSW,lonNE) else: malon = ma.masked_outside(lonv,lonNE,lonSW) if latSW < latNE: malat = ma.masked_outside(latv,latSW,latNE) else: malat = ma.masked_outside(latv,latNE,latSW) if len(lonobj.shape) == 1: dlonn = loninf.dimns[0] dlatn = latinf.dimns[0] dimx = lonv.shape[0] dimy = latv.shape[0] malonlat = np.meshgrid(malon.mask,malat.mask) malonv = ma.array(lonv, mask=malon) malatv = ma.array(latv, mask=malat) elif len(lonobj.shape) == 2: dlonn = loninf.dimns[1] dlatn = loninf.dimns[0] dimx = lonv.shape[1] dimy = lonv.shape[0] malonlat = malon.mask + malat.mask malonv = ma.array(lonv, mask=malonlat) malatv = ma.array(latv, mask=malonlat) elif len(lonobj.shape) == 3: dlonn = loninf.dimns[2] dlatn = loninf.dimns[1] dimx = lonv.shape[1] dimy = lonv.shape[0] malonlat = np.zeros((dimy,dimx), dtype=bool) malonlat = malon.mask + malat.mask malonv = ma.array(lonv, mask=malonlat) malatv = ma.array(latv, mask=malonlat) else: print errormsg print ' ' + fname + ': matrix shapes not ready!!' print ' shape lon:',lonv.shape,'lat:',latv.shape quit(-1) # Size of the new domain nlon = malon.min() nlat = malat.min() xlon = malon.max() xlat = malat.max() print ' ' + fname + ': data box: SW', nlon, nlat,'NE:',xlon,xlat print ' ' + fname + ": name of the longitudinal dimension: '" + dlonn + \ "' latitudinal: '" + dlatn ilon = dimx elon = 0 ilat = dimy elat = 0 if len(lonv.shape) > 1: for j in range(dimy): for i in range(dimx): if malonlat.mask[j,i] == False and i < ilon: ilon = i if malonlat.mask[j,i] == False and i > elon: elon = i if malonlat.mask[j,i] == False and j < ilat: ilat = j if malonlat.mask[j,i] == False and j > elat: elat = j else: for i in range(dimx): if malon.mask[i] == False and i < ilon: ilon = i if malon.mask[i] == False and i > elon: elon = i for j in range(dimy): if malat.mask[j] == False and j < ilat: ilat = j if malat.mask[j] == False and j > elat: elat = j print 'ilon elon ilat elat:', ilon, elon, ilat, elat newdimx = elon - ilon + 1 newdimy = elat - ilat + 1 if len(malon.shape) == 1: datalonSW = lonv[ilon] datalatSW = latv[ilat] datalonNE = lonv[elon] datalatNE = latv[elat] elif len(malon.shape) == 2: datalonSW = lonv[ilat,ilon] datalatSW = latv[ilat,ilon] datalonNE = lonv[elat,elon] datalatNE = latv[elat,elon] elif len(malon.shape) == 3: datalonSW = lonv[0,ilat,ilon] datalatSW = latv[0,ilat,ilon] datalonNE = lonv[0,elat,elon] datalatNE = latv[0,elat,elon] print ' ' + fname + ': final SW:', datalonSW, ',', datalatSW, 'NE:', datalonNE, \ ',', datalatNE print ' ' + fname + ': new lon size:', ilon, ',', elon, 'new dimx:', newdimx print ' ' + fname + ': new lat size:', ilat, ',', elat, 'new dimy:', newdimy newncobj = NetCDFFile(ofile, 'w') # Creation of lon,lat related dims if dlonn != dlatn: newdim = newncobj.createDimension(dlonn, newdimx) newdim = newncobj.createDimension(dlatn, newdimy) else: # Only in 1D data!! newdim = newncobj.createDimension(dlonn, newdimx) # Adding lonn/latn to the output file if not gen.searchInlist(varns,lonn): varns.append(lonn) if not gen.searchInlist(varns,latn): varns.append(latn) for vn in varns: print 'Adding "' + vn + "' ..." if not objfile.variables.has_key(lonn): print errormsg print ' ' + fname + ": file '" + ncfile + "' des not have variable '" + \ vn + "' !!" quit(-1) if not gen.searchInlist(objfile.variables, vn): print errormsg print ' ' + fname + ': variable name "' + vn + '" is not in file "' + \ ncfile + '" !!!!!' quit(-1) varobj = objfile.variables[vn] varinf = variable_inf(varobj) # Adding variable dimensions varslice = [] for dimn in varinf.dimns: if not gen.searchInlist(newncobj.dimensions, dimn): if objfile.dimensions[dimn].isunlimited(): newncobj.createDimension(dimn, None) else: newncobj.createDimension(dimn, len(objfile.dimensions[dimn])) # Very special case when we have for example a list of stations (which is the case!) if dimn == dlonn and dimx != dimy: varslice.append(slice(ilon,elon+1)) elif dimn == dlatn and dimx != dimy: varslice.append(slice(ilat,elat+1)) else: varslice.append(slice(0,len(objfile.dimensions[dimn]))) if varinf.FillValue is not None: newvar = newncobj.createVariable(vn, nctype(varinf.dtype), varinf.dimns) else: newvar = newncobj.createVariable(vn, nctype(varinf.dtype), varinf.dimns, \ fill_value = varinf.FillValue) if dimx != dimy: newvar[:] = varobj[tuple(varslice)] else: # Very special case when we have for example a list of stations (which is the case!) print ' ' + fname + ': woow we are in a stations-list like of file...' inpos0 = 0 for i in range(newdimx): inpos = gen.index_vec(malonlat[inpos0:dimx+1], False) inpos = inpos + inpos0 inpos0 = inpos + 1 varslice = [] newvarslice = [] for dimn in varinf.dimns: if not gen.searchInlist(newncobj.dimensions, dimn): newncobj.createDimension(dimn, len(objfile.dimensions[dimn])) # Very special case when we have for example a list of stations (which is the case!) if dimn == dlonn: varslice.append(inpos) newvarslice.append(i) else: varslice.append(slice(0,len(objfile.dimensions[dimn]))) newvarslice.append(slice(0,len(objfile.dimensions[dimn]))) newvar[tuple(newvarslice)] = varobj[tuple(varslice)] for atrn in varinf.attributes: if atrn != '_FillValue': attrv = varobj.getncattr(atrn) newattr = set_attributek(newvar, atrn, attrv, type(attrv)) newncobj.sync() for atrn in objfile.ncattrs(): attrv = objfile.getncattr(atrn) newattr = set_attributek(newncobj, atrn, attrv, type(attrv)) # global attributes newncobj.setncattr('author', 'L. Fita') newattr = set_attributek(newncobj, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') newncobj.setncattr('university', 'Pierre Marie Curie - Jussieu') newncobj.setncattr('center', 'Centre National de Recherches Scientifiques') newncobj.setncattr('city', 'Paris') newncobj.setncattr('country', 'France') newncobj.setncattr('script', 'nc_var_tools.py') newncobj.setncattr('function', 'sellonlatbox') newncobj.setncattr('version', '1.0') newncobj.setncattr('original_file', ncfile) newnattr = set_attributek(newncobj, 'lonSW', lonSW, 'R') newnattr = set_attributek(newncobj, 'latSW', latSW, 'R') newnattr = set_attributek(newncobj, 'lonNE', lonNE, 'R') newnattr = set_attributek(newncobj, 'latNE', latNE, 'R') newnattr = set_attributek(newncobj, 'data_lonSW', datalonSW, 'R') newnattr = set_attributek(newncobj, 'data_latSW', datalatSW, 'R') newnattr = set_attributek(newncobj, 'data_lonNE', datalonNE, 'R') newnattr = set_attributek(newncobj, 'data_latNE', datalatNE, 'R') newnattr = set_attributek(newncobj, 'data_pt_lonSW', ilon, 'I') newnattr = set_attributek(newncobj, 'data_pt_latSW', ilat, 'I') newnattr = set_attributek(newncobj, 'data_pt_lonNE', elon, 'I') newnattr = set_attributek(newncobj, 'data_pt_latNE', elat, 'I') objfile.close() newncobj.sync() newncobj.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return def sellonlatboxold(values, ncfile, varn): """ Function to select a lotlan box from a data-set values= [lonName],[latName],[lonSW],[latSW],[lonNE],[latNE] [lonName]: name of the variable with the longitudes [latName]: name of the variable with the latitudes [lonSW],[latSW]: longitude and latitude of the SW vertex of the box [lonNE],[latNE]: longitude and latitude of the NE vertex of the box ncfile= netCDF file varn= ',' list of names of the variables ('all', for all variables) """ fname = 'sellonlatboxold' if values == 'h': print fname + '_____________________________________________________________' print sellonlatbox.__doc__ quit() ofile = 'sellonlatbox_' + varn + '.nc' lonn = values.split(',')[0] latn = values.split(',')[1] lonSW = np.float(values.split(',')[2]) latSW = np.float(values.split(',')[3]) lonNE = np.float(values.split(',')[4]) latNE = np.float(values.split(',')[5]) objfile = NetCDFFile(ncfile, 'r') lonobj = objfile.variables[lonn] latobj = objfile.variables[latn] lonv = lonobj[:] latv = latobj[:] if varn == 'all': varns = objfile.variables elif varn.find(',') != -1: varns = varn.split(',') else: varns = [varn] for vn in varns: if not searchInlist(objfile.variables, vn): print errormsg print ' ' + fname + ': variable name "' + vn + '" is not in file "' + \ ncfile + '" !!!!!' quit(-1) varobj = objfile.variables[vn] Ndimslon = len(lonobj.shape) Ndimslat = len(latobj.shape) Ndimsvar = len(varobj.shape) # Looking for coincidence of dimensions samedim = [] for idv in range(Ndimsvar): for idl in range(Ndimslon): if varobj.dimensions[idv] == lonobj.dimensions[idl]: if not searchInlist(samedim,varobj.dimensions[idv]): samedim.append(varobj.dimensions[idv]) break for idl in range(Ndimslat): if varobj.dimensions[idv] == latobj.dimensions[idl]: if not searchInlist(samedim,varobj.dimensions[idv]): samedim.append(varobj.dimensions[idv]) break Ndimshare = len(samedim) print 'variable and lon/lat matrices share ', Ndimshare,' dimensions: ',samedim samedimslonlat = True for idL in range(Ndimslon): for idl in range(Ndimslat): if lonobj.dimensions[idl] != latobj.dimensions[idl]: samedimslonlat = False break if not samedimslonlat: break # Creation of the lon/lat matrices to search if Ndimshare == 1: lonmat = lonobj[:] latmat = latobj[:] elif Ndimshare == 2: if samedimslonlat: lonmat = lonobj[:] latmat = latobj[:] else: if Ndimslon != 1 and Ndimslat != 1: print errormsg print ' ' + fname + ': I do not know how to keep going!' print ' lon ', Ndimslon,' and lat ',Ndimslat, \ ' matrices do not share the same dimensions, and do not ' + \ 'have shape 1' quit(-1) else: lonmat = np.zeros((latobj.shape[0], lonobj.shape[0]), dtype=np.float) latmat = np.zeros((latobj.shape[0], lonobj.shape[0]), dtype=np.float) for i in range(lonobj.shape[0]): latmat[:,i] = latobj[:] for j in range(latobj.shape[0]): lonmat[j,:] = lonobj[:] elif Ndimshare == 3: if samedimslonlat: lonmat = lonobj[0,:,:] latmat = latobj[0,:,:] else: print errormsg print ' ' + fname + ': dimension sharing of lon/lat not ready!' quit(-1) # Searching for inside points iind = 0 if Ndimshare == 1: inside = {} for iL in range(lonobj.shape[0]): if lonobj[iL] >= lonSW and lonobj[iL] <= lonNE and latobj[iL] >= latSW\ and latobj[iL] <= latNE: inside[iind] = iL iind = iind + 1 elif Ndimshare == 2: newidx = [] newidy = [] if (len(lonobj.shape) == 3): inside = np.zeros((lonobj.shape[1], lonobj.shape[2]), dtype=bool) else: inside = np.zeros((lonobj.shape), dtype=bool) for iL in range(lonobj.shape[1]): for il in range(lonobj.shape[0]): if lonobj[il,iL] >= lonSW and lonobj[il,iL] <= lonNE and \ latobj[il,iL] >= latSW and latobj[il,iL] <= latNE: if not searchInlist(newidx, iL): newidx.append(iL) if not searchInlist(newidy, il): newidy.append(il) inside[il, iL] = True iind = iind + 1 elif Ndimshare == 3: newidx = [] newidy = [] inside = np.zeros((lonobj.shape[1], lonobj.shape[2]), dtype=bool) for iL in range(lonobj.shape[2]): for il in range(lonobj.shape[1]): if lonobj[0,il,iL] >= lonSW and lonobj[0,il,iL] <= lonNE and \ latobj[0,il,iL] >= latSW and latobj[0,il,iL] <= latNE: if not searchInlist(newidx, iL): newidx.append(iL) if not searchInlist(newidy, il): newidy.append(il) inside[il, iL] = True iind = iind + 1 Ninpts = len(inside) newdx = len(newidx) newdy = len(newidy) print ' ' + fname + ': ', Ninpts, 'pts found in the box:', lonSW, ',', \ latSW, 'x', lonNE, ',', latNE # Creation of cropped matrices inlon = np.zeros((Ninpts), dtype=np.float) inlat = np.zeros((Ninpts), dtype=np.float) invar = np.zeros((varobj.shape[0],Ninpts), dtype=np.float) # Creation of the netCDF file ## if vn in varns[0]: objofile = NetCDFFile(ofile, 'w') if Ndimshare == 1: # Dimensions newdim = objofile.createDimension('boxpt', Ninpts) newdim = objofile.createDimension('time', None) # var dimensions newvar = objofile.createVariable('lon', 'f8', ('boxpt')) newattr = basicvardef(newvar, 'lon', 'longitude', 'degrees west_east') newvar[:] = lonobj[inside[iin]] newvar = objofile.createVariable('lat', 'f8', ('boxpt')) newattr = basicvardef(newvar, 'lat', 'latitude', 'degrees north_south') newvar[:] = latobj[inside[iin]] newvar = objofile.createVariable('time', 'f8', ('time')) if objfile.variables.has_key('time'): otime = objfile.variables['time'] timevals = otime[:] if searchInlist(otime.ncattrs(),'units'): tunits = otime.getncattr['units'] else: tunits = 'unkown' else: timevals = np.arange(varobj.shape[0]) tunits = 'unkown' newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = timevals # variable newvar = objofile.createVariable(varn, 'f4', ('time', 'boxpt')) newvar[:] = varobj[invar] else: # Dimensions newdim = objofile.createDimension('x', newdx) newdim = objofile.createDimension('y', newdy) newdim = objofile.createDimension('time', None) # var dimensions newvar = objofile.createVariable('lon', 'f8', ('y', 'x')) if Ndimshare == 2: Vals = lonobj[:] else: Vals = lonobj[0,:,:] for it in range(varobj.shape[0]): newvar[:] = Vals[inside].reshape(newdy,newdx) newattr = basicvardef(newvar, 'lon', 'longitude', 'degrees west_east') newvar = objofile.createVariable('lat', 'f8', ('y', 'x')) newattr = basicvardef(newvar, 'lat', 'latitude', 'degrees north_south') if Ndimshare == 2: Vals = latobj[:] else: Vals = latobj[0,:,:] for it in range(varobj.shape[0]): newvar[:] = Vals[inside].reshape(newdy,newdx) newvar = objofile.createVariable('time', 'f8', ('time')) if objfile.variables.has_key('time'): otime = objfile.variables['time'] timevals = otime[:] if searchInlist(otime.ncattrs(),'units'): tunits = otime.getncattr['units'] else: tunits = 'unkown' else: timevals = np.arange(varobj.shape[0]) tunits = 'unkown' newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = timevals # variable newvar = objofile.createVariable(varn, 'f4', ('time', 'y', 'x')) for it in range(varobj.shape[0]): valsvar = varobj[it,:,:] newvar[it,:,:] = valsvar[inside] if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = varn if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = varn if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = 'unkown' newattr = basicvardef(newvar, vsname, vlname, vunits) # global attributes objofile.setncattr('author', 'L. Fita') objofile.setncattr('institution', 'Laboratire de Meteorologie Dynamique') objofile.setncattr('university', 'Pierre Marie Curie - Jussieu') objofile.setncattr('center', 'Centre National de Recherches Scientifiques') objofile.setncattr('city', 'Paris') objofile.setncattr('country', 'France') objofile.setncattr('script', 'nc_var_tools.py') objofile.setncattr('function', 'sellonlatbox') objofile.setncattr('version', '1.0') objfile.close() objofile.sync() objofile.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return #sellonlatbox('h', '/san0/lflmd/DATA/SAFRAN/ForcPRCP_france_SAFRAN_8Km_1hour_1996080100_1997073123_V1_01.nc', 'RR') #sellonlatbox('lon,lat,2.,42.5,5.,45.', '/home/lluis/DATA/SAFRAN/safran_RR_1996.nc', 'RR') #sellonlatbox('lon,lat,2.,42.5,5.,45.', '/san0/lflmd/DATA/SAFRAN/ForcPRCP_france_SAFRAN_8Km_1hour_1996080100_1997073123_V1_01.nc', 'product') #sellonlatbox('XLONG,XLAT,2.,42.5,5.,45.', '/san0/lflmd/etudes/WRF_LMDZ/WL_HyMeX/Cevennes96/control/wrfout/wrfout_d01_1996-09-17_00:00:00', 'RAINC') def compute_deaccum(values, ncfile, varname): """ Function to compute deaccum: deaccumulation of a variable (VAR[t+1]-VAR[t]) values= [timedimname] name of the time dimension ncfile= netCDF file to use varname= variable to deaccumulate """ fname='compute_deaccum' ofile = 'deaccumulated_' + varname + '.nc' if values == 'h': print fname + '_____________________________________________________________' print compute_deacccum.__doc__ quit() objnc = NetCDFFile(ncfile, 'r') if not objnc.variables.has_key(varname): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varname + '" !!!!' quit(-1) var1obj = objnc.variables[varname] vardims = var1obj.dimensions Ndimvar = len(vardims) vdim = [] deacdims = [] # Searcing for time dimension for idd in range(Ndimvar): vdim.append(slice(0,var1obj.shape[idd])) deacdims.append(var1obj.shape[idd]) if vardims[idd] == values: dimtid = idd dimt = var1obj.shape[dimtid] print 'Number of time-steps: ',dimt,' id dimt: ',dimtid deacdims[dimtid] = dimt varvals = np.zeros(tuple(deacdims), dtype=np.float) # Slicing for the deaccumulation for it in range(dimt): vals1 = list(vdim) vals2 = list(vdim) vals1[dimtid] = it if it > 0: vals2[dimtid] = it - 1 else: vals2[dimtid] = it var1 = var1obj[tuple(vals1)] var2 = var1obj[tuple(vals2)] varvals[tuple(vals1)] = var1-var2 # Creation of the new file objofile = NetCDFFile(ofile, 'w') # Dimensions fdims = objnc.dimensions for dimn in fdims: dimobj = objnc.dimensions[dimn] if dimobj.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(dimobj) newdim = objofile.createDimension(dimn, dimsize) # Variables fvars = objnc.variables for varn in fvars: varobj = objnc.variables[varn] if varn == varname: newvarn = 'deaccum' + varn else: newvarn = varn newvar = objofile.createVariable(newvarn, varobj.dtype, varobj.dimensions) for attrn in varobj.ncattrs(): attrval = varobj.getncattr(attrn) newattr = newvar.setncattr(attrn, attrval) if varn == varname: newvar[:] = varvals else: newvar[:] = varobj[:] # Global attributes for attrn in objnc.ncattrs(): attrval = objnc.getncattr(attrn) newattr = objofile.setncattr(attrn, attrval) objnc.close() objofile.sync() objofile.close() print ' ' + fname + ' success written of "' + ofile + '" !!!!!' return #compute_deaccum('time', 'control/sellonlatbox_PRECIP_dayaccum.nc', 'PRECIP') def compute_opersvarsfiles(values, varinfo): """ Function to compute opersvarfiles: operation of variables from different files (OPER1.FILE1_VAR1 OPER2.FILE2_VAR2), operations are going to be sequentially made compute_opersvarsfiles(values, varinfo). [prevalues]: values from the previous operations NOTE: to start with a multiple operations one can use as operation of the first variable 'addc,0' values= '%' separated list of dimension ranges and file,operation,variable to compute [dimranges1]@[operfile1var]%[dimranges2]@[operfile2var]%[...[dimrangesM]@[operfileMvar]] dimranges: ';' separated list of dimension, dimension-variable names and their ranges ('|' separated) [dim1]|[dimv1]|[range];[dim2]|[dimv2]|[range];[...|[dimN]|[dimvN]|[range]] with [range] as: -1, all the values -9, last values int, a single value [beg,end,frq], from a beginning to an end with a given frequency 'WRFtime': as time variable computed from WRF 'Times' variable NOTE: use 'contoperation' to continue an operation with the result of previous operations [prevalues] without using a variable from a file operfilevar: [oper]|[file]|[var] '|' separated list of triplets of [operation], [file], [variable name to use] * [oper]: operations (being [prevalues] the values computed until the operation) 'add': adding [var] ([prevalues] + [var]) 'addc',[modval1]: add [modval1] to [prevalues] 'arctan', arctangent [-pi, pi] 'centerderiv',[N],[ord],[dim]: un-scaled center [N]-derivative of order [ord] along dimension [dim] of [var] 'div': dividing by [var] ([prevalues] / [var]) 'divc',[modval1]: [prevalues] divide by [modval1] 'forwrdderiv',[N],[ord],[dim]: un-scaled forward [N]-derivative of order [ord] along dimension [dim] of [var] 'ifreq_anom',[stepdimn],[stepvardimn],[istep]: computing anomalies by substracting sub-means at each step by averaging from there all values taken every [istep] along dimension [stepdim]. mean(j) = sum(matA[j+k*istep]_k=0,Nstep)/Nstep; Nstep = len(stepdimn)/istep; j=[0,istep] [stepdimn]= name of the dimension along which to sample [stepvardimn]= name of the variable-dimension with the values for [stepdimn] [istep]= frequency to sample 'ifreq_mean',[stepdimn],[stepvardimn],[istep]: computing sub-means at each step by averaging from there all values taken every [istep] along dimension [stepdim]. mean(j) = sum(matA[j+k*istep]_k=0,Nstep)/Nstep; Nstep = len(stepdimn)/istep; j=[0,istep] [stepdimn]= name of the dimension along which to sample [stepvardimn]= name of the variable-dimension with the values for [stepdimn] [istep]= frequency to sample 'ifreq_normmeanstd',[stepdimn],[stepvardimn],[istep]: nbormalizing anomalies by substracting mean(j)/dtsv(j) sub-stats at each step by averaging from there all values taken every [istep] along dimension[stepdim]. mean(j) = sum(matA[j+k*istep]_k=0,Nstep)/Nstep; std(j) = std(matA[j+k*istep]_k=0,Nstep); Nstep = len(stepdimn)/istep; j=[0,istep] [stepdimn]= name of the dimension along which to sample [stepvardimn]= name of the variable-dimension with the values for [stepdimn] [istep]= frequency to sample 'inv': inverting [prevalues] (1/[prevalues]) 'lowthres',[modval1],[modval2]: if [prevalues] < [modval1]; prevalues = [modval2] 'lowthres@oper',[modval1],[oper],[modval2]: if [prevalues] < [modval1]; prevalues = [oper] (operation as [modval2]) 'mul': multiplying by [var] ([prevvalues] * [var]) 'mulc',[modval1]: [prevalues] multiplied by [modval1] 'norm_meanminmax',[NOTnormdims]: normalization of data as: (val-)/(max(val)-min(val)) except along dimensions [NOTnormdims] (':' list of dimension names, or 'any' for using all dimensions) 'norm_minmax',[NOTnormdims]: normalization of data as: [2val-(max(val)-min(val)]/[max(val)+min(val)] except along dimensions [NOTnormdims] (':' list of dimension names, or 'any' for using all dimensions) 'norm_meanstd',[NOTnormdims]: normalization of data as: (val-)/stdev(val) except along dimensions [NOTnormdims] (':' list of dimension names, or 'any' for using all dimensions) 'pot': powering with [var] ([prevalues] ** [var]) 'potc',[modval1]: [prevalues] ** [modval1] 'repl',[modval1]: replace values of [prevalues] with values from [operfile2var] except values in second file = [modval1] 'replbig',[modval1]: replace values of [prevalues] with values from [operfile2var] except values in second file > [modval1] 'repless',[modval1]: replace values of [prevalues] with values from [operfile2var] except values in second file < [modval1] 'sub': substracting [var] ([prevalues] - [var]) 'subc',[modval1]: remove [modval1] of [prevalues] 'upthres',[modval1],[modval2]: if [prevalues] > [modval1]; prevalues = [modval2] 'upthres@oper',[modval1],[oper],[modval2]: if [prevalues] > [modval1]; prevalues = [oper] (operation with [modval2]) * [file]: name of the file * [var]: variable to use ('None' for the case with constant operations: 'addc', 'divc', 'inv', 'lowthres', 'lowthres@oper', 'mulc', 'potc', 'subc', 'upthres', 'upthres@oper' varinfo= [varname],[Lvarname],[varunits] varname: name of the final variable (to be used as standard name) Lvarname: long name of the final variable ('!' for spaces) varunits: units of the final variable ('!' for spaces) """ import numpy.ma as ma fname='compute_opersvarsfiles' if values == 'h': print fname + '_____________________________________________________________' print compute_opersvarsfiles.__doc__ quit() # List of operations which need to be started with a matrix of zeros opinizero = ['add', 'centerderiv', 'forwrdderiv', 'sub'] # List of operations which need to be started with a matrix of ones opinione = ['mul', 'div', 'pot'] # Operations which need to deal with dimension's names and indices opdim = ['ifreq_anom', 'ifreq_mean', 'ifreq_normmeanstd', 'norm_meanminmax', \ 'norm_meanstd', 'norm_minmax'] # Operations which require to remove some dimensions oprmdim = ['norm_meanminmax', 'norm_meanstd', 'norm_minmax'] # Operations which require a new dimension opnewdim = ['ifreq_mean'] # istep Operations opistep = ['ifreq_anom', 'ifreq_mean', 'ifreq_normmeanstd'] varname = varinfo.split(',')[0] Lvarname = varinfo.split(',')[1].replace('!',' ') varunits = varinfo.split(',')[2].replace('!',' ') ofile = 'opersvarsfiles_' + varname + '.nc' fileopers = values.split('%') # Getting range,files,operations, ... Nfileopers = len(fileopers) filens = [] ranges = [] opers = [] varns = [] for fileoper in fileopers: arguments = '[dimranges]@[operfilevar]' gen.check_arguments(fname, fileoper, arguments, '@') fnop = fileoper.split('@')[1] if fileoper.split('@')[0] != 'contoperation': ranges.append(fileoper.split('@')[0]) arguments = '[oper]|[file]|[var]' gen.check_arguments(fname, fnop, arguments, '|') opers.append(fnop.split('|')[0]) filens.append(fnop.split('|')[1]) varns.append(fnop.split('|')[2]) else: ranges.append('continuation') opers.append(fnop) filens.append('continuation') varns.append('continuation') print ' ' + fname + 'Files/Operations to perform _______' for ifop in range(Nfileopers): if filens[ifop] != 'continuation': print ' ' + filens[ifop] + ' range: ' + ranges[ifop] + ' operation: '+\ opers[ifop] + ' variable: ' + varns[ifop] else: print ' continuing computation by: ' + opers[ifop] newunits = '-' opervars = [] # Dictionary with the sizes of the dimensions of prevalues Dds = {} # Dictionary with the values of the dimension-variables of the dimensions of `prevalues' Ddv = {} # Dictionary with the dimensions of the dimension-variables of the dimensions of `prevalues' Dddv = {} # Dictionary with the type of the dimension-variables of the dimensions of `prevalues' Dtdv = {} # Dictionary with the attributes dictionay of the dimension-variables of the dimensions of `prevalues' Dadv = {} # Dictionary with the additional values for certain operations of `prevalues' Dadditionalv = {} # Dictionary with the additional dimensions which are not used for certain operations of `prevalues' Dadditionaldim = {} # Dictionary with the additional units used for certain operations of `prevalues' Dadditionalunits = {} # Dictionary with the Headers for the names of the additional variables DHadditionalvars = {'ifreq_anom': ['ifreq_mean'], 'ifreq_normmeanstd': \ ['ifreq_mean', 'ifreq_std'], 'norm_meanminmax': ['mean','min','max'], \ 'norm_meanstd': ['mean','std'], 'norm_minmax': ['min','max']} prevalues = None slicewrft = None for ifv in range(Nfileopers): operN = opers[ifv] filen = filens[ifv] varn = varns[ifv] rng = ranges[ifv].split(';') if operN.find(',') != -1: opernS = operN.split(',')[0] else: opernS = operN opervars.append(varn) if filen != 'continuation': if not os.path.isfile(filen): print errormsg print ' ' + fname + ' file: "' + filen + '" does not exist !!' quit(-1) objnc = NetCDFFile(filen, 'r') # Operation with a variable if varn != 'WRFtime' and not objnc.variables.has_key(varn): print errormsg print ' ' + fname + ': netCDF file "' + filen + \ '" does not have variable "' + varn + '" !!!!' quit(-1) varobj = objnc.variables[varn] # Slice values dictslice = {} Ddimvs = [] for rg in rng: dimn = rg.split('|')[0] dimv = rg.split('|')[1] dimr = rg.split('|')[2] if dimr.find(',') != -1: dictslice[dimn] = list(np.array(dimr.split(','), dtype=int)) else: dictslice[dimn] = int(dimr) # Dimension-variable if not objnc.variables.has_key(dimv): if dimv == 'WRFtime': Ddimvs.append([dimn, 'WRFtime', dimr]) else: print warnmsg print ' ' +fname+ ": file '" + filen + "' does not have " + \ "dimension-variable '" + dimv + "' !!" print ' keep going on' #quit(-1) else: # Keeping the dimensions Ddimvs.append([dimn, dimv, dimr]) # Getting values of the variable varslice, dimvarvals = SliceVarDict(varobj, dictslice) # Avoiding segmentation fault in Curie... varvals = slice_createArray(varslice, varobj.dtype) varvals[:] = varobj[tuple(varslice)] # if len(varvals.shape) > 1: ## if varvals.shape[0] != 0: ## for ix in range(varvals.shape[0]): ## varvals[ix,...] = varobj[tuple(varslice[ix,...])] ## else: ## for ix in range(varvals.shape[len(varvals.shape)-1]): ## varvals[...,ix] = varobj[tuple(varslice[...,ix])] # varvals[:] = varobj[tuple(varslice)] # else: # varvals[:] = varobj[:] # Getting values of the correspondant dimensions of the variable for iddd in range(len(Ddimvs)): DD = Ddimvs[iddd] dimn = DD[0] dimv = DD[1] dimr = DD[2] if dimv != 'WRFtime': odimv = objnc.variables[dimv] varslice, Dddv[dimn] = SliceVarDict(odimv, dictslice) Ddv[dimn] = odimv[tuple(varslice)] Dtdv[dimn] = odimv.dtype dicattrs = {} for attrn in odimv.ncattrs(): attrval = odimv.getncattr(attrn) dicattrs[attrn] = attrval Dadv[dimn] = dicattrs else: print infmsg print ' ' + fname + ": computing time-variable from 'WRF'-Time" datesv = [] wrft = objnc.variables['Times'] slicewrft, dwrfd = SliceVarDict(wrft, dictslice) fdates = wrft[tuple(slicewrft)] refdate = '19491201000000' tunitsval = 'minutes' yrref=refdate[0:4] monref=refdate[4:6] dayref=refdate[6:8] horref=refdate[8:10] minref=refdate[10:12] secref=refdate[12:14] refdateS = yrref + '-' + monref + '-' + dayref + ' ' + horref + \ ':' + minref + ':' + secref dt = fdates.shape[0] cftimes = np.zeros((dt), dtype=np.float) for it in range(dt): wrfdates = gen.datetimeStr_conversion(wrft[it,:], \ 'WRFdatetime', 'matYmdHMS') cftimes[it] = gen.realdatetime1_CFcompilant(wrfdates, \ refdate, tunitsval) tunits = tunitsval + ' since ' + refdateS Dddv[dimn] = ['Time'] Ddv[dimn] = cftimes Dtdv[dimn] = np.float Dadv[dimn] = {'standard_name': 'time', 'long_name': 'time', \ 'units': tunitsval + ' since ' + refdateS} # Getting sizes of the dimensions for dvn in dimvarvals: dimobj = objnc.dimensions[dvn] if dimobj.isunlimited(): Dds[dvn] = None else: Dds[dvn] = len(dimobj) # Checking right slice for each dimension in variable for dimn in dimvarvals: found = False if not gen.searchInlist(Dddv.keys(), dimn): if not objnc.variables.has_key(dimn): print errormsg print ' ' + fname + ": no variable '" + dimn + "' found!!" print ' for the operation the dimension-variable is ' + \ 'required but it is not in file !' print ' Could you think in adding it?' print ' current varibales:', objnc.variables.keys() quit(-1) ovar = objnc.variables[dimn] print warnmsg print ' ' + fname + ": sliced variable has dimension '" + dimn+ \ "' but there is no slicing-dimension information" print " add range for the dimension as '[dim]|[dimv]|[range]'" print ' trying now tacking all values of the dimension-variable' print " " + dimn + "'s dimensions:", ovar.dimensions print ' provided slicing dimension-variables:', Dddv.keys() Dddv[dimn] = [dimn] Ddv[dimn] = ovar[:] Dtdv[dimn] = ovar.dtype dicattrs = {} for attrn in odimv.ncattrs(): attrval = odimv.getncattr(attrn) dicattrs[attrn] = attrval Dadv[dimn] = dicattrs #quit(-1) else: # Computing using results of previous computations varvals = prevalues.copy() dimvarvals = prevdims print infmsg print ' ' + fname + ': continuing operation with previous values' print ' shape:', varvals.shape print ' dimensions:', dimvarvals # Conditions for certain operations if gen.searchInlist(opdim, opernS): opn = operN.split(',')[0] normd = gen.str_list(operN.split(',')[1], ':') usedims = list(dimvarvals) if gen.searchInlist(oprmdim, opernS): if normd[0] != 'any': normi = [] for dn in normd: normi.append(dimvarvals.index(dn)) usedims.remove(dn) normi.sort() else: normi = range(len(varvals.shape)) usedims = 'any' for iv in range(len(normi)): normi[iv] = str(normi[iv]) opern = opernS + ',' + ':'.join(dimvarvals) + ',' + \ operN.split(',')[1] + ',' + ':'.join(normi) if gen.searchInlist(opistep, opernS): opern = opernS + ',' + ':'.join(dimvarvals) + ',' + \ operN.split(',')[3] + ',' + operN.split(',')[1] usedims = list(dimvarvals) if gen.searchInlist(opnewdim, opernS): # Creation of a new dimension for the operation dimv = list(dimvarvals) idim = dimv.index(operN.split(',')[1]) newdimn = opernS + dimv[idim] print infmsg print ' ' + fname + ": operation '" + opernS + "' will create " + \ " a new dimension '" + newdimn + "' !!" dimv[idim] = newdimn dimvarvals = tuple(dimv) vardimnstep = operN.split(',')[2] istep = int(operN.split(',')[3]) if vardimnstep != 'WRFtime': if not objnc.variables.has_key[vardimnstep]: print errormsg print ' ' + fname + ": file '" + filen + "' does not " + \ "have variable '" + vardimnstep + "' to be used for '" + \ "ifreq_mean' !!" print ' available ones:', objnc.variables.keys() quit(-1) odimv = objnc.variables[vardimnstep] varslice, Dddv[newdimn] = SliceVarDict(odimv, dictslice) allvalues = odimv[tuple(varslice)] Dtdv[newdimn] = odimv.dtype dicattrs = {} for attrn in odimv.ncattrs(): attrval = odimv.getncattr(attrn) dicattrs[attrn] = attrval Dadv[newdimn] = dicattrs else: if slicewrft is None: print infmsg print ' '+fname+ ": computing time-variable from 'WRF'-Time" datesv = [] wrft = objnc.variables['Times'] slicewrft, dwrfd = SliceVarDict(wrft, dictslice) fdates = wrft[tuple(slicewrft)] refdate = '19491201000000' tunitsval = 'minutes' yrref=refdate[0:4] monref=refdate[4:6] dayref=refdate[6:8] horref=refdate[8:10] minref=refdate[10:12] secref=refdate[12:14] refdateS= yrref +'-'+ monref + '-' + dayref + ' ' + horref + \ ':' + minref + ':' + secref dt = fdates.shape[0] allvalues = np.zeros((dt), dtype=np.float) for it in range(dt): wrfdates = gen.datetimeStr_conversion(wrft[it,:], \ 'WRFdatetime', 'matYmdHMS') allvalues[it] = gen.realdatetime1_CFcompilant(wrfdates, \ refdate, tunitsval) tunits = tunitsval + ' since ' + refdateS else: # Using previous computed WRFtime allvalues = cftimes Dtdv[newdimn] = np.float Dadv[newdimn] = {'standard_name': 'time', 'long_name': 'time', \ 'units': tunitsval + ' since ' + refdateS, 'calendar': \ 'standard'} # Filling-up new dimension characteristics Dds[newdimn] = istep Dddv[newdimn] = [newdimn] newdimvals = np.zeros((istep), dtype=Dtdv[newdimn]) dsize = allvalues.shape[0] Nsteps = dsize/istep for it in range(istep): newdimvals[it] = np.mean(allvalues[slice(it,istep*Nsteps,istep)]) Ddv[newdimn] = newdimvals[:] Dadditionaldim[opernS+'@'+varn] = usedims if varn != 'continuation': Dadditionalunits[opernS+'@'+varn] = get_varunits(varobj) else: Dadditionalunits[opernS+'@'+varn] = '?' else: opern = operN if varn != 'continuation': objnc.close() # operation if prevalues is None: # Creation of an empty variable to start with if gen.searchInlist(opinizero,opernS): inimat = np.zeros(varvals.shape, dtype=varobj.dtype) elif gen.searchInlist(opinione,opernS): inimat = np.ones(varvals.shape, dtype=varobj.dtype) else: inimat = varvals prevalues, operS, addvs = gen.matoperations(opern, varvals, inimat) else: prevalues, operS, addvs = gen.matoperations(opern, prevalues, varvals) operS = ' ' + operS + '[' + varn + ']' # elif varn == 'None' and prevalues is not None: # prevalues, operS, addvs = gen.matoperations(opern, prevalues) # operS = ' ' + operS if operN == opers[0]: operationS = operS else: operationS = operationS + operS Dadditionalv[opernS+'@'+varn] = addvs prevdims = list(dimvarvals) prevsizedims = varvals.shape # Creation of the new file objofile = NetCDFFile(ofile, 'w') # Dimensions for dimn in dimvarvals: newdim = objofile.createDimension(dimn, Dds[dimn]) objofile.sync() # Variables for dimensions for dvn in dimvarvals: for idn in Dddv[dvn]: if not objofile.dimensions.has_key(idn): objofile.createDimension(idn, \ Dds[idn]) newvar = objofile.createVariable(dvn, Dtdv[dvn], tuple(Dddv[dvn])) dictatr = Dadv[dvn] for attrn in dictatr.keys(): newattr = newvar.setncattr(attrn, dictatr[attrn]) newvar[:] = Ddv[dvn] objofile.sync() # Computed variable vartype = prevalues.dtype if type(prevalues) == type(ma.asarray([1,1])): maskvalue = prevalues.fill_value newvar = objofile.createVariable(varname, nctype(vartype), dimvarvals, \ fill_value=maskvalue) else: newvar = objofile.createVariable(varname, nctype(vartype), dimvarvals) newvar[:] = prevalues[:] newattr = basicvardef(newvar, varname, Lvarname, varunits) # Additional variables. Each necessary dimension to fill up the information will be created as: # dimension name: dim + '_' + operS + '_' + var # dimension size: value from the shape of the additional values for operSvarn in Dadditionalv.keys(): addvs = Dadditionalv[operSvarn] if addvs is not None: operS = operSvarn.split('@')[0] vnS = operSvarn.split('@')[1] addopdims = Dadditionaldim[operSvarn] addunits = Dadditionalunits[operSvarn] Hvarns = DHadditionalvars[operS] iHv = 0 if type(addopdims) == type('S') and addopdims == 'any': for Hvarn in Hvarns: newvn = Hvarn + '_' + operS + '_' + vnS newvar = objofile.createVariable(newvn, 'f4') newvar[:] = addvs[iHv] Lattr = Hvarn + ' value for operation ' + operS + ' with variable ' + vnS basicvardef(newvar, newvn, Lattr, addunits) iHv = iHv + 1 else: idim = 0 newvdims = [] for dimn in addopdims: newdimn = dimn + '_' + operS + '_' + vnS dsize = addvs[0].shape[idim] newdim = objofile.createDimension(newdimn,dsize) newvdims.append(newdimn) idim = idim + 1 for Hvarn in Hvarns: newvn = Hvarn + '_' + operS + '_' + vnS newvar = objofile.createVariable(newvn, 'f4', tuple(newvdims)) newvar[:] = addvs[iHv] Lattr = Hvarn + ' value for operation ' + operS + ' with variable ' + vnS basicvardef(newvar, newvn, Lattr, addunits) iHv = iHv + 1 # Global attributes add_global_PyNCplot(objofile, main, fname, '1.1') newattr = set_attributek(objofile, 'used_operation', operationS, 'S') newattr = set_attributek(objofile, 'used_files', ', '.join(filens) ,'S') newattr = set_attributek(objofile, 'used_ranges', ', '.join(ranges) ,'S') objofile.sync() objofile.close() print ' ' + fname + ' success written of "' + ofile + '" !!!!!' return #compute_opersvarsfiles('add|control/sellonlatbox_RAINC.nc|RAINC,add|control/sellonlatbox_RAINNC.nc|RAINNC', 'PRECIP') def compute_opervartimes(values, ncfile, varname): """ Function to compute opervartimes: operation of variable for a given sub-set of time-steps (OPER1.VAR[IT,ET]), operations are going to be secuentially made values= [oper]|[timen]|[intT] dimension [oper]: operations: add,sub,mul,div,pot [timen]: name of the dimension time [intT]: number time-steps to accumulate ncfile= netCDF file to use varname= variable to use """ fname='compute_opervartimes' if values == 'h': print fname + '_____________________________________________________________' print compute_opervartimes.__doc__ quit() ofile = 'opervartimes_' + varname + '.nc' if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ' file: "' + ncfile + '" does not exist !!' quit(-1) objnc = NetCDFFile(ncfile, 'r') if not objnc.variables.has_key(varname): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varname + '" !!!!' quit(-1) opern = values.split('|')[0] timen = values.split('|')[1] intT = int(values.split('|')[2]) varobj = objnc.variables[varname] vardims = varobj.dimensions vartype = varobj.dtype Ndims = len(vardims) newdimns = vardims newdims = np.zeros((Ndims), dtype=int) newslice = [] newunits = '-' idim = 0 for dimn in vardims: if dimn == timen: dimtid = idim newslice.append(0) origdimt = varobj.shape[dimtid] newdims[idim] = len(np.arange(origdimt)[intT/2::intT]) else: newdims[idim] = varobj.shape[idim] newslice.append(slice(0,varobj.shape[idim])) idim = idim + 1 print 'dimtid: ',dimtid if opern == 'add' or opern == 'sub': newvarv = np.zeros(tuple(newdims), dtype=np.float) elif opern == 'mul' or opern == 'div' or opern == 'pot': newvarv = np.ones(tuple(newdims), dtype=np.float) else: print errormsg print ' ' + fname + ': operation "' + opern + '" is not ready !!!!' quit(-1) # Creation of the accumulated operated variable idint = 0 itt = 0 nint = 0 for it in range(1,origdimt): tslice = list(newslice) tslice[dimtid] = it tnewvar = list(newslice) tnewvar[dimtid] = nint if opern == 'add': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] + varobj[tuple(tslice)] if searchInlist(varobj.ncattrs(), 'units') and newunits != '-': newunits = varobj.getncattr('units') elif opern == 'sub': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] - varobj[tuple(tslice)] if searchInlist(varobj.ncattrs(), 'units') and newunits != '-': newunits = varobj.getncattr('units') elif opern == 'mul': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] * varobj[tuple(tslice)] elif opern == 'div': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] / varobj[tuple(tslice)] elif opern == 'pot': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] **varobj[tuple(tslice)] it = it + 1 itt = itt + 1 if itt == intT: nint = nint + 1 itt = 0 # Creation of the new file objofile = NetCDFFile(ofile, 'w') # Dimensions for dimn in newdimns: dimobj = objnc.dimensions[dimn] if dimobj.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(dimobj) if dimn == timen: if np.mod(varobj.shape[dimtid],intT) != 0: dimsize = varobj.shape[dimtid]/intT + 1 newdim = objofile.createDimension(dimn, dimsize) newdim = objofile.createDimension('bnds', 2) # Variables fvars = objnc.variables for varn in fvars: if varn != varname: varobj = objnc.variables[varn] if varn != timen: if not searchInlist(varobj.dimensions,timen): newvar = objofile.createVariable(varn, varobj.dtype, \ varobj.dimensions) for attrn in varobj.ncattrs(): attrval = varobj.getncattr(attrn) newattr = newvar.setncattr(attrn, attrval) newvar[:] = varobj[:] else: newvar = objofile.createVariable(varn, varobj.dtype, ('time')) basicvardef(newvar, 'time', 'time steps','-') times = np.arange(origdimt)[intT/2::intT] newvar[:] = times newdimt = newvar.shape[0] newattr = newvar.setncattr('bounds','time_bnds') newvar = objofile.createVariable('time_bnds', 'i4', ('time', 'bnds')) basicvardef(newvar, 'time_bnds', 'time steps bnds of accumulation','-') for it in range(newdimt): newvar[it,0] = (it - 1)*intT newvar[it,1] = (it)*intT-1 newvar = objofile.createVariable('ac' + varname, vartype, vardims) newvar[:] = newvarv newattr = basicvardef(newvar, 'ac' + varname, 'accumulated ' + opern + \ ' values' , newunits) # Global attributes for attrn in objnc.ncattrs(): attrval = objnc.getncattr(attrn) newattr = objofile.setncattr(attrn, attrval) objnc.close() objofile.sync() objofile.close() print ' ' + fname + ' success written of "' + ofile + '" !!!!!' return #compute_opervartimes('add|time|24', 'obs/PRCP.nc', 'product') # def compute_opervaralltime(values, ncfile, varname): """ Function to compute opervaralltime: operation of variable successible allover the time-steps (OPER1.VAR), operations are going to be secuentially made values= [oper]|[timen] dimension [oper]: operations: add,sub,mul,div,pot [timen]: name of the dimension time ncfile= netCDF file to use varname= variable to use """ fname='compute_opervarlltime' ofile = 'opervaralltime_' + varname + '.nc' if values == 'h': print fname + '_____________________________________________________________' print compute_opervaralltime.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ' file: "' + ncfile + '" does not exist !!' quit(-1) objnc = NetCDFFile(ncfile, 'r') if not objnc.variables.has_key(varname): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varname + '" !!!!' quit(-1) opern = values.split('|')[0] timen = values.split('|')[1] varobj = objnc.variables[varname] vardims = varobj.dimensions vartype = varobj.dtype Ndims = len(vardims) newdimns = vardims newdims = np.zeros((Ndims), dtype=int) newslice = [] prevslice = [] newunits = '-' idim = 0 for dimn in vardims: if dimn == timen: dimtid = idim newslice.append(0) origdimt = varobj.shape[dimtid] newdims[idim] = origdimt else: newdims[idim] = varobj.shape[idim] newslice.append(slice(0,varobj.shape[idim])) idim = idim + 1 print 'dimtid: ',dimtid if opern == 'add' or opern == 'sub': newvarv = np.zeros(tuple(newdims), dtype=np.float) elif opern == 'mul' or opern == 'div' or opern == 'pot': newvarv = np.ones(tuple(newdims), dtype=np.float) else: print errormsg print ' ' + fname + ': operation "' + opern + '" is not ready !!!!' quit(-1) # Creation of the accumulated operated variable idint = 0 for it in range(1,origdimt): tslice = list(newslice) tslice[dimtid] = it prevslice = list(newslice) prevslice[dimtid] = it - 1 if opern == 'add': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] + varobj[tuple(tslice)] if searchInlist(varobj.ncattrs(), 'units') and newunits != '-': newunits = varobj.getncattr('units') elif opern == 'sub': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] - varobj[tuple(tslice)] if searchInlist(varobj.ncattrs(), 'units') and newunits != '-': newunits = varobj.getncattr('units') elif opern == 'mul': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] * varobj[tuple(tslice)] elif opern == 'div': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] / varobj[tuple(tslice)] elif opern == 'pot': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] **varobj[tuple(tslice)] it = it + 1 # Creation of the new file objofile = NetCDFFile(ofile, 'w') # Dimensions for dimn in newdimns: dimobj = objnc.dimensions[dimn] if dimobj.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(dimobj) newdim = objofile.createDimension(dimn, dimsize) newdim = objofile.createDimension('bnds', 2) # Variables fvars = objnc.variables for varn in fvars: if varn != varname: varobj = objnc.variables[varn] if varn != timen: if not searchInlist(varobj.dimensions,timen): newvar = objofile.createVariable(varn, varobj.dtype, \ varobj.dimensions) for attrn in varobj.ncattrs(): attrval = varobj.getncattr(attrn) newattr = newvar.setncattr(attrn, attrval) newvar[:] = varobj[:] else: newvar = objofile.createVariable(varn, varobj.dtype, ('time')) basicvardef(newvar, 'time', 'time steps','-') timevals = varobj[:] newvar[:] = timevals newdimt = newvar.shape[0] newattr = newvar.setncattr('bounds','time_bnds') newvar = objofile.createVariable('time_bnds', 'i4', ('bnds')) basicvardef(newvar, 'time_bnds', 'time steps bnds of accumulation','-') newvar[0] = timevals[0] newvar[1] = timevals[origdimt-1] newvar = objofile.createVariable('ac' + varname, vartype, vardims) newvar[:] = newvarv newattr = basicvardef(newvar, 'ac' + varname, 'accumulated ' + opern + \ ' values' , newunits) # Global attributes for attrn in objnc.ncattrs(): attrval = objnc.getncattr(attrn) newattr = objofile.setncattr(attrn, attrval) objnc.close() objofile.sync() objofile.close() print ' ' + fname + ' success written of "' + ofile + '" !!!!!' return #compute_opervaralltime('add|time', 'obs/PRCP.nc', 'product') def setvar_asciivalues(values, ncfile, varname): """ Function to set de values of a variable with an ASCII file (common Fortran-like format) values= asciifile: ASCII file with a Fortran-like structure ('#' for commentaries) [dim1]_1 [[dim2]_1 ... [dim2]_M] ... [dim1]_N NOTE: for character values, use '!' for space ncfile= netCDF file to use varname= variable to use """ fname='setvar_asciivalues' if values == 'h': print fname + '_____________________________________________________________' print setvar_asciivalues.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ' file: "' + ncfile + '" does not exist !!' quit(-1) if not os.path.isfile(values): print errormsg print ' ' + fname + ' ASCII file: "' + values + '" does not exist !!' quit(-1) objnc = NetCDFFile(ncfile, 'a') if not objnc.variables.has_key(varname): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varname + '" !!!!' quit(-1) objvar = objnc.variables[varname] vardims = objvar.shape vartype = objvar.dtype Ndims = len(vardims) objasciif = open(values, 'r') values = np.zeros((objvar.shape), dtype=vartype) it = 0 if Ndims == 1: valfinal = np.zeros((1), dtype=vartype) for line in objasciif: if line[0:1] != '#': val = (line.replace('\n','').replace(' ','').replace('\t','')) if vartype != '|S1': valfinal[0] = gen.retype(val, vartype) if it <= values.shape[0]-1: values[it] = gen.retype(valfinal[0], vartype) it = it +1 else: for i1 in range(len(val)): values[it] = val.replace('!',' ') elif Ndims == 2: iline=0 for line in objasciif: if line[0:1] != '#': vals = line.replace('\n','').replace('\t','').split(' ') if vartype != '|S1': if len(vals) != values.shape[1]: print errormsg print ' ' + fname + ': given: ',len(vals),' but variable '+ \ 'requires: ',values.shape[1],'!!!' exit(-1) for i1 in range(len(vals)): values[iline,i1] = gen.retype(vals[i1], vartype) else: for i1 in range(len(vals)): values[iline,i1] = vals[i1].replace('!',' ') iline=iline+1 elif Ndims == 3: d0 = objvar.shape[0] d1 = objvar.shape[1] d2 = objvar.shape[2] iline=0 icol=0 for line in objasciif: if line[0:1] != '#' and len(line) > 2: vals = line.replace('\n','').replace('\t','').split(' ') if vals.count('') != 0: vals.remove('') if vartype != '|S1': if len(vals) != values.shape[2]: print vals print errormsg print ' ' + fname + ': given: ',len(vals),' but variable '+ \ 'requires: ',values.shape[2],'!!!' exit(-1) for i1 in range(len(vals)): values[icol,iline,i1] = gen.retype(vals[i1], vartype) else: for i1 in range(len(vals)): values[icol,iline,i1] = vals[i1].replace('!',' ') iline=iline+1 if iline > d1-1: iline = 0 icol = icol + 1 else: print errormsg print ' ' + fname + ': number of dimensions', Ndims, 'not ready !!!!' quit(-1) objasciif.close() objvar[:] = values objnc.sync() objnc.close() return #setvar_asciivalues('obstimes.inf', 'obs/re_project_accumulated-1h_PRCP.nc', 'time') def statcompare_files(values): """ Python script to statistically compare two different files values= [file1]:[var1],[file2][var2] [file1/2]: files to compare any diferent would be [file1]-[file2] [var1/2]: variable in file1/2 to compare """ fname = 'statcompare_files' ofile='statcompare_files.nc' if values == 'h': print fname + '_____________________________________________________________' print statcompare_files.__doc__ quit() file1=values.split(',')[0].split(':')[0] var1=values.split(',')[0].split(':')[1] file2=values.split(',')[1].split(':')[0] var2=values.split(',')[1].split(':')[1] if not os.path.isfile(file1): print errormsg print ' ' + fname + ' file: "' + file1 + '" does not exist !!' quit(-1) objnc1 = NetCDFFile(file1, 'r') if not objnc1.variables.has_key(var1): print errormsg print ' ' + fname + ': netCDF file "' + file1 + \ '" does not have variable "' + var1 + '" !!!!' quit(-1) if not os.path.isfile(file2): print errormsg print ' ' + fname + ' file: "' + file2 + '" does not exist !!' quit(-1) objnc2 = NetCDFFile(file2, 'r') if not objnc2.variables.has_key(var2): print errormsg print ' ' + fname + ': netCDF file "' + file2 + \ '" does not have variable "' + var2 + '" !!!!' quit(-1) objvar1 = objnc1.variables[var1] objvar2 = objnc2.variables[var2] Ndim1 = len(objvar1.shape) Ndim2 = len(objvar2.shape) if Ndim1 != Ndim2: print errormsg print ' ' + fname + ' variable: "' + var1 + '" from file "' + file1 + \ '" does not have the same size:' , Ndim1, 'as variable: "' + var2 + \ '" from file "' + file2 + '": ',Ndim2 quit(-1) dims1 = objvar1.shape dims2 = objvar2.shape for iid in range(Ndim1): if dims1[iid] != dims2[iid]: print errormsg print ' ' + fname + ' shape:',iid,'is different between variables!!!' print ' dim_var1:', dims1[iid], 'dim_var2:', dims2[iid] print ' var1 shape:', dims1 print ' var2 shape:', dims2 quit(-1) varvals1 = objvar1[:] varvals2 = objvar2[:] if Ndim1 == 3: timecompare = stats_time2Dvars(varvals1,varvals2) spacecompare = stats_space2Dvars(varvals1,varvals2) dimt = varvals1.shape[0] dimy = varvals1.shape[1] dimx = varvals1.shape[2] # print 'bias std.dev. ___________________________________' # for it in range(dimt): # print it,':',spacecompare.bias[it],spacecompare.stdv1Sv2[it] # print 'global spatial correlation (% significance)______' # for it in range(dimt): # print it, ':', spacecompare.corr[it], '(', \ # (1.-spacecompare.p_value[it])*100.,')' # print 'spatial means temporal correlation:',spacecompare.meancorr,'(', \ # (1.-spacecompare.meanp_value)*100.,' % significance)' # print 'temporal means temporal correlation:',timecompare.meancorr,'(', \ # (1.-timecompare.meanp_value)*100.,' % significance)' # print 'temporal bias map________________________________' # print timecompare.meanv1Sv2 # print 'temporal correlation map_________________________' # print timeecompare.corr else: print errormsg print ' ' + fname + ' numbe of dimensions:' + Ndim1 + ' not ready!!!' quit(-1) # Creation of results file ## objofile = NetCDFFile(ofile, 'w') # Dimensions for dimn in objnc1.dimensions: dimobj = objnc1.dimensions[dimn] if dimobj.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(dimobj) newdim = objofile.createDimension(dimn, dimsize) # Variables fvars = objnc1.variables for varn in fvars: if varn != var1: varobj = objnc1.variables[varn] if varn != var1: newvar = objofile.createVariable(varn, varobj.dtype, \ varobj.dimensions) for attrn in varobj.ncattrs(): attrval = varobj.getncattr(attrn) newattr = newvar.setncattr(attrn, attrval) newvar[:] = varobj[:] # Statistical variables vartype = objvar1.dtype varunits = objvar2.getncattr('units') print 'vartype=',vartype # temporal bias newvar = objofile.createVariable('t_bias_'+var1+'_'+var2, vartype, ('y', 'x'), \ fill_value = fillValue) newvar[:] = timecompare.bias newattr = basicvardef(newvar, 't_bias_'+var1+'_'+var2, 'temporal bias between ' +\ var1 + '&' + var2, varunits) # temporal Standard deviation newvar = objofile.createVariable('t_stddev_'+var1+'_'+var2, vartype, ('y', 'x'), \ fill_value = fillValue) newvar[:] = timecompare.stdv1Sv2 newattr = basicvardef(newvar, 't_stddev_'+var1+'_'+var2, \ 'temporal standard deviation between '+ var1 + '&' + var2, varunits) # space bias newvar = objofile.createVariable('s_bias_'+var1+'_'+var2, vartype, ('time'), \ fill_value = fillValue) newvar[:] = spacecompare.bias newattr = basicvardef(newvar, 's_bias_'+var1+'_'+var2, 'spatial bias between ' + \ var1 + '&' + var2, varunits) # space Standard deviation newvar = objofile.createVariable('s_stddev_'+var1+'_'+var2, vartype, ('time'), \ fill_value = fillValue) newvar[:] = spacecompare.stdv1Sv2 newattr = basicvardef(newvar, 's_stddev_'+var1+'_'+var2, \ 'spatial standard deviation between '+ var1 + '&' + var2, varunits) # t Correlation map newvar = objofile.createVariable('t_corr_'+var1+'_'+var2, vartype, ('y','x'), \ fill_value = fillValue) newvar[:] = timecompare.corr newattr = basicvardef(newvar, 't_corr_'+var1+'_'+var2, \ 'temporal correlation between ' + var1 + '&' + var2, '-') newattr = newvar.setncattr('projection', 'lon lat') # t p-value map newvar = objofile.createVariable('t_p_value_'+var1+'_'+var2, vartype, ('y','x'), \ fill_value = fillValue) newvar[:] = timecompare.p_value newattr = basicvardef(newvar, 't_p_value_'+var1+'_'+var2, \ 'temporal p_value between ' + var1 + '&' + var2, '-') newattr = newvar.setncattr('projection', 'lon lat') # s Correlation map newvar = objofile.createVariable('s_corr_'+var1+'_'+var2, vartype, ('time'), \ fill_value = fillValue) newvar[:] = spacecompare.corr newattr = basicvardef(newvar, 's_corr_'+var1+'_'+var2, \ 'spatial correlation between ' + var1 + '&' + var2, '-') # s p-value map newvar = objofile.createVariable('s_p_value_'+var1+'_'+var2, vartype, ('time'), \ fill_value = fillValue) newvar[:] = spacecompare.p_value newattr = basicvardef(newvar, 's_p_value_'+var1+'_'+var2, \ 'spatial p_value between ' + var1 + '&' + var2, '-') # print 'spatial means temporal correlation:',spacecompare.meancorr,'(', \ # (1.-spacecompare.meanp_value)*100.,' % significance)' # t mean Correlation newvar = objofile.createVariable('t_mean_corr_'+var1+'_'+var2, 'f4') newvar[:] = timecompare.meancorr newattr = basicvardef(newvar, 't_mean_corr_'+var1+'_'+var2, \ 'time mean values correlation between ' + var1 + '&' + var2, '-') # t mean p-value newvar = objofile.createVariable('t_mean_p_value_'+var1+'_'+var2, 'f4') newvar[:] = timecompare.meanp_value newattr = basicvardef(newvar, 't_mean_p_value_'+var1+'_'+var2, \ 'time mean p_value between ' + var1 + '&' + var2, '-') # s mean Correlation newvar = objofile.createVariable('s_mean_corr_'+var1+'_'+var2, 'f4') newvar[:] = spacecompare.meancorr newattr = basicvardef(newvar, 's_mean_corr_'+var1+'_'+var2, \ 'space mean values correlation between ' + var1 + '&' + var2, '-') # s mean p-value newvar = objofile.createVariable('s_mean_p_value_'+var1+'_'+var2, 'f4') newvar[:] = spacecompare.meanp_value newattr = basicvardef(newvar, 's_mean_p_value_'+var1+'_'+var2, \ 'space mean p_value between ' + var1 + '&' + var2, '-') # Global attributes for attrn in objnc1.ncattrs(): attrval = objnc1.getncattr(attrn) newattr = objofile.setncattr(attrn, attrval) newattr = objofile.setncattr('statisitcs', 'variables retrieved from files ' + \ file1 + ' & ' + file2) objnc1.close() objnc2.close() objofile.sync() objofile.close() varsfillvalue = ['t_bias_', 't_stddev_', 's_bias_', 's_stddev_', 't_corr_', \ 't_p_value', 's_corr_', 's_p_value'] # for varn in varsfillvalue: # print varn # fill = varaddattrk('_FillValue|'+ str(fillValue) + '|R32', ofile, \ # varn+var1+'_'+var2) print ' ' + fname + ': successfull generation of file "' + ofile + '" !!!!!' return #statcompare_files('control/sellonlatbox_wss_17-20.nc:wss,obs/re_project_Vu_17-20.nc:wss') def sellonlatlevbox(values, ncfile, varn): """ Function to select a lotlan box and a given level from a data-set values= [lonName],[latName],[lonSW],[latSW],[lonNE],[latNE],[levi],[levf] [lonName]: name of the variable with the longitudes [latName]: name of the variable with the latitudes [levName]: name of the variable with the value of the levels [lonSW],[latSW]: longitude and latitude of the SW vertex of the box [lonNE],[latNE]: longitude and latitude of the NE vertex of the box [levI],[levF]: range of levels to retrieve ncfile= netCDF file varn= name of the variable """ fname = 'sellonlatlevbox' ofile = 'sellonlatlevbox_' + varn + '.nc' if values == 'h': print fname + '_____________________________________________________________' print sellonlatlevbox.__doc__ quit() lonn = values.split(',')[0] latn = values.split(',')[1] levn = values.split(',')[2] lonSW = np.float(values.split(',')[3]) latSW = np.float(values.split(',')[4]) lonNE = np.float(values.split(',')[5]) latNE = np.float(values.split(',')[6]) levI = int(values.split(',')[7]) levF = int(values.split(',')[8]) objfile = NetCDFFile(ncfile, 'r') lonobj = objfile.variables[lonn] latobj = objfile.variables[latn] levobj = objfile.variables[levn] lonv = lonobj[:] latv = latobj[:] levv = levobj[:] if not searchInlist(objfile.variables, varn): print errormsg print ' ' + fname + ': variable name "' + varn + '" is not in file "' + \ ncfile + '" !!!!!' quit(-1) varobj = objfile.variables[varn] Ndimslon = len(lonobj.shape) Ndimslat = len(latobj.shape) Ndimslev = len(levobj.shape) Ndimsvar = len(varobj.shape) # Looking for coincidence of dimensions samedim = [] for idv in range(Ndimsvar): for idl in range(Ndimslon): if varobj.dimensions[idv] == lonobj.dimensions[idl]: if not searchInlist(samedim,varobj.dimensions[idv]): samedim.append(varobj.dimensions[idv]) break for idl in range(Ndimslat): if varobj.dimensions[idv] == latobj.dimensions[idl]: if not searchInlist(samedim,varobj.dimensions[idv]): samedim.append(varobj.dimensions[idv]) break Ndimshare = len(samedim) print 'variable and lon/lat matrices share ', Ndimshare,' dimensions: ',samedim samedimslonlat = True for idL in range(Ndimslon): for idl in range(Ndimslat): if lonobj.dimensions[idl] != latobj.dimensions[idl]: samedimslonlat = False break if not samedimslonlat: break # Creation of the lon/lat matrices to search if Ndimshare == 1: lonmat = lonobj[:] latmat = latobj[:] elif Ndimshare == 2: if samedimslonlat: lonmat = lonobj[:] latmat = latobj[:] else: if Ndimslon != 1 and Ndimslat != 1: print errormsg print ' ' + fname + ': I do not know how to keep going!' print ' lon ', Ndimslon,' and lat ',Ndimslat, \ ' matrices do not share the same dimensions, and do not have shape 1' quit(-1) else: lonmat = np.zeros((latobj.shape[0], lonobj.shape[0]), dtype=np.float) latmat = np.zeros((latobj.shape[0], lonobj.shape[0]), dtype=np.float) for i in range(lonobj.shape[0]): latmat[:,i] = latobj[:] for j in range(latobj.shape[0]): lonmat[j,:] = lonobj[:] elif Ndimshare == 3: if samedimslonlat: lonmat = lonobj[0,:,:] latmat = latobj[0,:,:] else: print errormsg print ' ' + fname + ': dimension sharing of lon/lat not ready!' quit(-1) # Searching for inside points iind = 0 if Ndimshare == 1: inside = {} for iL in range(lonobj.shape[0]): if lonobj[iL] >= lonSW and lonobj[iL] <= lonNE and latobj[iL] >= latSW \ and latobj[iL] <= latNE: inside[iind] = iL iind = iind + 1 elif Ndimshare == 2: newidx = [] newidy = [] newidz = [] if (len(lonobj.shape) == 3): inside3D = np.zeros((varobj.shape[1], lonobj.shape[1], lonobj.shape[2]), dtype=bool) inside2D = np.zeros((lonobj.shape[1], lonobj.shape[2]), dtype=bool) insideZ = np.zeros((varobj.shape[1]), dtype=bool) else: inside3D = np.zeros((varobj.shape[1], lonobj.shape), dtype=bool) inside2D = np.zeros((lonobj.shape), dtype=bool) insideZ = np.zeros((varobj.shape[1]), dtype=bool) for iL in range(lonobj.shape[1]): for il in range(lonobj.shape[0]): if lonobj[il,iL] >= lonSW and lonobj[il,iL] <= lonNE and \ latobj[il,iL] >= latSW and latobj[il,iL] <= latNE: inside2D[il, iL] = True if not searchInlist(newidx, iL): newidx.append(iL) if not searchInlist(newidy, il): newidy.append(il) for iz in range(Ndimslev): if iz >= levI and iz <= levF: if not searchInlist(newidz, iz): newidz.append(iz) inside3D[iz,il, iL] = True insideZ[iz] = True iind = iind + 1 elif Ndimshare == 3: newidx = [] newidy = [] newidz = [] inside3D = np.zeros((varobj.shape[1], lonobj.shape[1], lonobj.shape[2]), dtype=bool) inside2D = np.zeros((lonobj.shape[1], lonobj.shape[2]), dtype=bool) insideZ = np.zeros((varobj.shape[1]), dtype=bool) for iL in range(lonobj.shape[2]): for il in range(lonobj.shape[1]): if lonobj[0,il,iL] >= lonSW and lonobj[0,il,iL] <= lonNE and \ latobj[0,il,iL] >= latSW and latobj[0,il,iL] <= latNE: inside2D[il, iL] = True if not searchInlist(newidx, iL): newidx.append(iL) if not searchInlist(newidy, il): newidy.append(il) for iz in range(Ndimslev): if iz >= levI and iz <= levF: if not searchInlist(newidz, iz): newidz.append(iz) inside3D[iz, il, iL] = True insideZ[iz] = True iind = iind + 1 Ninpts3D = len(inside3D) Ninpts2D = len(inside2D) NinptsZ = len(insideZ) newdx = len(newidx) newdy = len(newidy) newdz = len(newidz) print ' ' + fname + ': ',Ninpts3D,' pts found in the box: ',lonSW,',',latSW,',', \ levI,' x ', lonNE,',',latNE,',',levF print ' ' + fname + ': dimensions output matrix:', newdz, ',', newdy, ',', newdx # Creation of cropped matrices Ninlev = levF - levI + 1 inlon = np.zeros((Ninpts2D), dtype=np.float) inlat = np.zeros((Ninpts2D), dtype=np.float) inlev = np.zeros((NinptsZ), dtype=np.float) invar = np.zeros((varobj.shape[0],Ninpts3D), dtype=np.float) # Creation of the netCDF file ## objofile = NetCDFFile(ofile, 'w') if Ndimshare == 1: # Dimensions newdim = objofile.createDimension('boxpt', Ninpts3D) newdim = objofile.createDimension('time', None) # var dimensions newvar = objofile.createVariable('lon', 'f8', ('boxpt')) newattr = basicvardef(newvar, 'lon', 'longitude', 'degrees west_east') newvar[:] = lonobj[inside2D[iin]] newvar = objofile.createVariable('lat', 'f8', ('boxpt')) newattr = basicvardef(newvar, 'lat', 'latitude', 'degrees north_south') newvar[:] = latobj[inside2D[iin]] newvar = objofile.createVariable('lev', 'f8', ('NinptsZ')) newattr = basicvardef(newvar, 'lev', 'level', '-') newvar[:] = levobj[insideZ[iin]] newvar = objofile.createVariable('time', 'f8', ('time')) if objfile.variables.has_key('time'): otime = objfile.variables['time'] timevals = otime[:] if searchInlist(otime.ncattrs(),'units'): tunits = otime.getncattr['units'] else: tunits = 'unkown' else: timevals = np.arange(varobj.shape[0]) tunits = 'unkown' newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = timevals # variable newvar = objofile.createVariable(varn, 'f4', ('time', 'boxpt')) newvar[:] = varobj[invar] else: # Dimensions newdim = objofile.createDimension('x', newdx) newdim = objofile.createDimension('y', newdy) newdim = objofile.createDimension('z', newdz) newdim = objofile.createDimension('time', None) # var dimensions newvar = objofile.createVariable('lon', 'f8', ('y', 'x')) if Ndimshare == 2: Vals = lonobj[:] else: Vals = lonobj[0,:,:] for it in range(varobj.shape[0]): newvar[:] = Vals[inside2D].reshape(newdy,newdx) newattr = basicvardef(newvar, 'lon', 'longitude', 'degrees west_east') newvar = objofile.createVariable('lat', 'f8', ('y', 'x')) newattr = basicvardef(newvar, 'lat', 'latitude', 'degrees north_south') if Ndimshare == 2: Vals = latobj[:] else: Vals = latobj[0,:,:] for it in range(varobj.shape[0]): newvar[:] = Vals[inside2D].reshape(newdy,newdx) newvar = objofile.createVariable('lev', 'f8', ('z')) newattr = basicvardef(newvar, 'lev', 'level', '-') if len(levobj.shape) == 2: newvar[:] = levobj[0,insideZ] else: newvar[:] = levobj[insideZ] newvar = objofile.createVariable('time', 'f8', ('time')) if objfile.variables.has_key('time'): otime = objfile.variables['time'] timevals = otime[:] if searchInlist(otime.ncattrs(),'units'): tunits = otime.getncattr['units'] else: tunits = 'unkown' else: timevals = np.arange(varobj.shape[0]) tunits = 'unkown' newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = timevals # variable newvar = objofile.createVariable(varn, 'f4', ('time', 'z', 'y', 'x')) for it in range(varobj.shape[0]): valsvar = varobj[it,:,:,:] newvar[it,:,:,:] = valsvar[inside3D] if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = varn if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = varn if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = 'unkown' newattr = basicvardef(newvar, vsname, vlname, vunits) # global attributes objofile.setncattr('author', 'L. Fita') objofile.setncattr('institution', 'Laboratire de Meteorologie Dynamique') objofile.setncattr('university', 'Pierre Marie Curie - Jussieu') objofile.setncattr('center', 'Centre National de Recherches Scientifiques') objofile.setncattr('city', 'Paris') objofile.setncattr('country', 'France') objofile.setncattr('script', 'nc_var_tools.py') objofile.setncattr('function', 'sellonlatlevbox') objofile.setncattr('version', '1.0') objfile.close() objofile.sync() objofile.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return #sellonlatlevbox('XLONG,XLAT,ZNU,-12.4,25.35,32.4,52.65,0,0', 'control/wrfout_d01_2001-11-09_00:00:00', 'P') def compute_tevolboxtraj(values, ncfile, varn): """ Function to compute tevolboxtraj: temporal evolution at a given point along a box following a trajectory values= [trajfile]@[Tbeg],[lonname],[latname],[zname],[timename],[timekind],[boxsize],[circler] [trajfile]: ASCII file with the trajectory ('#' not readed) [time] [xpoint] [ypoint] [Tbeg]: equivalent first time-step of the trajectory within the netCDF file [lonname],[latname],[zname],[timename]: longitude, latitude, z and time variables names [timekind]: kind of time 'cf': cf-compilant 'wrf': WRF kind [boxsize]: size in grid points of the box (square centered, better even number!) [circler]: radius in grid points of a centerd circle (as 2*circler+1) ncfile= netCDF file to use varn= ',' list of variables' name ('all', for all variables) EMERGENCY version, assuming 3D [time,lat,lon] variable ! """ import numpy.ma as ma import subprocess as sub fname='compute_tevolboxtraj' if values == 'h': print fname + '_____________________________________________________________' print compute_tevolboxtraj.__doc__ quit() arguments = '[trajfile]@[Tbeg],[lonname],[latname],[zname],[timename],' + \ '[timekind],[boxsize],[circler]]' check_arguments(fname,values,arguments,',') trajfile = values.split(',')[0].split('@')[0] Tbeg = int(values.split(',')[0].split('@')[1]) lonn = values.split(',')[1] latn = values.split(',')[2] zn = values.split(',')[3] timn = values.split(',')[4] timekind = values.split(',')[5] boxs = int(values.split(',')[6]) circler = int(values.split(',')[7]) if np.mod(boxs,2) != 1: print errormsg print ' ' + fname + ': box size:', boxs, 'is not even !!!' quit(-1) box2 = int(boxs/2) if not os.path.isfile(ncfile): print errormsg print ' ' + fname + " netCDF file: '" + ncfile + "' does not exist !!" quit(-1) if not os.path.isfile(trajfile): print errormsg print ' ' + fname + " trajectory file: '" + trajfile + "' does not exist !!" quit(-1) # circle values # radius = np.float(rboxs.split('#')[0]) # ddist = np.float(rboxs.split('#')[1]) # if radius < ddist: # print errormsg # print ' ' + fname + ': radius of the circle:',radius,' is smaller than ' + \ # 'the distance between grid points !!' # quit(-1) # else: # Nrad = int(radius/ddist) Nrad = int(circler) objfile = NetCDFFile(ncfile, 'r') lonobj = objfile.variables[lonn] latobj = objfile.variables[latn] timobj = objfile.variables[timn] if varn.find(',') != -1: varns = varn.split(',') else: if varn == 'all': varns = objfile.variables else: varns = [varn] lonv, latv = lonlat2D(lonobj[:],latobj[:]) dimx = lonv.shape[1] dimy = lonv.shape[0] timv = timobj[:] # Selecting accordingly a trajectory ## Ttraj = file_nlines(trajfile,'#') if timekind == 'wrf': dimt = objfile.variables[timn].shape[0] else: dimt = objfile.variables[timn].shape[0] if Tbeg + Ttraj > dimt: print errormsg print ' ' + fname + ': trajectory has ', Ttraj, ' time steps and starts ' + \ ' at',Tbeg,' and data',dimt, '" !!!!!' quit(-1) print ' ' + fname + ': Number of time-steps in trajectory file: ',Ttraj, 'dimt',dimt trajobj = open(trajfile,'r') # Trajectory values/slices ## iline = 0 it = 0 xrangeslice = [] yrangeslice = [] xrangeslice2D = [] yrangeslice2D = [] cxrangeslice = [] cyrangeslice = [] cxrangeslice2D = [] cyrangeslice2D = [] cslicev = [] cslice2D = [] cslicevnoT = [] gtrajvals = np.zeros((Ttraj,3), dtype=int) trajvals = np.zeros((Ttraj,3), dtype=np.float) circdist = np.zeros((Ttraj, dimy, dimx), dtype=np.float) it = 0 iline = 0 for line in trajobj: ## Skipping first line ## if not iline == 0: ## it = iline - 1 ## it = iline # Slicing brings to reduce 1 time-step.... ??? if line[0:1] != '#': gtrajvals[it,0] = Tbeg + iline - 1 gtrajvals[it,1] = int(line.split(' ')[1]) gtrajvals[it,2] = int(line.split(' ')[2]) # print it,'t:',gtrajvals[it,0],'x y:', gtrajvals[it,1], gtrajvals[it,2] if timekind == 'wrf': gdate = datetimeStr_conversion(timv[gtrajvals[it,0]],'WRFdatetime', \ 'matYmdHMS') trajvals[it,0] = realdatetime1_CFcompilant(gdate, '19491201000000', \ 'hours') tunits = 'hours since 1949-12-01 00:00:00' elif timekind == 'cf': trajvals[it,0] = timv[gtrajvals[it,0]] tunits = timobj.getncattr('units') else: print errormsg print ' ' + fname + ' time kind: "' + timekind + '" not ready !!' quit(-1) trajvals[it,1] = lonv[gtrajvals[it,2],gtrajvals[it,1]] trajvals[it,2] = latv[gtrajvals[it,2],gtrajvals[it,1]] # print iline, it,'time:',trajvals[it,0],'lon lat:', trajvals[it,1], \ # trajvals[it,2] # Assuming time as the first dimension in a fortran like way (t=0 as 1) # trajcut[0] = tstept - 1 # Assuming time as the first dimension in a C/python like way (t=0) # box values if gtrajvals[it,2]-box2 < 0 or gtrajvals[it,2]+box2 + 1 > dimy + 1 \ or gtrajvals[it,1]-box2 < 0 or gtrajvals[it,1]+box2 + 1 > dimx + 1: varvalst = np.ones((boxs, boxs), dtype=np.float)*fillValue if gtrajvals[it,2]-box2 < 0: yinit = 0 yinit2D = box2 - gtrajvals[it,2] else: yinit = gtrajvals[it,2]-box2 yinit2D = 0 if gtrajvals[it,2]+box2 + 1 > dimy: yend = dimy - 1 yend2D = dimy - 1 - gtrajvals[it,2] + box2 else: yend = gtrajvals[it,2]+box2 + 1 yend2D = boxs if gtrajvals[it,1]-box2 < 0: xinit = 0 xinit2D = box2 - gtrajvals[it,1] else: xinit = gtrajvals[it,1]-box2 xinit2D = 0 if gtrajvals[it,1]+box2 + 1 > dimx: xend = dimx - 1 xend2D = dimx - 1 - gtrajvals[it,1] - box2 else: xend = gtrajvals[it,1]+box2 + 1 xend2D = boxs yrangeslice.append([yinit, yend]) xrangeslice.append([xinit, xend]) yrangeslice2D.append([yinit2D, yend2D]) xrangeslice2D.append([xinit2D, xend2D]) else: yrangeslice.append([gtrajvals[it,2]-box2, gtrajvals[it,2]+box2 + 1]) xrangeslice.append([gtrajvals[it,1]-box2, gtrajvals[it,1]+box2 + 1]) yrangeslice2D.append([gtrajvals[it,2]-box2, gtrajvals[it,2]+box2 + 1]) xrangeslice2D.append([gtrajvals[it,1]-box2, gtrajvals[it,1]+box2 + 1]) # circle values circdist[it,:,:] = radius_dist(dimy,dimx,gtrajvals[it,2],gtrajvals[it,1]) if gtrajvals[it,2]-Nrad < 0 or gtrajvals[it,2]+Nrad + 1 > dimy + 1 \ or gtrajvals[it,1]-Nrad < 0 or gtrajvals[it,1]+Nrad + 1 > dimx + 1: if gtrajvals[it,2]-Nrad < 0: yinit = 0 yinit2D = Nrad - gtrajvals[it,2] else: yinit = gtrajvals[it,2]-Nrad yinit2D = 0 if gtrajvals[it,2]+Nrad + 1 > dimy: yend = dimy - 1 yend2D = dimy - 1 - gtrajvals[it,2] + Nrad else: yend = gtrajvals[it,2]+Nrad + 1 yend2D = 2*Nrad+1 if gtrajvals[it,1]-Nrad < 0: xinit = 0 xinit2D = Nrad - gtrajvals[it,1] else: xinit = gtrajvals[it,1]-Nrad xinit2D = 0 if gtrajvals[it,1]+Nrad + 1 > dimx: xend = dimx - 1 xend2D = dimx - 1 - gtrajvals[it,1] - Nrad else: xend = gtrajvals[it,1]+Nrad + 1 xend2D = 2*Nrad+1 cyrangeslice.append([yinit, yend]) cxrangeslice.append([xinit, xend]) cyrangeslice2D.append([yinit2D, yend2D]) cxrangeslice2D.append([xinit2D, xend2D]) else: cyrangeslice.append([gtrajvals[it,2]-Nrad, gtrajvals[it,2]+Nrad + 1]) cxrangeslice.append([gtrajvals[it,1]-Nrad, gtrajvals[it,1]+Nrad + 1]) cyrangeslice2D.append([gtrajvals[it,2]-Nrad, gtrajvals[it,2]+Nrad+1]) cxrangeslice2D.append([gtrajvals[it,1]-Nrad, gtrajvals[it,1]+Nrad+1]) it = it + 1 iline = iline + 1 trajobj.close() # Creation of the netCDF file ## if varn.find(',') != -1: varnS = 'multi-var' else: varnS = varn.replace(',','-') ofile = 'tevolboxtraj_' + varnS + '.nc' objofile = NetCDFFile(ofile, 'w') # Dimensions newdim = objofile.createDimension('x', boxs) newdim = objofile.createDimension('y', boxs) newdim = objofile.createDimension('xr', Nrad*2+1) newdim = objofile.createDimension('yr', Nrad*2+1) newdim = objofile.createDimension('time', None) stsn = ['min', 'max', 'mean', 'mean2', 'stdev', 'ac'] vstlname = ['minimum value within', 'maximum value within', 'mean value within', \ 'squared mean value within', 'standard deviation value within', \ 'accumulated value within'] Nsts = len(stsn) statnames = [] cstatnames = [] for i in range(Nsts): statnames.append(stsn[i] + 'box') cstatnames.append(stsn[i] + 'circle') # Getting values ## ivar = 0 for vn in varns: vnst = variables_values(vn)[0] if not searchInlist(objfile.variables, vn): print errormsg print ' ' + fname + ": variable name '" + vn + "' is not in file " + \ ncfile + '" !!!!!' quit(-1) varobj = objfile.variables[vn] Nvardims = len(varobj.shape) print ' ' + fname + ": getting '" + vn + "' ... .. ." slicev = [] slice2D = [] slicevnoT = [] cslicev = [] cslice2D = [] cslicevnoT = [] if Nvardims == 4: # Too optimistic, but at this stage... if not objofile.dimensions.has_key('z'): varzobj = objfile.variables[zn] if len(varzobj.shape) == 1: dimz = varzobj.shape zvals = varzobj[:] elif len(varzobj.shape) == 2: dimz = varzobj.shape[1] zvals = varzobj[0,:] elif len(varzobj.shape) == 3: dimz = varzobj.shape[2] zvals = varzobj[0,0,:] objofile.createDimension('z', dimz) newvar = objofile.createVariable(zn, 'f4', ('z'),fill_value=fillValue) if searchInlist(varzobj.ncattrs(),'standard_name'): vsname = varzobj.getncattr('standard_name') else: vsname = variables_values(zn)[1] if searchInlist(varzobj.ncattrs(),'long_name'): vlname = varzobj.getncattr('long_name') else: vlname = variables_values(zn)[4].replace('|',' ') if searchInlist(varzobj.ncattrs(),'units'): vunits = varzobj.getncattr('units') else: vunits = variables_values(zn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newvar[:] = zvals varvals = np.ones(tuple([Ttraj,dimz,boxs,boxs]), dtype=np.float) lonvals = np.ones(tuple([Ttraj,dimz,boxs,boxs]), dtype=np.float) latvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) rvarvals = np.ones(tuple([Ttraj,dimz,Nrad*2+1,Nrad*2+1]), dtype=np.float) rlonvals = np.ones(tuple([Ttraj,dimz,Nrad*2+1,Nrad*2+1]), dtype=np.float) rlatvals = np.ones(tuple([Ttraj,dimz,Nrad*2+1,Nrad*2+1]), dtype=np.float) # One dimension plus for the values at the center of the trajectory statvarvals = np.ones(tuple([Ttraj,dimz,Nsts+1]), dtype=np.float) rstatvarvals = np.ones(tuple([Ttraj,dimz,Nsts+1]), dtype=np.float) for it in range(Ttraj): it0 = Tbeg + it slicev = [] slice2D = [] slicevnoT = [] cslicev = [] cslice2D = [] cslice2Dhor = [] cslicevnoT = [] cslicevnoThor = [] slicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-box2 < 0 or gtrajvals[it,2]+box2 + 1 > dimy + 1 \ or gtrajvals[it,1]-box2 < 0 or gtrajvals[it,1]+box2 + 1 > dimx +1: # box values slicev.append(slice(0,dimz)) slicev.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicev.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slicevnoT.append(slice(0,dimz)) slicevnoT.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicevnoT.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slice2D.append(slice(0,dimz)) slice2D.append(slice(0,yrangeslice[it][1]-yrangeslice[it][0])) slice2D.append(slice(0,xrangeslice[it][1]-xrangeslice[it][0])) rvarvalst = np.ones((dimz, Nrad*2+1, Nrad*2+1),dtype=np.float)* \ fillValue varvalst[tuple(slice2D)] = varobj[tuple(slicev)] varvals[it,:,:,:] = varvalst # box stats values maskedvals = ma.masked_values (varvalst, fillValue) maskedvals2 = maskedvals*maskedvals for iz in range(dimz): statvarvals[it,iz,0] = varvalst[iz,box2,box2] statvarvals[it,iz,1] = np.min(varvalst[iz,:,:]) statvarvals[it,iz,2] = np.max(varvalst[iz,:,:]) statvarvals[it,iz,3] = np.mean(varvalst[iz,:,:]) statvarvals[it,iz,4] = maskedvals2[iz,:,:].mean() statvarvals[it,iz,5] = np.sqrt(statvarvals[it,iz,4] - \ statvarvals[it,iz,3]*statvarvals[it,iz,3]) statvarvals[it,iz,6] = np.sum(varvalst[iz,:,:]) else: slicev.append(slice(0,dimz)) slicev.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2+1)) slicev.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2+1)) slicevnoT.append(slice(0,dimz)) slicevnoT.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+ \ box2+1)) slicevnoT.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+ \ box2+1)) slice2D.append(slice(0,dimz)) slice2D.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2] + \ box2 + 1)) slice2D.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1] + \ box2 + 1)) varvalst = varobj[tuple(slicev)] # box values varvals[it,:,:,:] = varvalst # print 'values at time t______' # print varvalst # box stats values for iz in range(dimz): statvarvals[it,:,0] = varvalst[:,box2,box2] statvarvals[it,iz,1] = np.min(varvalst[iz,:,:]) statvarvals[it,iz,2] = np.max(varvalst[iz,:,:]) statvarvals[it,iz,3] = np.mean(varvalst[iz,:,:]) statvarvals[it,iz,4] = np.mean(varvalst[iz,:,:]* \ varvalst[iz,:,:]) statvarvals[it,iz,5] = np.sqrt(statvarvals[it,iz,4] - \ statvarvals[it,iz,3]*statvarvals[it,iz,3]) statvarvals[it,iz,6] = np.sum(varvalst[iz,:,:]) # Circle values cslicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-Nrad < 0 or gtrajvals[it,2]+Nrad + 1 >= dimy \ or gtrajvals[it,1]-Nrad < 0 or gtrajvals[it,1]+Nrad + 1 >= dimx: maxx = np.min([cxrangeslice[it][1], dimx]) maxy = np.min([cyrangeslice[it][1], dimy]) cslicev.append(slice(0,dimz)) cslicev.append(slice(cyrangeslice[it][0], maxy)) cslicev.append(slice(cxrangeslice[it][0], maxx)) cslicevnoT.append(slice(0,dimz)) cslicevnoT.append(slice(cyrangeslice[it][0], cyrangeslice[it][1])) cslicevnoT.append(slice(cxrangeslice[it][0], cxrangeslice[it][1])) cslice2D.append(slice(0,dimz)) cslice2D.append(slice(0,maxy-cyrangeslice[it][0])) cslice2D.append(slice(0,maxx-cxrangeslice[it][0])) cslice2Dhor.append(slice(0, maxy - cyrangeslice[it][0])) cslice2Dhor.append(slice(0, maxx -cxrangeslice[it][0])) rvarvalst = np.ones((dimz,Nrad*2+1,Nrad*2+1),dtype=np.float)* \ fillValue rvarvalst[tuple(cslice2D)] = varobj[tuple(cslicev)] for iz in range(dimz): tslice = [slice(it,it)]+cslice2Dhor zslice = [slice(iz,iz)]+cslice2Dhor rvarvalst[tuple(zslice)] = np.where(circdist[tuple(tslice)] > \ np.float(Nrad), fillValue, rvarvalst[tuple(zslice)]) rvarvals[it,:,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) maskedvals2 = maskedvals*maskedvals for iz in range(dimz): rstatvarvals[it,iz,0] = varvalst[iz,box2,box2] rstatvarvals[it,iz,1] = np.min(varvalst[iz,:,:]) rstatvarvals[it,iz,2] = np.max(varvalst[iz,:,:]) rstatvarvals[it,iz,3] = np.mean(varvalst[iz,:,:]) rstatvarvals[it,iz,4] = maskedvals2[iz,:,:].mean() rstatvarvals[it,iz,5] = np.sqrt(rstatvarvals[it,iz,4] - \ rstatvarvals[it,iz,3]*rstatvarvals[it,iz,3]) rstatvarvals[it,iz,6] = np.sum(varvalst[iz,:,:]) else: cslicev.append(slice(0,dimz)) cslicev.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslicev.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) cslicevnoT.append(slice(0,dimz)) cslicevnoT.append(slice(gtrajvals[it,2]-Nrad, gtrajvals[it,2]+ \ Nrad+1)) cslicevnoT.append(slice(gtrajvals[it,1]-Nrad, gtrajvals[it,1]+ \ Nrad+1)) cslicevnoThor.append(slice(gtrajvals[it,2]-Nrad, \ gtrajvals[it,2] + Nrad+1)) cslicevnoThor.append(slice(gtrajvals[it,1]-Nrad, \ gtrajvals[it,1] + Nrad+1)) cslice2D.append(slice(0,dimz)) cslice2D.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2] + \ Nrad+1)) cslice2D.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1] + \ Nrad+1)) rvarvalst = varobj[tuple(cslicev)] # circle values for iz in range(dimz): tslice = [it]+cslicevnoThor rvarvalst[iz,:,:] = np.where(circdist[tuple(tslice)] > \ np.float(Nrad), fillValue, rvarvalst[iz,:,:]) rvarvals[it,:,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) maskedvals2 = maskedvals*maskedvals for iz in range(dimz): rstatvarvals[it,iz,0] = rvarvalst[iz,Nrad,Nrad] rstatvarvals[it,iz,1] = maskedvals[iz,:,:].min() rstatvarvals[it,iz,2] = maskedvals[iz,:,:].max() rstatvarvals[it,iz,3] = maskedvals[iz,:,:].mean() rstatvarvals[it,iz,4] = maskedvals2[iz,:,:].mean() rstatvarvals[it,iz,5] = np.sqrt(rstatvarvals[it,iz,4] - \ rstatvarvals[it,iz,3]*rstatvarvals[it,iz,3]) rstatvarvals[it,iz,6] = maskedvals[iz,:,:].sum() # print 'statistics:',rstatvarvals[it,:] # variable box values newvar = objofile.createVariable(vnst + 'box', 'f4', ('time','z','y','x'),\ fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = varvals # center of the trajectory newvar = objofile.createVariable('trj_' + vnst, 'f', ('time','z'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'trj_' + vsname, 'value along the ' + \ 'trajectory of '+ vn, vunits) newvar[:] = statvarvals[:,:,0] # variable box statistics ist = 0 for statn in statnames: newvar = objofile.createVariable(statn + '_' + vnst,'f',('time','z'),\ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the box ('+str(boxs)+'x'+str(boxs)+') of ' + vnst, vunits) newvar[:] = statvarvals[:,:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 # variable circle values newvar = objofile.createVariable(vnst + 'circle','f4',('time','z','yr', \ 'xr'), fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = rvarvals # variable circle statistics ist = 0 for statn in cstatnames: newvar = objofile.createVariable(statn + '_' + vnst,'f',('time','z'),\ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the circle of radius ('+ str(circler)+') of ' + vnst, vunits) newvar[:] = rstatvarvals[:,:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 elif Nvardims == 3: # Too optimistic, but at this stage... dimt = varobj.shape[0] varvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) lonvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) latvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) rvarvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) rlonvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) rlatvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) # One dimension plus for the values at the center of the trajectory statvarvals = np.ones(tuple([Ttraj,Nsts+1]), dtype=np.float) rstatvarvals = np.ones(tuple([Ttraj,Nsts+1]), dtype=np.float) for it in range(Ttraj): it0 = Tbeg + it slicev = [] slice2D = [] slicevnoT = [] cslicev = [] cslice2D = [] cslicevnoT = [] cslicevC = [] slicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-box2 < 0 or gtrajvals[it,2]+box2 + 1 > dimy + 1 \ or gtrajvals[it,1]-box2 < 0 or gtrajvals[it,1]+box2 + 1 > dimx + 1: # box values slicev.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicev.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slicevnoT.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicevnoT.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slice2D.append(slice(0,yrangeslice[it][1]-yrangeslice[it][0])) slice2D.append(slice(0,xrangeslice[it][1]-xrangeslice[it][0])) rvarvalst = np.ones((Nrad*2+1, Nrad*2+1),dtype=np.float)*fillValue varvalst[tuple(slice2D)] = varobj[tuple(slicev)] varvals[it,:,:] = varvalst # box stats values maskedvals = ma.masked_values (varvalst, fillValue) statvarvals[it,0] = varvalst[box2,box2] statvarvals[it,1] = maskedvals.min() statvarvals[it,2] = maskedvals.max() statvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals statvarvals[it,4] = maskedvals2.mean() statvarvals[it,5] = np.sqrt(statvarvals[it,4] - \ statvarvals[it,3]*statvarvals[it,3]) statvarvals[it,6] = maskedvals.sum() varvalst[tuple(slice2D)] = lonv[tuple(slicevnoT)] lonvals[it,:,:] = varvalst varvalst[tuple(slice2D)] = latv[tuple(slicevnoT)] latvals[it,:,:] = varvalst else: slicev.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2+1)) slicev.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2+1)) slicevnoT.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+ \ box2+1)) slicevnoT.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+ \ box2+1)) slice2D.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2 +\ 1)) slice2D.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2 +\ 1)) varvalst = varobj[tuple(slicev)] # box values varvals[it,:,:] = varvalst # print 'values at time t______' # print varvalst # box stats values statvarvals[it,0] = varvalst[box2,box2] statvarvals[it,1] = np.min(varvalst) statvarvals[it,2] = np.max(varvalst) statvarvals[it,3] = np.mean(varvalst) statvarvals[it,4] = np.mean(varvalst*varvalst) statvarvals[it,5] = np.sqrt(statvarvals[it,4] - \ statvarvals[it,3]*statvarvals[it,3]) statvarvals[it,6] = np.sum(varvalst) varvalst = lonv[tuple(slicevnoT)] lonvals[it,:,:] = varvalst varvalst = latv[tuple(slicevnoT)] latvals[it,:,:] = varvalst # Circle values cslicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-Nrad < 0 or gtrajvals[it,2]+Nrad + 1 >= dimy \ or gtrajvals[it,1]-Nrad < 0 or gtrajvals[it,1]+Nrad + 1 >= dimx: maxx = np.min([cxrangeslice[it][1], dimx]) maxy = np.min([cyrangeslice[it][1], dimy]) cslicev.append(slice(cyrangeslice[it][0],maxy)) cslicev.append(slice(cxrangeslice[it][0],maxx)) cslicevnoT.append(slice(cyrangeslice[it][0],cyrangeslice[it][1])) cslicevnoT.append(slice(cxrangeslice[it][0],cxrangeslice[it][1])) cslice2D.append(slice(0,maxy - cyrangeslice[it][0])) cslice2D.append(slice(0,maxx - cxrangeslice[it][0])) rvarvalst = np.ones((Nrad*2+1,Nrad*2+1),dtype=np.float)*fillValue rvarvalst[tuple(cslice2D)] = varobj[tuple(cslicev)] rvarvalst[tuple(cslice2D)] = np.where(circdist[tuple(cslicev)] >\ np.float(Nrad), fillValue, rvarvalst[tuple(cslice2D)]) rvarvals[it,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) rstatvarvals[it,0] = rvarvalst[Nrad,Nrad] rstatvarvals[it,1] = maskedvals.min() rstatvarvals[it,2] = maskedvals.max() rstatvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals rstatvarvals[it,4] = maskedvals2.mean() rstatvarvals[it,5] = np.sqrt(rstatvarvals[it,4] - \ rstatvarvals[it,3]*rstatvarvals[it,3]) rstatvarvals[it,6] = maskedvals2.sum() else: cslicev.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslicev.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) cslicevnoT.append(slice(gtrajvals[it,2]-Nrad, gtrajvals[it,2]+ \ Nrad+1)) cslicevnoT.append(slice(gtrajvals[it,1]-Nrad, gtrajvals[it,1]+ \ Nrad+1)) cslice2D.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslice2D.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) cslicevC.append(it) cslicevC.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslicevC.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) rvarvalst = varobj[tuple(cslicev)] cdist = circdist[tuple(cslicevC)] # circle values rvarvalst = np.where(cdist > np.float(Nrad),fillValue,rvarvalst) rvarvals[it,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) rstatvarvals[it,0] = rvarvalst[Nrad,Nrad] rstatvarvals[it,1] = maskedvals.min() rstatvarvals[it,2] = maskedvals.max() rstatvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals rstatvarvals[it,4] = maskedvals2.mean() rstatvarvals[it,5] = np.sqrt(rstatvarvals[it,4] - \ rstatvarvals[it,3]*rstatvarvals[it,3]) rstatvarvals[it,6] = maskedvals.sum() # print 'statistics:',rstatvarvals[it,:] # variable box values newvar = objofile.createVariable(vnst + 'box', 'f4', ('time', 'y', 'x'), \ fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = varvals # center of the trajectory newvar = objofile.createVariable('trj_' + vnst, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'trj_' + vsname, 'value along the ' + \ 'trajectory of '+ vnst, vunits) newvar[:] = statvarvals[:,0] # variable box statistics ist = 0 for statn in statnames: newvar = objofile.createVariable(statn + '_' + vnst, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the box ('+str(boxs)+'x'+str(boxs)+') of ' + vnst, vunits) newvar[:] = statvarvals[:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 # variable circle values newvar = objofile.createVariable(vnst + 'circle','f4',('time','yr','xr'),\ fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = rvarvals # variable circle statistics ist = 0 for statn in cstatnames: newvar = objofile.createVariable(statn + '_' + vnst, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the circle of radius ('+ str(circler)+') of ' + vnst, vunits) newvar[:] = rstatvarvals[:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 else: # Other variables print warnmsg print ' ' + fname + ": variable '" + vn + "' shape:",varobj.shape,' not'\ + ' ready!!' print ' skipping variable' if len(varns) == 1: objofile.close() sub.call(['rm', ofile]) print ' uniq variable! removing file and finishing program' quit() if not objofile.variables.has_key('trlon') and Nvardims == 3: # var dimensions newvar = objofile.createVariable('trlon', 'f8', ('time')) newattr = basicvardef(newvar,'trlon','trajectory longitude', \ 'degrees west_east') newvar[:] = trajvals[:,1] newvar = objofile.createVariable('trlat', 'f8', ('time')) newattr = basicvardef(newvar,'trlat','trajectory latitude', \ 'degrees north_south') newvar[:] = trajvals[:,2] newvar = objofile.createVariable('time', 'f8', ('time')) newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = trajvals[:,0] newvar = objofile.createVariable('lon', 'f8', ('time', 'y', 'x'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'longitude', 'longitude', \ 'degrees west_east') newvar[:] = lonvals newvar = objofile.createVariable('lat', 'f8', ('time', 'y', 'x'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'latitude', 'latitude', \ 'degrees north_south') newvar[:] = latvals ivar = ivar + 1 # global attributes objofile.setncattr('author', 'L. Fita') objofile.setncattr('institution', 'Laboratire de Meteorologie Dynamique') objofile.setncattr('university', 'Pierre Marie Curie - Jussieu') objofile.setncattr('center', 'Centre National de Recherches Scientifiques') objofile.setncattr('city', 'Paris') objofile.setncattr('country', 'France') objofile.setncattr('script', 'nc_var_tools.py') objofile.setncattr('function', 'compute_tevolboxtraj') objofile.setncattr('version', '1.0') objofile.setncattr('data_file',ncfile) objfile.close() objofile.sync() objofile.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return #compute_tevolboxtraj('h', 'wrfout*', 'PSFC') #compute_tevolboxtraj('control/trajectory.dat@0,XLONG,XLAT,ZNU,Times,wrf,3,3', \ # '../../superstorm/control/wrfout/wrfout_d01_2001-11-09_00:00:00', 'PSFC') #compute_tevolboxtraj('control/trajectory.dat@0,lon,lat,time,cf,5,5', \ # 'control/wss.nc', 'wss') # in camelot, /bdd/PCER/workspace/lfita/etudes/FF/50sims #compute_tevolboxtraj('001/trajectory.dat@0,XLONG,XLAT,ZNU,Times,wrf,3,3', \ # '001/full_concatenated.nc', 'PSFC') def slice_variable(varobj, dimslice): """ Function to return a slice of a given variable according to values to its dimensions slice_variable(varobj, dimslice) varobj= object wit the variable dimslice= [[dimname1]:[value1]|[[dimname2]:[value2], ...] pairs of dimension [value]: * [integer]: which value of the dimension * -1: all along the dimension * -9: last value of the dimension * [beg]@[end]@[freq] slice from [beg] to [end] every [freq] """ fname = 'slice_variable' if varobj == 'h': print fname + '_____________________________________________________________' print slice_variable.__doc__ quit() vardims = varobj.dimensions Ndimvar = len(vardims) Ndimcut = len(dimslice.split('|')) if Ndimcut == 0: Ndimcut = 1 dimcut = list(dimslice) dimsl = dimslice.split('|') varvalsdim = [] dimnslice = [] for idd in range(Ndimvar): found = False for idc in range(Ndimcut): dimcutn = dimsl[idc].split(':')[0] dimcutv = dimsl[idc].split(':')[1] if vardims[idd] == dimcutn: posfrac = dimcutv.find('@') if posfrac != -1: inifrac = int(dimcutv.split('@')[0]) endfrac = int(dimcutv.split('@')[1]) if endfrac == -9: endfrac=varobj.shape[idd] freqfrac = int(dimcutv.split('@')[2]) varvalsdim.append(slice(inifrac,endfrac,freqfrac)) if endfrac > varobj.shape[idd]: print errormsg print ' ' + fname + ": value for dimension '" + \ dimcutn + "' is too big (", endfrac, '>', \ varobj.shape[idd],'[dimension length] ) !!' quit(-1) dimnslice.append(vardims[idd]) else: if int(dimcutv) == -1: varvalsdim.append(slice(0,varobj.shape[idd])) dimnslice.append(vardims[idd]) elif int(dimcutv) == -9: varvalsdim.append(int(varobj.shape[idd])-1) else: if int(dimcutv) >= varobj.shape[idd]: print errormsg print ' ' + fname + ": value for dimension '" + \ dimcutn + "' is too big (", int(dimcutv), '>', \ varobj.shape[idd],'[dimension length] ) !!' quit(-1) varvalsdim.append(int(dimcutv)) found = True break if not found and not gen.searchInlist(dimnslice,vardims[idd]): varvalsdim.append(slice(0,varobj.shape[idd])) dimnslice.append(vardims[idd]) varvalues = np.squeeze(varobj[tuple(varvalsdim)]) return varvalues, dimnslice def operdim(varv, vardims, doper, oper): """ Function to operate along a series of dimensions varv: matrix values vardims: dimension names of the variable doper: names of the dimensions to operate oper: operation to perform >>> variable = genericNCvariable(5,4,3,2,'lon','lat','pres','time') >>> varv = variable[:] >>> vardims = variable.dimensions >>> doper = ['time', 'pres'] >>> oper = 'mean' >>> operv, operd = operdim(varv, vardims, doper, oper) [[ 50. 51. 52. 53. 54.] [ 55. 56. 57. 58. 59.] [ 60. 61. 62. 63. 64.] [ 65. 66. 67. 68. 69.]] ['lat', 'lon'] """ fname = 'operdim' readyop = ['max', 'mean', 'mean2', 'min', 'std', 'sum', 'turb', 'var'] operslice = [] for dn in vardims: if not gen.searchInlist(doper, dn): operslice.append(False) else: operslice.append(True) opereddims = [] noper = 0 varoper = varv.copy() for idim in range(len(varv.shape)): if operslice[idim]: if oper == 'max': varoper = np.max(varoper, axis=idim-noper) elif oper == 'mean': varoper = np.mean(varoper, axis=idim-noper) elif oper == 'mean2': varoper1 = np.mean(varoper, axis=idim-noper) varoper = varoper1*varoper1 elif oper == 'min': varoper = np.min(varoper, axis=idim-noper) elif oper == 'std': varoper = np.std(varoper, axis=idim-noper) elif oper == 'sum': varoper = np.sum(varoper, axis=idim-noper) elif oper == 'turb': varoper = np.mean(varoper, axis=idim-noper) opereddims.append(vardims[idim]) elif oper == 'var': varoper = np.var(varoper, axis=idim-noper) else: print errormsg print ' ' + fname + ': operation "' + oper + '" not ready!!' print ' operations available:', readyop quit(-1) noper = noper + 1 else: opereddims.append(vardims[idim]) # In 'turb' operation, resultant values have the original shape if oper == 'turb': operedvals = gen.oper_submatrix(varv, vardims, varoper, 'subc', doper) else: operedvals = varoper return operedvals, opereddims def operation_alongdims(ovar,dimvals,dimsoper,opkind): """ Function to operate along different dimensions of a variable operation_alongdims(ovar,dimvals,opkind) ovar= netCDF variable object dimvals= [dimname1]:[val1]|[dimdname2]:[val2]|[...[dimnameN]:[valN]] [value]: * [integer]: which value of the dimension * -1: all along the dimension * [beg]:[end] slice from [beg] to [end] dimsoper= [dimname1]:[dimname2]:[...[dimnameN]] names of the dimensions along which operation has to be done opkind= operation to perform along the dimensions with a range: max, mean, mean2, min, sum, turb, var """ fname = 'operation_alongdims' if ovar == 'h': print fname + '_____________________________________________________________' print operation_alongdims.__doc__ quit() vardims = ovar.dimensions Nvardims = len(vardims) cutdims = dimvals.split('|') Ncutdims = len(cutdims) if Nvardims != Ncutdims: print warnmsg print ' ' + fname + ': cutting for',Ncutdims,'but variable has:',Nvardims, \ '!!' # quit(-1) cutdimvals = '' for idv in range(Nvardims): found = False Lcutdimvals = len(cutdimvals) for idc in range(Ncutdims): if vardims[idv] == cutdims[idc].split(':')[0]: if Lcutdimvals > 0: cutdimvals = cutdimvals + '|' + cutdims[idc] else: cutdimvals = cutdims[idc] found = True break if not found: # print errormsg # print ' ' + fname + ": cutting by '",cutdims[idc].split(':')[0], \ # "' but variable has not it !!!" # print ' variable dims:',vardims print warnmsg print ' ' + fname + ": variable has dimension '",vardims[idv], \ "' but is not cut taking all length !!!" if Lcutdimvals > 0: cutdimvals = cutdimvals + '|' + vardims[idv] + ':-1' else: cutdimvals = vardims[idv] + ':-1' # quit(-1) slicedvar, dimslice = slice_variable(ovar, cutdimvals) # operation dimnsoper = dimsoper.split(':') if opkind == 'mean2': origvar = slicedvar*slicedvar else: origvar = slicedvar.copy() dimsnewvar = [] Ndimslice = len(dimslice) Ndimsoper = len(dimnsoper) noper=0 for ids in range(Ndimslice): found = False for ido in range(Ndimsoper): if dimslice[ids] == dimnsoper[ido]: if opkind == 'max': varoper = np.max(origvar, axis=ids-noper) elif opkind == 'mean': varoper = np.mean(origvar, axis=ids-noper) elif opkind == 'mean2': varoper = np.mean(origvar, axis=ids-noper) elif opkind == 'min': varoper = np.min(origvar, axis=ids-noper) elif opkind == 'sum': varoper = np.sum(origvar, axis=ids-noper) elif opkind == 'std': varoper = np.std(origvar, axis=ids-noper) elif opkind == 'var': varoper = np.var(origvar, axis=ids-noper) else: print errormsg print ' ' + fname + ': operation "' + opkind + '" not ready!!' quit(-1) del(origvar) origvar = varoper.copy() noper = noper + 1 found = True break if not found: dimsnewvar.append(dimslice[ids]) return origvar, dimsnewvar def var_3desc(ovar): """ Function to provide std_name, long_name and units from an object variable ovar= object variable """ fname = 'var_desc' varattrs = ovar.ncattrs() if searchInlist(varattrs,'std_name'): stdn = ovar.getncattr('std_name') else: vvalues = variables_values(ovar._name) stdn = vvalues[1] if searchInlist(varattrs,'long_name'): lonn = ovar.getncattr('long_name') else: lonn = vvalues[4].replace('|',' ') if searchInlist(varattrs,'units'): un = ovar.getncattr('units') else: un = vvalues[5] return stdn, lonn, un def file_oper_alongdims(values, ncfile, varn): """ Function to operate a file along different dimensions of a variable file_oper_alongdims(values, ncfile, varn) values= [dimvals],[dimsoper],[opkind],[dimvn] [dimvals]; [dimname1]|[val1]:[...[dimnameN]|[valN]] [value]; * [integer]: which value of the dimension * -1: all along the dimension * -9: last value along the dimension * [beg]:[end] slice from [beg] to [end] NOTE: that dimension of the variables which not appear al values will be used [dimsoper]; [dimname1]:[dimname2]:[...[dimnameN]] names of the dimensions along which operation has to be done [opkind]; operation to perform along the dimensions with a range: max, mean, mean2, min, sum, var [dimvn]; [varname1]:[varname2]:[...[varnameM]] variables with the values of the dimensions ncfile= netCDF file varn= ',' list of variable names ('all' for all) """ fname = 'file_oper_alongdims' if values == 'h': print fname + '_____________________________________________________________' print file_oper_alongdims.__doc__ quit() dimvals = values.split(',')[0].split(':') dimsoper = gen.str_list(values.split(',')[1], ':') operkind = values.split(',')[2] dimvn = values.split(',')[3].split(':') # Operations with which when the variable dimensions will be ajusted to the file # the zero value along the operated dimensions will be taken zerodimops = ['max', 'mean', 'mean2', 'min', 'sum', 'std', 'turb', 'var'] dimsoperS = gen.numVector_String(dimsoper,', ') objnc = NetCDFFile(ncfile, 'r') if varn.find(',') != -1: varns = varn.split(',') else: if varn == 'all': varns = objnc.variables else: varns = [varn] ofile = 'file_oper_alongdims_' + operkind + '.nc' # Getting slice dimensions dimslice = {} for dv in dimvals: dnv = dv.split('|')[0] dvv = dv.split('|')[1] if dvv.find(':') != -1: beg = int(dvv.split(':')[0]) end = int(dvv.split(':')[1]) dimslice[dnv] = [beg, end, 1] else: dimslice[dnv] = int(dvv) # Creation of output file ## objnewnc = NetCDFFile(ofile, 'w') # Dimensions which should appear at the final file (from the computed variables) dimfinalfile = [] print ' ' + fname + ": '" + operkind + "' with:", dimsoper, '_______' newNdim = 0 newNvar = 0 for vn in varns: print " '" + vn + "' ... .. ." if not objnc.variables.has_key(vn): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + vn + '" !!' quit(-1) # Do we have to compute this variable? Only to compute that variables with # coincidence on the dimensions to operate ov = objnc.variables[vn] tocompute = False for vd in ov.dimensions: if gen.searchInlist(dimsoper, vd): tocompute = True if tocompute: # Dimensions which need to appear at the final file: that ones not opered # from all opered variables for vd in ov.dimensions: if not gen.searchInlist(dimfinalfile, vd) and \ not gen.searchInlist(dimsoper, vd): dimfinalfile.append(vd) varslice, varsliceddims = SliceVarDict(ov,dimslice) # This is absolutely weird, but if I do not make this, there is segmentation fault! if len(varsliceddims) == len(ov.shape): varvals00 = np.zeros(tuple(ov.shape), dtype=np.float) for ix in range(ov.shape[0]): varvals00[ix,...] = ov[ix,...] if type(ov[:]) == type(ma.array((3), dtype=np.float)): print warnmsg print ' ' + fname + ': masked variale !!' varvals0 = ma.masked_array(varvals00, mask=ov[:].mask) else: varvals0 = varvals00 else: varvals0 = ov[tuple(varslice)] varvals = np.squeeze(varvals0) finalvarvals, finaldims = operdim(varvals, varsliceddims, dimsoper, operkind) # Adding dimensions idn = 0 for dn in finaldims: if newNdim == 0: if not objnc.dimensions[dn].isunlimited(): objnewnc.createDimension(dn, finalvarvals.shape[idn]) else: objnewnc.createDimension(dn, None) newNdim = newNdim + 1 else: if not gen.searchInlist(objnewnc.dimensions, dn): if not objnc.dimensions[dn].isunlimited(): objnewnc.createDimension(dn, finalvarvals.shape[idn]) else: objnewnc.createDimension(dn, None) newNdim = newNdim + 1 idn = idn + 1 # Adding variable varinf = variable_inf(ov) if varinf.FillValue is not None: newvar = objnewnc.createVariable(vn+operkind, nctype(varinf.dtype), \ tuple(finaldims), fill_value=varinf.FillValue) else: newvar = objnewnc.createVariable(vn+operkind, nctype(varinf.dtype), \ tuple(finaldims)) newNvar = newNvar + 1 if len(finaldims) > 0: newvar[:] = finalvarvals[:] else: newvar[:] = finalvarvals # new variable oldvarattr = ov.ncattrs() varname = gen.variables_values(vn)[0] if gen.searchInlist(oldvarattr, 'standard_name'): stdname = ov.getncattr('standard_name') else: stdname = gen.variables_values(vn)[1] if gen.searchInlist(oldvarattr, 'long_name'): lname = ov.getncattr('long_name') else: lname = gen.variables_values(vn)[4].replace('|',' ') if gen.searchInlist(oldvarattr, 'units'): uname = ov.getncattr('units') else: uname = gen.variables_values(vn)[5] newattr = basicvardef(newvar, stdname + operkind, lname + ' '+ operkind +\ ' along ' + dimsoperS, uname) for idim in ov.dimensions: for dv in dimslice.keys(): if dv == idim: if type(dimslice[dv]) == type([2]): Sdv = gen.numVector_Sring(dimslice[dv],', ') + ', 1' elif type(dimslice[dv]) == type(int(2)): if dimslice[dv] == -1: Sdv = str(len(objnc.dimensions[idim])) else: Sdv = str(dimslice[dv]) else: Sdv = str(len(objnc.dimensions[idim])) if idim == ov.dimensions[0]: origdimsS = idim + '(' + Sdv + ')' else: origdimsS = origdimsS + ' ' + idim + '(' + Sdv + ')' newattr = set_attribute(newvar, 'orig_dimensions', origdimsS) # Adding original attributes for attrn in ov.ncattrs(): if attrn != '_FillValue': attrv = ov.getncattr(attrn) newattr = set_attribute(newvar,attrn,attrv) objnewnc.sync() else: varinf = variable_inf(ov) dimvars = varinf.dimns shapevar = ov.shape idim = 0 for dimn in dimvars: if not gen.searchInlist(list(objnewnc.dimensions), dimn): if objnc.dimensions[dimn].isunlimited: newdim = objnewnc.createDimension(dimn,None) else: newdim = objnewnc.createDimension(dimn,shapevar[idim]) idim = idim + 1 # Adding that variables which are not computed via 'operation' if varinf.FillValue is not None: newvar = objnewnc.createVariable(vn, nctype(varinf.dtype), \ tuple(varinf.dimns), fill_value=varinf.FillValue) else: newvar = objnewnc.createVariable(vn, nctype(varinf.dtype), \ tuple(varinf.dimns)) newvar[:] = ov[:] for attrn in ov.ncattrs(): if attrn != '_FillValue': attrv = ov.getncattr(attrn) newattr = set_attribute(newvar,attrn,attrv) objnewnc.sync() # Statistics of the variable dimension along the dimensions operated basicoper = ['min', 'max', 'mean', 'mean2', 'sum', 'std', 'var'] # operating variable dimension print ' ' + fname + ': Opering slice of variable dimension _______' for vn in dimvn: print " '" + vn + "' ... .. ." if not objnc.variables.has_key(vn): print warnmsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + vn + '" skipping it !!' continue # Do we have to compute this variable? ov = objnc.variables[vn] tocompute = False # Variable might already be in the file if not gen.searchInlist(objnewnc.variables.keys(), vn+operkind): for vd in ov.dimensions: if gen.searchInlist(dimfinalfile, vd): tocompute = True if tocompute: varslice, varsliceddims = SliceVarDict(ov,dimslice) varvals0 = ov[tuple(varslice)] varvals = np.squeeze(varvals0) tooper = False for idn in varsliceddims: if gen.searchInlist(dimsoper,idn): tooper = True if tooper: finalvarvals, finaldims = operdim(varvals, varsliceddims, dimsoper, operkind) else: finalvarvals = varvals finaldims = varsliceddims # Adding dimensions idn = 0 for dn in finaldims: if newNdim == 0: if not searchInlist(objnewnc.dimensions, dn): if not objnc.dimensions[dn].isunlimited(): objnewnc.createDimension(dn, finalvarvals.shape[idn]) else: objnewnc.createDimension(dn, None) newNdim = newNdim + 1 else: if not gen.searchInlist(objnewnc.dimensions, dn): if not objnc.dimensions[dn].isunlimited(): objnewnc.createDimension(dn, finalvarvals.shape[idn]) else: objnewnc.createDimension(dn, None) newNdim = newNdim + 1 idn = idn + 1 # Adding variable varinf = variable_inf(ov) if varinf.FillValue is not None: newvar = objnewnc.createVariable(vn+operkind, nctype(varinf.dtype), \ tuple(finaldims), fillValue=varinf.FillValue) else: newvar = objnewnc.createVariable(vn+operkind, nctype(varinf.dtype), \ tuple(finaldims)) newNvar = newNvar + 1 newvar[:] = finalvarvals[:] # new variable oldvarattr = ov.ncattrs() varname = gen.variables_values(vn)[0] if gen.searchInlist(oldvarattr, 'standard_name'): stdname = ov.getncattr('standard_name') else: stdname = gen.variables_values(vn)[1] if gen.searchInlist(oldvarattr, 'long_name'): lname = ov.getncattr('long_name') else: lname = gen.variables_values(vn)[4].replace('|',' ') if gen.searchInlist(oldvarattr, 'units'): uname = ov.getncattr('units') else: uname = gen.variables_values(vn)[5] newattr = basicvardef(newvar, stdname + operkind, lname + ' '+ operkind +\ ' along ' + dimsoperS, uname) for idim in ov.dimensions: for dv in dimslice.keys(): if dv == idim: if type(dimslice[dv]) == type([2]): Sdv = gen.numVector_Sring(dimslice[dv],', ') + ', 1' elif type(dimslice[dv]) == type(int(2)): if dimslice[dv] == -1: Sdv = str(len(objnc.dimensions[idim])) else: Sdv = str(dimslice[dv]) else: Sdv = str(len(objnc.dimensions[idim])) if idim == ov.dimensions[0]: origdimsS = idim + '(' + Sdv + ')' else: origdimsS = origdimsS + ' ' + idim + '(' + Sdv + ')' newattr = set_attribute(newvar, 'orig_dimensions', origdimsS) objnewnc.sync() # Statistics of the variable dimension along the dimensions operated in case # dimension after operation disapears for vn in dimvn: if not objnc.variables.has_key(vn): print warnmsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + vn + '" skipping it !!' continue # Do we have to compute this variable? ov = objnc.variables[vn] tocompute = False for vd in ov.dimensions: if gen.searchInlist(dimsoper, vd): tocompute = True if tocompute: varslice, varsliceddims = SliceVarDict(ov,dimslice) varvals0 = ov[tuple(varslice)] varvals = np.squeeze(varvals0) varname = gen.variables_values(vn)[0] if gen.searchInlist(oldvarattr, 'standard_name'): stdname = ov.getncattr('standard_name') else: stdname = gen.variables_values(vn)[1] if gen.searchInlist(oldvarattr, 'long_name'): lname = ov.getncattr('long_name') else: lname = gen.variables_values(vn)[4].replace('|',' ') if gen.searchInlist(oldvarattr, 'units'): uname = ov.getncattr('units') else: uname = gen.variables_values(vn)[5] # No statistics for `turb' if len(varvals.shape) == 1 and operkind != 'turb': print ' ' + fname + ': Statisitcs of operated variable dimension _______' print " '" + vn + "' ... .. ." # Statistics of the variable dimension along the dimensions operated newvar = objnewnc.createVariable(vn+'stats', 'f4' ) newattr = basicvardef(newvar, stdname + 'statistics', lname + \ ' statistics', uname) newattr = set_attributek(newvar, 'Nvalues', len(varvals), 'I') for op in basicoper: finalvarv, finalds = operdim(varvals,varsliceddims,dimsoper,op) newattr = set_attributek(newvar, op, finalvarv, 'R') objnewnc.sync() # Including operation dimensions on the new slice for the variables dimension # but, which value along the operated dimension should be taken? sliceaddvars = dimslice if gen.searchInlist(zerodimops, operkind): dimsliceval = 0 for cdn in dimsoper: sliceaddvars[cdn] = dimsliceval # slicing variable dimension print ' ' + fname + ': Adding slice of variable dimension _______' for vn in dimvn: print " '" + vn + "' ... .. ." if not objnc.variables.has_key(vn): print warnmsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + vn + '" skippping it!!' continue # Do we have to add this dimension-variable? (only that ones with same # dimensions as desired variable) ov = objnc.variables[vn] toadd = False for vd in ov.dimensions: if gen.searchInlist(dimfinalfile, vd): toadd = True if toadd: varinf = variable_inf(ov) varslice, varsliceddims = SliceVarDict(ov,sliceaddvars) varvals0 = ov[tuple(varslice)] varvals = np.squeeze(varvals0) for idv in range(len(varvals.shape)): if not gen.searchInlist(objnewnc.dimensions, varsliceddims[idv]): newdim = objnewnc.createDimension(varsliceddims[idv], \ varvals.shape[idv]) newvar = objnewnc.createVariable(vn, nctype(varinf.dtype), \ tuple(varsliceddims)) newvar[:] = varvals for attrn in varinf.attributes: attrv = ov.getncattr(attrn) newattr = set_attribute(newvar, attrn, attrv) objnewnc.sync() # global attributes objnewnc.setncattr('author', 'L. Fita') # Not working in ciclad #newattr = set_attributek(objnewnc, 'institution', unicode('Laboratoire de M' + \ # unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') newattr = set_attributek(objnewnc, 'institution', 'Laboratoire de Meteorologie'+ \ ' Dynamique', 'S') objnewnc.setncattr('university', 'Pierre Marie Curie - Jussieu') objnewnc.setncattr('center', 'Centre National de Recherches Scientifiques') objnewnc.setncattr('city', 'Paris') objnewnc.setncattr('country', 'France') objnewnc.setncattr('script', 'nc_var_tools.py') objnewnc.setncattr('function', 'file_oper_alongdims') objnewnc.setncattr('version', '1.0') objnewnc.setncattr('data_file', ncfile) gorigattrs = objnc.ncattrs() for attr in gorigattrs: attrv = objnc.getncattr(attr) atvar = set_attributek(objnewnc, attr, attrv, type(attrv)) objnc.close() objnewnc.sync() objnewnc.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return def file_oper_alongdims_old(values, ncfile, varn): """ Function to operate a file along different dimensions of a variable file_oper_alongdims(values, ncfile, varn) values= [dimvals],[dimsoper],[opkind],[dimvn] [dimvals]; [dimname1]:[val1]|[dimdname2]:[val2]|[...[dimnameN]:[valN]] [value]; * [integer]: which value of the dimension * -1: all along the dimension * [beg]:[end] slice from [beg] to [end] [dimsoper]; [dimname1]:[dimname2]:[...[dimnameN]] names of the dimensions along which operation has to be done [opkind]; operation to perform along the dimensions with a range: max, mean, mean2, min, sum [dimvn]; [varname1]:[varname2]:[...[varnameM]] variables with the values of the dimensions of the resultant operation ncfile= netCDF file varn= ',' list of variable names ('all' for all) """ fname = 'file_oper_alongdims' if values == 'h': print fname + '_____________________________________________________________' print file_oper_alongdims.__doc__ quit() dimvals = values.split(',')[0] dimsoper = values.split(',')[1] operkind = values.split(',')[2] dimvn = values.split(',')[3] dimsoperS = gen.numVector_String(dimsoper.split(':'),', ') objnc = NetCDFFile(ncfile, 'r') if varn.find(',') != -1: varns = varn.split(',') else: if varn == 'all': varns = objnc.variables else: varns = [varn] ofile = 'file_oper_alongdims_' + operkind + '.nc' if dimsoper.find(':') != -1: dimstouse = dimsoper.split(':') else: dimstouse = [dimsoper] # Creation of output file ## objnewnc = NetCDFFile(ofile, 'w') print ' ' + fname + ": '" + operkind + "' with:",dimsoper,'_______' for vn in varns: print "'" + vn + "' ... .. ." if not objnc.variables.has_key(vn): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + vn + '" !!!!' quit(-1) # Variable objvar = objnc.variables[vn] allfound = True for idimt in dimstouse: if not gen.searchInlist(objvar.dimensions, idimt): allfound = False break if not allfound: print warnmsg print ' ' + fname + ": variable '" + vn + "' (", objvar.dimensions, \ ') does not have all the equired dimensions:', dimstouse, '!!' print ' skipping it' else: newvarv, newvardimns=operation_alongdims(objvar,dimvals,dimsoper,operkind) # Dimensions for idim in range(len(newvardimns)): objdim = objnc.dimensions[newvardimns[idim]] if not objnewnc.dimensions.has_key(newvardimns[idim]): if objdim.isunlimited(): objnewnc.createDimension(newvardimns[idim], None) else: objnewnc.createDimension(newvardimns[idim], len(objdim)) # Variables with dimension values vardims = dimvn.split(':') Nvardims = len(vardims) for ivd in range(Nvardims): vdname = vardims[ivd] if not objnewnc.variables.has_key(vdname) and \ objnc.variables.has_key(vdname): objvdim = objnc.variables[vdname] coinvardims = objvdim.dimensions # Introducing only that variable dimensions with coincident dims toadd = False for vdn in coinvardims: if gen.searchInlist(newvardimns, vdn): toadd = True vdimtype = objvdim.dtype if toadd: newdvarv, newdvards=operation_alongdims(objvdim,dimvals, \ dimsoper,operkind) # Introducing only that dimension variables with any dimension after the operation if len(newdvards) > 0: newvar = objnewnc.createVariable(vdname, vdimtype, \ tuple(newdvards)) newvar[:] = newdvarv dimvattrs = objvdim.ncattrs() for attrn in dimvattrs: attrv = objvdim.getncattr(attrn) newattr = set_attribute(newvar,attrn,attrv) newattr = set_attribute(newvar, 'operation', operkind + \ ' along ' + dimsoperS) # new variable oldvarattr = objvar.ncattrs() varname = gen.variables_values(vn)[0] if gen.searchInlist(oldvarattr, 'standard_name'): stdname = objvar.getncattr('standard_name') else: stdname = gen.variables_values(vn)[1] if gen.searchInlist(oldvarattr, 'long_name'): lname = objvar.getncattr('long_name') else: lname = gen.variables_values(vn)[4].replace('|',' ') if gen.searchInlist(oldvarattr, 'units'): uname = objvar.getncattr('units') else: uname = gen.variables_values(vn)[5] newvar = objnewnc.createVariable(varname + operkind, 'f4', tuple(newvardimns)) newattr = basicvardef(newvar, stdname + operkind, lname + ' '+ operkind +\ ' along ' + dimsoperS, uname) newvar[:] = newvarv for idim in range(len(objvar.shape)): if dimvals.find('|') != -1: dimn = dimvals.split('|')[idim].split(':')[0] else: dimn = dimvals.split(':')[0] dimv = '{:d}'.format(len(objnc.dimensions[dimn])) if idim == 0: origdimsS = dimn + '(' + dimv + ')' else: origdimsS = origdimsS + ' ' + dimn + '(' + dimv + ')' newattr = set_attribute(newvar, 'orig_dimensions', origdimsS) # global attributes objnewnc.setncattr('author', 'L. Fita') objnewnc.setncattr('institution', 'Laboratire de Meteorologie Dynamique') objnewnc.setncattr('university', 'Pierre Marie Curie - Jussieu') objnewnc.setncattr('center', 'Centre National de Recherches Scientifiques') objnewnc.setncattr('city', 'Paris') objnewnc.setncattr('country', 'France') objnewnc.setncattr('script', 'nc_var_tools.py') objnewnc.setncattr('function', 'file_oper_alongdims') objnewnc.setncattr('version', '1.0') objnewnc.setncattr('data_file', ncfile) gorigattrs = objnc.ncattrs() for attr in gorigattrs: attrv = objnc.getncattr(attr) atvar = set_attribute(objnewnc, attr, attrv) objnc.close() objnewnc.sync() objnewnc.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return #file_oper_alongdims('Time:-1|bottom_top:-1|south_north:-1|west_east:-1,bottom_top,sum,Times:XLAT:XLONG', # '/home/lluis/PY/wrfout_d01_2001-11-11_00:00:00.tests', 'QVAPOR') #file_oper_alongdims('time:-1|z:-1|x:-1|y:-1,time:y,mean,pressure:lat', '/home/lluis/etudes/WRF_LMDZ/WaquaL/WRF_LMDZ/AR40/vertical_interpolation_WRFp.nc', 'WRFt') def WRF_d0Nref(values, geofold): """ Function for the generation of an extra WRF domain from a given one field_stats(values, ncfile, varn) [values]= [namelist],[minLON],[minLAT],[maxLON],[maxLAT],[Ndomref],[factor],[geogres] [namelist]: namelist.wps to use [minLON],[minLAT]: minimum longitude and latitude [maxLON],[maxLAT]: maximum longitude and latitude [Ndomref]: domain reference number [factor]: resolution factor with respect d01 [geogres]: which topography resolution to use [geofold]= folder where to found the geo_em.d0[N-1].nc file """ fname='WRF_d0Nref' if values == 'h': print fname + '_____________________________________________________________' print WRF_d0Nref.__doc__ quit() expectargs = '[namelist],[minLON],[minLAT],[maxLON],[maxLAT],[Ndomref],[factor],' expectargs = expectargs + '[geogres]' check_arguments(fname,values,expectargs,',') namelist = values.split(',')[0] minLON = np.float(values.split(',')[1]) minLAT = np.float(values.split(',')[2]) maxLON = np.float(values.split(',')[3]) maxLAT = np.float(values.split(',')[4]) Ndomref = int(values.split(',')[5]) factor = int(values.split(',')[6]) geogres = values.split(',')[7] onml = 'namelist.wps_d0' + str(Ndomref+1) # Namelist variables to which new value is a copy of the last one repeatvalue = ['start_date' ,'end_date'] # Namelist variables to which value is increased by 1 (single value) add1 = ['max_dom'] # Namelist variables to which new value is an increase by 1 of the last one add1value = ['parent_id'] # ncfile = geofold + '/geo_em.d' + zeros(Ndomref,2) + '.nc' ncfile = geofold + '/geo_em.d0' + str(Ndomref) + '.nc' if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': geo_em file "' + ncfile + '" does not exist !!' quit(-1) ncobj = NetCDFFile(ncfile, 'r') nlon='XLONG_M' nlat='XLAT_M' lonobj = ncobj.variables[nlon] latobj = ncobj.variables[nlat] lon = lonobj[0,:,:] lat = latobj[0,:,:] ncobj.close() diffmin = np.sqrt((lon - minLON)**2. + (lat - minLAT)**2.) diffmax = np.sqrt((lon - maxLON)**2. + (lat - maxLAT)**2.) posdiffmin = index_mat(diffmin,np.min(diffmin)) posdiffmax = index_mat(diffmax,np.min(diffmax)) # print 'min vals. min:',np.min(diffmin),'max:',np.min(diffmax) # print 'min:', minLON, ',', minLAT, ':', posdiffmin, '.', lon[tuple(posdiffmin)], \ # ',', lat[tuple(posdiffmin)] # print 'max:', maxLON, ',', maxLAT, ':', posdiffmax, '.', lon[tuple(posdiffmax)], \ # ',', lat[tuple(posdiffmax)] print ' ' + fname +': coordinates of the SW corner:', posdiffmin, ' NE corner:',\ posdiffmax nmlobj = open(namelist, 'r') onmlobj = open(onml, 'w') for line in nmlobj: listline = reduce_spaces(line) nvals = len(listline) if nvals > 0: if searchInlist(add1,listline[0]): nmlval = int(listline[nvals-1].replace(',','')) + 1 onmlobj.write(' ' + listline[0] + ' = ' + str(nmlval) + ',\n') elif searchInlist(add1value,listline[0]): if Ndomref == 1: nmlval = 1 else: nmlval = int(listline[nvals-1].replace(',','')) + 1 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1], ' ') + ' ' +str(nmlval)+',\n') elif searchInlist(repeatvalue,listline[0]): nmlval = listline[nvals-1] onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+'\n') elif listline[0] == 'parent_grid_ratio': nmlval = factor onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'i_parent_start': nmlval = posdiffmin[1] - 10 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'j_parent_start': nmlval = posdiffmin[0] - 10 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'e_we': nmlval = (posdiffmax[1] - posdiffmin[1] + 20)*factor nmlval = nmlval + np.mod(nmlval,factor) + 1 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'e_sn': nmlval = (posdiffmax[0] - posdiffmin[0] + 20)*factor nmlval = nmlval + np.mod(nmlval,factor) + 1 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'geog_data_res': nmlval = "'" + geogres + "'" onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') else: onmlobj.write(line) else: onmlobj.write('\n') nmlobj.close() onmlobj.close() print fname + ": successfully written new namelist '" + onml + "' !!" return def WRF_CFtime_creation(values, ncfile, varn): """ Function to add a CF-convention time unit in a WRF file WRF_CFtime_creation(values, ncfile, varn) [values]= [refdate],[units] [refdate]: reference date to compute the times in [YYYY][MM][DD][HH][MI][SS] format [units]: time units to use: 'days','hours','minutes','seconds' [ncfile]= WRF file to add the CF-convention time variable [varn]= name of the variable time to add """ fname='WRF_CFtime_creation' if values == 'h': print fname + '_____________________________________________________________' print WRF_CFtime_creation.__doc__ quit() refdate=values.split(',')[0] tunitsval=values.split(',')[1] wrfnc = NetCDFFile(ncfile, 'a') if wrfnc.variables.has_key(varn): print warnmsg print ' ' + fname + ": WRF file '" + ncfile + "' already has variable '" + \ varn + "' !!!" print ' overwritting values' timeobj = wrfnc.variables['Times'] timewrfv = timeobj[:] yrref=refdate[0:4] monref=refdate[4:6] dayref=refdate[6:8] horref=refdate[8:10] minref=refdate[10:12] secref=refdate[12:14] refdateS = yrref + '-' + monref + '-' + dayref + ' ' + horref + ':' + minref + \ ':' + secref dt = timeobj.shape[0] cftimes = np.zeros((dt), dtype=np.float) for it in range(dt): wrfdates = gen.datetimeStr_conversion(timewrfv[it,:],'WRFdatetime', \ 'matYmdHMS') cftimes[it] = gen.realdatetime1_CFcompilant(wrfdates, refdate, tunitsval) tunits = tunitsval + ' since ' + refdateS # Create dimension if not wrfnc.dimensions.has_key('time'): newdim = wrfnc.createDimension('time', None) if wrfnc.variables.has_key(varn): newvar = wrfnc.variables[varn] else: newvar = wrfnc.createVariable('time','f8',('time')) newattr = basicvardef(newvar, 'time','time',tunits) newvar[:] = cftimes newattr = set_attribute(newvar, 'calendar', 'standard') newattr = set_attribute(newvar, 'axis', 'T') newattr = set_attribute(newvar, '_CoordinateAxisType', 'Time') wrfnc.sync() wrfnc.close() return #WRF_CFtime_creation('19491201000000,hours', 'wrfout_d01_1995-01-15_00:00:00', 'time') def WRF_CFxtime_creation(values, ncfile, varn): """ Function to add a CF-convention time unit in a WRF file using variable 'XTIME' WRF_CFxtime_creation(values, ncfile, varn) [values]= [refdate],[units] [refdate]: reference date to compute the times in [YYYY][MM][DD][HH][MI][SS] format [units]: time units to use: 'days','hours','minutes','seconds' [ncfile]= WRF file to add the CF-convention time variable [varn]= name of the variable time to add """ fname='WRF_CFxtime_creation' if values == 'h': print fname + '_____________________________________________________________' print WRF_CFxtime_creation.__doc__ quit() refdate=values.split(',')[0] tunitsval=values.split(',')[1] wrfnc = NetCDFFile(ncfile, 'a') if not wrfnc.variables.has_key('XTIME'): print errormsg print ' ' + fname + ": WRF file '" + ncfile + "' does not have variable '" +\ "XTIME' !!" quit(-1) if wrfnc.variables.has_key(varn): print warnmsg print ' ' + fname + ": WRF file '" + ncfile + "' already has variable '" + \ varn + "' !!!" print ' over-writting values' if not searchInlist(wrfnc.ncattrs(), 'SIMULATION_START_DATE'): print errormsg print ' ' + fname + ": WRF file '" + ncfile + "' does not have attribute '"+\ "'SIMULATION_START_DATE' !!!" quit(-1) startdate = wrfnc.SIMULATION_START_DATE timeobj = wrfnc.variables['XTIME'] timewrfv = timeobj[:] yrref=refdate[0:4] monref=refdate[4:6] dayref=refdate[6:8] horref=refdate[8:10] minref=refdate[10:12] secref=refdate[12:14] refdateS = yrref + '-' + monref + '-' + dayref + '_' + horref + ':' + minref + \ ':' + secref dt = timeobj.shape[0] cftimes = np.zeros((dt), dtype=np.float) for it in range(dt): wrfdates = datetimeStr_conversion(str(timewrfv[it]) +',minutes since ' + \ startdate, 'cfTime', 'matYmdHMS') cftimes[it] = realdatetime1_CFcompilant(wrfdates, refdate, tunitsval) tunits = tunitsval + ' since ' + refdateS if wrfnc.variables.has_key('time'): newvar = wrfnc.variables['time'] else: newvar = wrfnc.createVariable('time','f4',('Time')) newattr = basicvardef(newvar, 'time','time',tunits) newvar[:] = cftimes newattr = set_attribute(newvar, 'calendar', 'standard') wrfnc.sync() wrfnc.close() return def insert_variable(onc, varn, varv, vardims, varvdims, onewnc): """ Function to insert a variable in an existing file insert_variable(onc, varn, varv, vardims, onewnc) [onc]= object of the original file [varn]= name of the variable [varv]= values of the variable [vardims]= dimension names of the variable [varvdims]= name of the variables with the values of the dimensions [onewnc]= object of the new file """ fname = 'insert_variable' if onc == 'h': print fname + '_____________________________________________________________' print insert_variable.__doc__ quit() ininewncdims = onewnc.dimensions ininewncvars = onewnc.variables # Creation of dimensions for idim in vardims: if not ininewncdims.has_key(idim): if not onc.dimensions.has_key(idim): print errormsg print ' ' + fname + ': initial file does not have dimension: "' + \ idim + '"' quit(-1) else: refdimv = onc.dimensions[idim] if refdimv.isunlimited(): dimsize = None else: dimsize = len(refdimv) onewnc.createDimension(idim, dimsize) # Variables with dimension values for ivdim in varvdims: if not ininewncvars.has_key(ivdim): if not onc.variables.has_key(ivdim): print errormsg print ' ' + fname + ': initial file does not have variable: "' + \ ivdim + '"' quit(-1) else: refvdimv = onc.variables[ivdim] refvdimvals = refvdimv[:] for dmn in refvdimv.dimensions: if not onewnc.dimensions.has_key(dmn): print 'dmn:',dmn if onc.dimensions[dmn].isunlimited(): newdim = onewnc.createDimension(dmn, None ) else: newdim = onewnc.createDimension(dmn,len(onc.dimensions[dmn])) newvar = onewnc.createVariable(ivdim, refvdimv.dtype, \ (refvdimv.dimensions)) varattrs = refvdimv.ncattrs() newvar[:] = refvdimvals for attrn in varattrs: attrv = refvdimv.getncattr(attrn) newattr = set_attribute(newvar, attrn, attrv) # Variable varvalues = gen.variables_values(varn) for st in gen.statsurname: if varn.find(st) > -1: print ' '+ fname + ": varibale '" + varn + "' with a " + \ "statistical surname: '",st,"' !!" varvalues[0] = varn break if onewnc.variables.has_key(varvalues[0]): print warnmsg print ' ' + fname + ": file already has variable '" + varvalues[0] + "' !!" print ' re-writting values' newvar = onewnc.variables[varvalues[0]] else: newvar = onewnc.createVariable(varvalues[0], 'f4', (vardims)) newattr = basicvardef(newvar, varvalues[1], varvalues[4].replace('|',' '), \ varvalues[5]) print ' ' + fname + ': shape; newvar',newvar.shape,'varv:',varv.shape newvar[:] = varv return def var_dim_dimv(dimvar, dimns, dimvns): """ Function to bring back which variables with the values should be use for a given variable var_dim_dimv(ovar, dimns, dimvns) [dimvar]= list of the dimensions of the variable [dimns]= names of the list of available dimensions [dimvns]= correspondant name of variables with the values for the dimensions >>> dimsvar = ['t', 'y', 'x'] >>> dimnames = ['t', 'z', 'l', 'y' , 'x'] >>> dimvarnames = ['time', 'pressure', 'soil_levels', 'lat' ,' lon'] >>> var_dim_dimv(dimsvar, dimnames, dimvarnames) ['time', 'lat', ' lon'] """ fname = 'var_dim_dimv' if dimvar == 'h': print fname + '_____________________________________________________________' print var_dim_dimv.__doc__ quit() dimvvar = list(dimvns) Ndimvar = len(dimvar) Ndimns = len(dimns) for indimv in range(Ndimns): for indima in range(Ndimvar): found = False if dimns[indimv] == dimvar[indima]: found = True break if not found: dimvvar.remove(dimvns[indimv]) return dimvvar def WRF_CFlonlat_creation(values, ncfile, varn): """ Function to add a CF-convention longitude/latitude variables in a WRF file WRF_CFlon_creation(values, ncfile, varn) [values]= [lonname],[latname],[chdimname] [lonname],[latname]: names of the variables to add (standard names 'longitude', 'latitude') [chdimname]: whether west_east, south_north dimensions should change names to 'lon','lat' [ncfile]= WRF file to add the CF-convention time variable [varn]= [lonname],[latname] name of the variables to use """ fname='WRF_CFlonlat_creation' if values == 'h': print fname + '_____________________________________________________________' print WRF_CFlonlat_creation.__doc__ quit() cflonn = values.split(',')[0] cflatn = values.split(',')[1] chdimn = gen.Str_Bool(values.split(',')[2]) wlonn = varn.split(',')[0] wlatn = varn.split(',')[1] wrfnc = NetCDFFile(ncfile, 'a') if not gen.searchInlist(wrfnc.variables.keys(),wlonn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not lon variable '" + \ wlonn + "' !!!" quit(-1) if not gen.searchInlist(wrfnc.variables.keys(),wlatn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not lat variable '" + \ wlatn + "' !!!" quit(-1) if not gen.searchInlist(wrfnc.dimensions.keys(),'west_east'): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not 'west_east' dimension !!!" quit(-1) if not gen.searchInlist(wrfnc.dimensions.keys(),'south_north'): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not 'south_north' dimension !!!" quit(-1) lonobj = wrfnc.variables[wlonn] lonv = lonobj[:] latobj = wrfnc.variables[wlatn] latv = latobj[:] if len(lonv.shape) == 3: cflons = np.zeros((lonv.shape[1],lonv.shape[2]), dtype=np.float) cflons = np.where(lonv[0,:,:] < 0., lonv[0,:,:] + 360., lonv[0,:,:]) cflats = np.zeros((latv.shape[1],latv.shape[2]), dtype=np.float) cflats = latv[0,:,:] dimx = lonv.shape[2] dimy = lonv.shape[1] elif len(lonv.shape) == 2: cflons = np.zeros((lonv.shape), dtype=np.float) cflons = np.where(lonv < 0., lonv + 360., lonv) cflats = latv dimx = lonv.shape[1] dimy = lonv.shape[0] elif len(lonv.shape) == 1: cflons = np.zeros((lonv.shape), dtype=np.float) cflons = np.where(lonv < 0., lonv + 360., lonv) cflats = latv dimx = lonv.shape[0] dimy = latv.shape[0] else: print errormsg print ' ' + fname + ": original longitude variable '" + varn + "' shape:" + \ lonv.shape + "not ready !!!" quit(-1) # if len(lonv.shape) == 3: # for ix in range(lonv.shape[2]): # print ix, lonv[0,10,ix], cflons[10,ix] # elif len(lonv.shape) == 2: # for ix in range(lonv.shape[1]): # print ix, lonv[10,ix], cflons[10,ix] # Creation of CF-dimensions if chdimn: wrfnc.renameDimension('west_east', 'lon') wrfnc.renameDimension('south_north', 'lat') else: if not wrfnc.dimensions.has_key('lon'): newdim = wrfnc.createDimension('lon', dimx) if not wrfnc.dimensions.has_key('lat'): newdim = wrfnc.createDimension('lat', dimy) if wrfnc.variables.has_key(cflonn): print warnmsg print ' ' + fname + ": file '" + ncfile + "' already has variable '" + \ cflonn + "' !!!" print ' over-writting values' newvar = wrfnc.variables[cflonn] else: newvar = wrfnc.createVariable(cflonn,'f8',('lat','lon')) newattr = basicvardef(newvar, 'longitude','Longitude','degrees_East') newvar[:] = cflons newattr = set_attribute(newvar, 'axis', 'X') newattr = set_attribute(newvar, '_CoordinateAxisType', 'Lon') if wrfnc.variables.has_key(cflatn): print warnmsg print ' ' + fname + ": file '" + ncfile + "' already has variable '" + \ cflatn + "' !!!" print ' over-writting values' newvar = wrfnc.variables[cflatn] else: newvar = wrfnc.createVariable(cflatn,'f8',('lat','lon')) newattr = basicvardef(newvar, 'latitude','Latitude','degrees_North') newvar[:] = cflats newattr = set_attribute(newvar, 'axis', 'Y') newattr = set_attribute(newvar, '_CoordinateAxisType', 'Lat') wrfnc.sync() wrfnc.close() return #WRF_CFlon_creation('longitude', 'wrfout_d01_1980-03-01_00:00:00_Time_B0-E48-I1.nc', 'XLONG') def dimVar_creation(values, ncfile): """ Function to add a 1D variable with the size of a given dimension in a file dimVar_creation(values, ncfile) [values]= [dimname] name of the dimesion to use [ncfile]= file to add the variable """ fname='dimVar_creation' if values == 'h': print fname + '_____________________________________________________________' print dimVar_creation.__doc__ quit() ncobj = NetCDFFile(ncfile, 'a') varn = 'varDIM' + values if not searchInlist(ncobj.dimensions.keys(),values): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not '" + values + \ "' dimension !!" quit(-1) dimsize = len(ncobj.dimensions[values]) if ncobj.variables.has_key(varn): print warnmsg print ' ' + fname + ": file '" + ncfile + "' already has variable '" + \ varn + "' !!!" print ' over-writting values' newvar = ncobj.variables[varn] else: newvar = ncobj.createVariable(varn,'f4',(values)) newattr = basicvardef(newvar, varn,"variable from '" +values+ "' dimension" ,'-') newvar[:] = np.arange(dimsize)*1. ncobj.sync() ncobj.close() print fname + ": successfull creation of variable '" + varn + "' !!" return #dimVar_creation('bottom_top', 'wrfout_d01_1979-12-01_00:00:00_south_north_B3-E3-I1_west_east_B26-E26-I1.nc') def netcdf_fold_concatenation_HMT(values, ncfile, varn): """ Function to concatenate netCDF files in a given folder for a given set of variables giving Header, Middle, Tail for the name files netcdf_fold_concateation(values, ncfile, varn) [values]= [fold],[dimname] [fold]: folder with the location of the netCDF files [dimname]: dimension along which files should be concatenated [vardimname]: name of the dimension variable with the values [ncfile]= [header],[middle],[tail] [header] = header of the name of the files to concatenate [ncfile]*[middle]*[tail] [middle] = middle of the name of the files to concatenate [ncfile]*[middle]*[tail] [tail] = tail of the name of the files to concatenate [ncfile]*[middle]*[tail] [varn]= ',' separated list of variables to concatenate [var1],[var2],[...[varN]] or 'all' for all variables """ import subprocess as sub fname='netcdf_fold_concatenation_HMT' ofile = 'netcdf_fold_concatenated_HMT.nc' if values == 'h': print fname + '_____________________________________________________________' print netcdf_fold_concatenation_HMT.__doc__ quit() fold = values.split(',')[0] condim = values.split(',')[1] vardimname = values.split(',')[2] HMT = ncfile.split(',') confiles = gen.files_folder_HMT(fold, HMT[0], HMT[1], HMT[2]) Nfiles = len(confiles) print ' ' + fname +': concatenating:', Nfiles, 'files' if Nfiles == 0: print errormsg print ' ' +fname+ ": there are no files as '" + fold + '/' + '*'.join(HMT) +\ "*' !!" quit(-1) print confiles # Opening all files ncobjs = [] for filen in confiles: print 'charging: ',filen if filen.find(fold) != -1: ncobjs.append(NetCDFFile(filen, 'r')) else: ncobjs.append(NetCDFFile(fold + '/' + filen, 'r')) # Looking for the new length of the concatenation dimension if not ncobjs[0].dimensions.has_key(condim): print errormsg print ' ' + fname + ": files do not have dimensions '" + condim + "' !!!" quit(-1) if not ncobjs[0].variables.has_key(vardimname): print errormsg print ' ' + fname + ": files do not have variable '" + vardimname + "' !!!" quit(-1) totcondim = 0 for ifile in range(Nfiles): ncobj = ncobjs[ifile] totcondim = totcondim + len(ncobjs[ifile].dimensions[condim]) print "total concatenated dimension '" + condim + "': ", totcondim # concatenated file creation ## newnc = NetCDFFile(ofile, 'w') # dimension creation ## if ncobjs[0].dimensions[condim].isunlimited(): newnc.createDimension(condim, None) else: newnc.createDimension(condim, totcondim) newnc.createDimension('file', Nfiles) newnc.createDimension('Lstring', 256) # dim-variables newvar = newnc.createVariable('file','c',('file','Lstring')) newvals = writing_str_nc(newvar, confiles, 256) if confiles[0].find(fold) != -1: str="concatenated files following dimension '" + condim + "'" else: str="concatenated files from folder '" + fold + "' following dimension '" + \ condim + "'" vardef = basicvardef(newvar, 'files', str, '-') # Fake variable for the dimension print 'condim:',condim dims = [] dims.append(condim) newvar = newnc.createVariable('int' + condim, 'i4', tuple(dims)) newvar[:] = range(totcondim) # Dimension variable for the concatenation odimvar = ncobjs[0].variables[vardimname] odimvardims = odimvar.dimensions odimattrs = odimvar.ncattrs() for dimn in odimvardims: if not gen.searchInlist(newnc.dimensions,dimn): Ldim = len(odimvar.dimensions[dimn]) newvar = newnc.createVariable(vardimname, odimvar.dtype, odimvardims) for attrn in odimattrs: attrv = odimvar.getncattr(attrn) newattr = set_attribute(newvar, attrn, attrv) if attrn == 'units': urefvals = attrv # Checking units of condimvar uvalsecs = urefvals.split(' ') if uvalsecs[1] == 'since': tunits = True if len(uvalsecs) == 4: date=uvalsecs[2] time=uvalsecs[3] else: if uvalsecs[2].find('_') != -1: date=uvalsecs[2] time='00:00:00' else: date=uvalsecs[2].split('_')[0] time=uvalsecs[2].split('_')[1] refdate = gen.datetimeStr_datetime(date+'_'+time) print warnmsg print ' ' + fname + ': concatenating dimension-variable has time units!' print ' shifting temporal-values since the date of the first file', refdate begslicetot = 0 for ifile in range(Nfiles): slicevartot = [] if not ncobjs[ifile].variables.has_key(vardimname): print errormsg print ' ' + fname + ": file '" + confiles[ifile] + "' does not have " + \ " dimension-variable '" + vardimname + "' !!" quit(-1) objvar = ncobjs[ifile].variables[vardimname] for dimn in objvar.dimensions: ldimfile = len(ncobjs[ifile].dimensions[dimn]) if dimn == condim: slicevartot.append(slice(begslicetot,begslicetot+ldimfile)) begslicetot = begslicetot + ldimfile else: slicevartot.append(slice(0,ldimfile)) if tunits: uvals = objvar.getncattr('units') #urefvals, objvals = gen.check_timestep(odimvar) #uvals, objvals = gen.check_timestep(objvar) newtimes = gen.coincident_CFtimes(objvar[:], urefvals, uvals) newvar[tuple(slicevartot)] = newtimes[:] else: newvar[tuple(slicevartot)] = objvar[:] newnc.sync() # Looping variables ## if varn == 'all': desvars = ncobjs[0].variables.keys() else: desvars = varn.split(',') for dvar in desvars: if not ncobjs[0].variables.has_key(dvar): print errormsg print ' ' + fname + ": files do not have variable '" + dvar + "' !!!" quit(-1) if gen.searchInlist(desvars, vardimname): desvars.remove(vardimname) for dvar in desvars: print ' ' + fname + ": concatenating '" + dvar + "'..." objvar = ncobjs[0].variables[dvar] # creation of dimensions vardims = objvar.dimensions for ndim in vardims: if not newnc.dimensions.has_key(ndim): vardimo = ncobjs[0].dimensions[ndim] if vardimo.isunlimited(): newnc.createDimension(ndim, None) else: newnc.createDimension(ndim, len(vardimo)) # creation of variable kvar = objvar.dtype newvar = newnc.createVariable(dvar, kvar, objvar.dimensions) varattrs = objvar.ncattrs() for atr in varattrs: attrv = objvar.getncattr(atr) newattr = set_attribute(newvar, atr, attrv) # variable values # vartotv = np.zeros((newvar.shape), dtype=kvar) objvar = ncobjs[0].variables[dvar] if gen.searchInlist(list(objvar.dimensions),condim): begslicetot = 0 for ifile in range(Nfiles): slicevartot = [] if ncobjs[ifile].variables.has_key(dvar): objvar = ncobjs[ifile].variables[dvar] for dimn in objvar.dimensions: ldimfile = len(ncobjs[ifile].dimensions[dimn]) if dimn == condim: slicevartot.append(slice(begslicetot,begslicetot+ldimfile)) begslicetot = begslicetot + ldimfile else: slicevartot.append(slice(0,ldimfile)) newvar[tuple(slicevartot)] = objvar[:] newnc.sync() else: print errormsg print ' ' + fname + ": file '" + fold + '/' + confiles[ifile] + \ "' does not have variable '" + dvar + "' !!" quit(-1) else: newvar[:] = objvar[:] newnc.sync() # newvar[:] = vartotv gattrs = ncobjs[0].ncattrs() for attr in gattrs: attrv = ncobjs[0].getncattr(attr) newattr = set_attribute(newnc, attr, attrv) for ifile in range(Nfiles): ncobjs[ifile].close() newnc.sync() add_global_PyNCplot(newnc, main, fname, '1.1') newnc.close() print "Successfull creation of concatenated file '" + ofile + "' !!!" return def netcdf_fold_concatenation(values, ncfile, varn): """ Function to concatenate netCDF files in a given folder for a given set of variables netcdf_fold_concateation(values, ncfile, varn) [values]= [fold],[dimname] [fold]: folder with the location of the netCDF files [dimname]: dimension along which files should be concatenated [tail]: tail of the files to concatenate (optional) [ncfile]= header of the name of the files to concatenate [ncfile]* [varn]= ',' separated list of variables to concatenate [var1],[var2],[...[varN]] or 'all' for all variables """ import subprocess as sub fname='netcdf_fold_concatenate' ofile = 'netcdf_fold_concatenated.nc' if values == 'h': print fname + '_____________________________________________________________' print netcdf_fold_concatenation.__doc__ quit() fold = values.split(',')[0] condim = values.split(',')[1] confiles = gen.files_folder(fold,ncfile) Nfiles = len(confiles) print ' ' + fname +': concatenating:', Nfiles, 'files' if Nfiles == 0: print errormsg print ' ' + fname + ": there are no files as '" + fold + '/' + ncfile + "*' !!" quit(-1) print confiles # Opening all files ncobjs = [] for filen in confiles: print 'charging: ',filen ncobjs.append(NetCDFFile(fold + '/' + filen, 'r')) # Looking for the new length of the concatenation dimension if not ncobjs[0].dimensions.has_key(condim): print errormsg print ' ' + fname + ": files do not have dimensions '" + condim + "' !!!" quit(-1) totcondim = 0 for ifile in range(Nfiles): ncobj = ncobjs[ifile] totcondim = totcondim + len(ncobjs[ifile].dimensions[condim]) print "total concatenated dimension '" + condim + "': ", totcondim # concatenated file creation ## newnc = NetCDFFile(ofile, 'w') # dimension creation ## if ncobjs[0].dimensions[condim].isunlimited(): newnc.createDimension(condim, None) else: newnc.createDimension(condim, totcondim) # Fake variable for the dimension print 'condim:',condim dims = [] dims.append(condim) newvar = newnc.createVariable('int' + condim, 'i4', tuple(dims)) newvar[:] = range(totcondim) # Looping variables ## if varn == 'all': desvars = ncobjs[0].variables else: desvars = varn.split(',') for dvar in desvars: if not ncobjs[0].variables.has_key(dvar): print errormsg print ' ' + fname + ": files do not have variable '" + dvar + "' !!!" quit(-1) for dvar in desvars: print ' ' + fname + ": concatenating '" + dvar + "'..." objvar = ncobjs[0].variables[dvar] # creation of dimensions vardims = objvar.dimensions for ndim in vardims: if not newnc.dimensions.has_key(ndim): vardimo = ncobjs[0].dimensions[ndim] if vardimo.isunlimited(): newnc.createDimension(ndim, None) else: newnc.createDimension(ndim, len(vardimo)) # creation of variable kvar = objvar.dtype newvar = newnc.createVariable(dvar, kvar, objvar.dimensions) varattrs = objvar.ncattrs() for atr in varattrs: attrv = objvar.getncattr(atr) newattr = set_attribute(newvar, atr, attrv) # variable values # vartotv = np.zeros((newvar.shape), dtype=kvar) objvar = ncobjs[0].variables[dvar] if gen.searchInlist(list(objvar.dimensions),condim): begslicetot = 0 for ifile in range(Nfiles): slicevartot = [] if ncobjs[ifile].variables.has_key(dvar): objvar = ncobjs[ifile].variables[dvar] for dimn in objvar.dimensions: ldimfile = len(ncobjs[ifile].dimensions[dimn]) if dimn == condim: slicevartot.append(slice(begslicetot,begslicetot+ldimfile)) begslicetot = begslicetot + ldimfile else: slicevartot.append(slice(0,ldimfile)) newvar[tuple(slicevartot)] = objvar[:] newnc.sync() else: print errormsg print ' ' + fname + ": file '" + fold + '/' + confiles[ifile] + \ "' does not have variable '" + dvar + "' !!" quit(-1) else: newvar[:] = objvar[:] newnc.sync() # newvar[:] = vartotv gattrs = ncobjs[0].ncattrs() for attr in gattrs: attrv = ncobjs[0].getncattr(attr) newattr = set_attribute(newnc, attr, attrv) for ifile in range(Nfiles): ncobjs[ifile].close() newnc.sync() newnc.close() print "Successfull creation of concatenated file '" + ofile + "' !!!" return #netcdf_fold_concatenation('/media/data2/etudes/WRF_LMDZ/WL_HyMeX_HighRes/medic950116,Time', 'wrfout_d03', 'Times,XLONG,XLAT,T2,TH2,PSFC,U10,V10,MAPFAC_M') def netcdf_concatenation(ncfile): """ Function to concatenate netCDF files for a given set of variables netcdf_concateation(values, ncfile, varn) [ncfile]= [filename1]@[varn1]|[filename1]@[varn1] '|' list of file names and variables to concatenate [varn]= ',' separated list of variables to concatenate [var1],[var2],[...[varN]] or 'all' for all variables """ fname='netcdf_concatenate' ofile = 'netcdf_concatenated.nc' if ncfile == 'h': print fname + '_____________________________________________________________' print netcdf_concatenation.__doc__ quit() confiles = ncfile.split('|') Nfiles = len(confiles) print ' ' + fname +': concatenating:',Nfiles,'files' print confiles # Opening all files filenames = [] ncobjs = [] for files in confiles: filen = files.split('@')[0] filenames.append(filen) if not os.path.isfile(filen): print errormsg print ' ' + fname + ": file '" + filen + "' does no exist!!" quit(-1) print 'charging: ',filen ncobjs.append(NetCDFFile(filen, 'r')) # concatenated file creation ## newnc = NetCDFFile(ofile, 'w') # Looping variables ## ifile = 0 for files in confiles: filen = files.split('@')[0] desvar = files.split('@')[1] if desvar == 'all': desvars = ncobjs[ifile].variables else: desvars = desvar.split(',') for dvar in desvars: if not ncobjs[ifile].variables.has_key(dvar): print errormsg print ' ' + fname + ": files do not have variable '" +dvar+ "' !!!" quit(-1) for dvar in desvars: if not newnc.variables.has_key(dvar): print ' ' + fname + ": concatenating '" + dvar + "' from '" + filen+\ "'..." objvar = ncobjs[ifile].variables[dvar] # creation of dimensions vardims = objvar.dimensions for ndim in vardims: if not newnc.dimensions.has_key(ndim): vardimo = ncobjs[ifile].dimensions[ndim] if vardimo.isunlimited(): newnc.createDimension(ndim, None) else: newnc.createDimension(ndim, len(vardimo)) # creation of variable kvar = objvar.dtype newvar = newnc.createVariable(dvar, kvar, objvar.dimensions) varattrs = objvar.ncattrs() for atr in varattrs: attrv = objvar.getncattr(atr) newattr = set_attribute(newvar, atr, attrv) # variable values # vartotv = np.zeros((newvar.shape), dtype=kvar) objvar = ncobjs[ifile].variables[dvar] newvar[:] = objvar[:] newnc.sync() # newvar[:] = vartotv ifile = ifile + 1 gattrs = ncobjs[0].ncattrs() for attr in gattrs: attrv = ncobjs[0].getncattr(attr) newattr = set_attribute(newnc, attr, attrv) filesnvec = numVector_String(filenames,', ') newattr = set_attribute(newnc, 'concatenated', filesnvec) for ifile in range(Nfiles): ncobjs[ifile].close() newnc.sync() newnc.close() print "Successfull creation of concatenated file '" + ofile + "' !!!" return #netcdf_concatenation('mean_pluc.nc@all|mean_dtcon.nc@all') def field_stats(values, ncfile, varn): """ Function to retrieve statistics from a field field_stats(values, ncfile, varn) [values]= [stats],[fillVals],[countVals] [stats]: kind of statistics 'full': all statistics given variable [fillVals]: ':' list of _fillValues ('None' for any) [countVals]: ':' list of Values@cond to use to count conditions ('None' for any) [ncfile]= name of the netCDF file to use [varn]= variable name to use ('all' for all variables) """ import numpy.ma as ma fname='field_stats' if values == 'h': print fname + '_____________________________________________________________' print field_stats.__doc__ quit() arguments = '[stats],[fillVals],[coutnVals]' gen.check_arguments(fname,values,arguments,',') stats=values.split(',')[0] fillVals=values.split(',')[1] countVals=values.split(',')[2] # Fill Values if fillVals == 'None': fillV = None else: if fillVals.find(':') != -1: fillV = fillVals.split(':') NfillV = len(fillV) else: fillV = [fillVals] NfillV = 1 # Count Values if countVals == 'None': countV = None NcountV = 0 else: if countVals.find(':') != -1: countV = countVals.split(':') NcountV = len(countV) else: countV = [countVals] NcountV = 1 ncobj = NetCDFFile(ncfile, 'r') if varn == 'all': varstats = list(ncobj.variables) else: varstats = [varn] statsvariable = {} print ' ' + fname + " file: '" + ncfile + "'..." print ' ' + fname + '______ ______ _____ ____ ___ __ _' for vn in varstats: if not ncobj.variables.has_key(vn): print errormsg print ' ' + fname + ": file do not have variable '" + vn + "' !!" quit(-1) objfield = ncobj.variables[vn] field = objfield[:] dtype = objfield.dtype counts = [] if countV is not None: for icV in range(NcountV): cV = countV[icV].split('@')[0] cC = countV[icV].split('@')[1] countval = gen.retype(cV, dtype) counts.append([gen.count_cond(field, countval, cC), cC, countval]) if fillV is not None: for ifV in range(NfillV): fillval = gen.retype(fillV[ifV], dtype) field = ma.masked_equal(field, fillval) print ' ' + vn + '... .. .' if stats == 'full': minv = np.min(field) maxv = np.max(field) meanv = np.mean(field) mean2v = np.mean(field*field) #varv = np.sqrt(mean2v - meanv*meanv) varv = np.var(field) print ' Fstats shape:',field.shape print ' Fstats min:', minv print ' Fstats max:', maxv print ' Fstats mean:', meanv print ' Fstats mean2:', mean2v print ' Fstats variability:', varv if countV is not None: for icV in range(NcountV): convals = counts[icV] print " Fstats N values '" + convals[1] + "'", convals[2], \ ':', convals[0] contvalsS = 'count(', + convals[1] + ',',convals[2],')' else: convals = [int(0)] convalsS = '' statsvariable[vn] = [minv, maxv, meanv, mean2v, varv, convals[0]] else: print errormsg print ' ' + fname + ": kind of statistics '" + values + "' not ready!!" print ' valid statistics: full' quit(-1) ncobj.close() stsn = ['varn', 'min', 'max', 'mean', 'mean2', 'var', convalsS] Sm = 'MAT: ' print ' ' print Sm + 'Matrix of the statisitcs _______' print Sm + ' '.join('{:<15}'.format(vn) for vn in stsn) for key in statsvariable.keys(): vals = statsvariable[key] print Sm+'{:<15}'.format(key)+' '+' '.join('{:<15g}'.format(v) for v in vals) return #field_stats('full', 'geo_em.d01.nc', 'HGT_M') def field_stats_dim(values, ncfile, varn): """ Function to retrieve statistics from a field along dimensions field_stats_dim(values, ncfile, varn) [values]= [stats],[fillVals],[countVals],[dimns],[vardims],[stdout] [stats]: kind of statistics 'full': all statistics given variable (min, max, mean, mean2, variability, [countVals]) [fillVals]: ':' list of _fillValues ('None' for any) [countVals]: ':' list of Values@cond to use to count conditions ('None' for any) [dims]: ':', list of names of the dimension along which statistics is required [vardims]: ':', list of names of the variable with the values for the dimension along which statistics is required [stdout]: Whether should be printed in standard output the results ('True'/'False') [ncfile]= name of the netCDF file to use [varn]= variable name to use ('all' for all variables) """ import numpy.ma as ma fname='field_stats_dim' if values == 'h': print fname + '_____________________________________________________________' print field_stats_dim.__doc__ quit() Lstr = 100 fullstats = ['min', 'max', 'mean', 'mean2', 'variability'] arguments = '[stats],[fillVals],[coutnVals],[dims],[vardims],[stdout]' gen.check_arguments(fname,values,arguments,',') ofile = fname + '.nc' stats=values.split(',')[0] fillVals=values.split(',')[1] countVals=values.split(',')[2] dims=gen.str_list(values.split(',')[3],':') vardims=gen.str_list(values.split(',')[4],':') stdout=gen.Str_Bool(values.split(',')[5]) # Fill Values if fillVals == 'None': fillV = None else: if fillVals.find(':') != -1: fillV = fillVals.split(':') NfillV = len(fillV) else: fillV = [fillVals] NfillV = 1 # Statistics if stats == 'full': statns = fullstats else: print errormsg print ' ' + fname + ": kind of statistics '" + stats + "' not ready!!" print ' valid statistics: full' quit(-1) # Count Values if countVals == 'None': countV = None NcountV = 0 else: if countVals.find(':') != -1: countV = countVals.split(':') NcountV = len(countV) for cV in countV: statns = statns.append('count_' + str(cV)) else: countV = [countVals] NcountV = 1 statns = statns.append('count_' + str(countV)) Nstats = len(statns) ncobj = NetCDFFile(ncfile, 'r') for dim in dims: if not gen.searchInlist(ncobj.dimensions.keys(), dim): print errormsg print ' ' +fname+ ": file '" + ncfile + "' does not have dimension '" + \ dim + "' !!" quit(-1) for vardim in vardims: if not gen.searchInlist(ncobj.variables.keys(), vardim): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have variable-" + \ "dimension '" + vardim + "' !!" quit(-1) if varn == 'all': varstats = list(ncobj.variables.keys()) else: varstats = [varn] # File creation onewnc = NetCDFFile(ofile, 'w') # Dimensions stdims = {} rundims = [] runndims = [] dimslice = {} for dim in dims: add_dims(ncobj,onewnc,[dim]) stdims[dim] = len(ncobj.dimensions[dim]) runndims.append(dim) rundims.append(stdims[dim]) dimslice[dim] = 0 onewnc.createDimension('stats',Nstats) onewnc.createDimension('Lstring',Lstr) # Dimension variables add_vars(ncobj,onewnc,vardims) for vardn in vardims: if gen.searchInlist(varstats,vardn): varstats.remove(vardn) # Statistics var newvar = onewnc.createVariable('stats','c',('stats','Lstring')) vardef = basicvardef(newvar, 'stats', 'statistics along running dimensions','-') newvals = writing_str_nc(newvar, statns, Lstr) onewnc.sync() for Vn in varstats: print ' ' + fname + ": statistics of '" + Vn + "' ..." statsvariable = {} Ntotrun = 1 if stdout: print ' ' + fname + " file: '" + ncfile + "'..." print ' ' + fname + '______ ______ _____ ____ ___ __ _' for vn in varstats: if not ncobj.variables.has_key(Vn): print errormsg print ' ' + fname + ": file do not have variable '" + Vn + "' !!" quit(-1) objfield = ncobj.variables[Vn] dtype = objfield.dtype # Dimensions to run along runvdimns = [] runvLdims = [] for vdn in objfield.dimensions: if gen.searchInlist(runndims,vdn): Ntotrun = Ntotrun * stdims[vdn] runvdimns.append(vdn) runvLdims.append(stdims[vdn]) statsresults = np.zeros(tuple(runvLdims+[Nstats]), dtype=np.float) if len(runvLdims) == 0: print warnmsg print ' ' + fname + ': variable without running dimension!' print ' computing statistics for the whole variable' Ntotrun = 1 for idd in range(Ntotrun): # Getting variable slice at this iteration if len(runvLdims) > 0: slicedims = gen.index_flatten_mat(idd,runvLdims) else: slicedims = [] slicevar = [] for vdn in objfield.dimensions: runfound = False Srundim = '' for irvdn in range(len(runvdimns)): if vdn == runvdimns[irvdn]: slicevar.append(slicedims[irvdn]) runfound = True Srundim = Srundim + vdn + ': ' + str(slicedims[irvdn]) + ' ' break if not runfound: slicevar.append(slice(0,len(ncobj.dimensions[vdn]))) # Getting field values field = objfield[tuple(slicevar)] # Computing statistics counts = [] if countV is not None: for icV in range(NcountV): cV = countV[icV].split('@')[0] cC = countV[icV].split('@')[1] countval = gen.retype(cV, dtype) counts.append([gen.count_cond(field, countval, cC), cC, countval]) if fillV is not None: for ifV in range(NfillV): fillval = gen.retype(fillV[ifV], dtype) field = ma.masked_equal(field, fillval) if stdout: print ' ' + Vn + '[' + Srundim + ']... .. .' if stats == 'full': minv = np.min(field) maxv = np.max(field) meanv = np.mean(field) mean2v = np.mean(field*field) #varv = np.sqrt(mean2v - meanv*meanv) varv = np.var(field) if stdout: print ' Fstats shape:',field.shape print ' Fstats min:', minv print ' Fstats max:', maxv print ' Fstats mean:', meanv print ' Fstats mean2:', mean2v print ' Fstats variability:', varv if countV is not None: for icV in range(NcountV): convals = counts[icV] if stdout: print " Fstats N values '" + convals[1] + "'", \ convals[2], ':', convals[0] contvalsS = 'count(', + convals[1] + ',',convals[2],')' else: convals = [int(0)] convalsS = '' statsvariable[vn] = [minv, maxv, meanv, mean2v, varv, convals[0]] if countV is not None: statsresults[tuple(list(slicedims)+[slice(0,Nstats)])] = [minv, \ maxv, meanv, mean2v, varv, convals[0]] else: statsresults[tuple(list(slicedims)+[slice(0,Nstats)])] = [minv, \ maxv, meanv, mean2v, varv] else: print errormsg print ' ' + fname + ": kind of statistics '" + stats + "' not ready!!" print ' valid statistics: full' quit(-1) Sm = 'MAT: ' if stdout: print ' ' print Sm + 'Matrix of the statisitcs _______' print Sm + ' '.join('{:<15}'.format(Vn) for vn in statns) for key in statsvariable.keys(): vals = statsvariable[key] print Sm+'{:<15}'.format(key)+' '+' '.join('{:<15g}'.format(v) for v in vals) newvar = onewnc.createVariable(Vn+'_stats', 'f4', tuple(runvdimns+['stats'])) newvar[:] = statsresults[:] for attrn in objfield.ncattrs(): attrv = objfield.getncattr(attrn) if attrn != '_FillValue': newattr = set_attribute(newvar,attrn,attrv) newattr = set_attribute(newvar,'running_stats',', '.join(runvdimns)) onewnc.sync() # Global attributes add_globattrs(ncobj,onewnc,'all') onewnc.sync() ncobj.close() onewnc.close() print fname + ": successfull written of file '" + ofile + "' !!" return #field_stats_dim('full,None,None,west_east,XLONG,False', '/home/lluis/PY/wrfout_d01_2001-11-11_00:00:00', 'T2') def filter_2dim(values, ncfile, varn): """ Function to filter along 2 dimensions (moving grid-box means of a given size) values of the netCDF file filter_2dim(values, ncfile, variable) values= [Ngrid],[dimnh1],[dimnh2],[dimvnh1],[dimvnh2] Ngrid: box size to make means dimnh[1/2]: name of the 2 dimensions to make the averages dimvnh[1/2]: name of the 2 variables with the values of the dimensions to make the averages ncfile= netCDF file to use varn= name of the variable to filter ('all' for all variables) filter_2dim('80,y,x,lon,lat', 'tahiti_5m_ll.grd', 'z') """ fname = 'filter_2dim' if values == 'h': print fname + '_____________________________________________________________' print filter_2dim.__doc__ quit() expectargs = '[Ngrid],[dimnh1],[dimnh2],[dimvnh1],[dimvnh2]' check_arguments(fname,values,expectargs,',') Ngrid = int(values.split(',')[0]) dimnh1 = values.split(',')[1] dimnh2 = values.split(',')[2] dimvnh1 = values.split(',')[3] dimvnh2 = values.split(',')[4] Ngrid2 = Ngrid/2 dimhorns = [dimnh1, dimnh2] dimvhorns = [dimvnh1, dimvnh2] if varn == 'all': varns = ncobj.variables ofile = 'filter_hor.nc' else: varns = [varn] ofile = 'filter_hor_' + varn + '.nc' ncobj = NetCDFFile(ncfile,'r') newnc = NetCDFFile(ofile, 'w') for vn in varns: varvobj = ncobj.variables[vn] vardims = varvobj.dimensions print ' ' + fname + ' var:', vn, 'variable shape:', varvobj.shape for dimn in vardims: dimobj = ncobj.dimensions[dimn] if not newnc.dimensions.has_key(dimn): if dimobj.isunlimited(): newnc.createDimension(dimn, None) else: newnc.createDimension(dimn, len(dimobj)) if searchInlist(varvobj.ncattrs(),'_FillValue'): vfillValue = varvobj._FillValue else: vfillValue = fillValue vtype = varvobj.dtype if searchInlist(dimvhorns,vn): print ' ' + fname + ': dimension values variable!' print " keeping original values as '" + vn + "' and filtered as '" + \ vn+'_flt'+str(Ngrid)+'x'+str(Ngrid) + "'" newvar = newnc.createVariable(vn, vtype, vardims) newvar[:] = varvobj[:] vattr = varvobj.ncattrs() for nattr in vattr: attrv = varvobj.getncattr(nattr) if nattr != '_FillValue': newattr = set_attribute(newvar,nattr,attrv) newvar = newnc.createVariable(vn+'_flt'+str(Ngrid)+'x'+str(Ngrid), \ vtype, vardims) else: newvar = newnc.createVariable(vn, vtype, vardims, fill_value = fillValue) if searchInlist(vardims,dimnh1) or searchInlist(vardims,dimnh2): varv = varvobj[:] varfilt = varvobj[:] dimh1id = -1 dimh2id = -1 if searchInlist(vardims,dimnh1): dimh1id= vardims.index(dimnh1) if searchInlist(vardims,dimnh2): dimh2id= vardims.index(dimnh2) if dimh1id != -1 and dimh2id != -1: print ' ' + fname + ': 2D filtered variable' dh1 = varvobj.shape[dimh1id] dh2 = varvobj.shape[dimh2id] for i in range(Ngrid2,dh1-Ngrid2+1): for j in range(Ngrid2,dh2-Ngrid2+1): percendone(i*(dh1-Ngrid)*1.+j,(dh1-Ngrid)*(dh2-Ngrid)*1., 1., \ vn + ' filtered: ' + str(i) + ', ' + str(j)) varslice = [] varslicesum = [] for dimn in vardims: if dimn == dimnh1: varslicesum.append(slice(i-Ngrid2,i+Ngrid2+1)) varslice.append(slice(i,i+1)) elif dimn == dimnh2: varslicesum.append(slice(j-Ngrid2,j+Ngrid2+1)) varslice.append(slice(j,j+1)) else: varslicesum.append(slice(0, \ len(ncobj.dimensions[dimn]))) varslice.append(slice(0, len(ncobj.dimensions[dimn]))) # Doing nothing for _FillValue values varvalssum = varv[tuple(varslicesum)] if varvalssum.mask.all(): varfilt[tuple(varslice)] = fillValue else: numberValues = len(varvalssum.compressed()) if dimh1id < dimh2id: varfilt[tuple(varslice)] = np.sum(np.sum(varvalssum, \ axis=dimh1id), axis=dimh2id-1)/(numberValues*1.) else: varfilt[tuple(varslice)] = np.sum(np.sum(varvvlssum, \ axis=dimh2id), axis=dimh1id-1)/(numberValues*1.) print ' ' + fname + ': 2D Finished!!' elif dimh1id != -1 and dimh2id == -1: print ' ' + fname + ': 1D filtered variable' for i in range(Ngrid2,varvobj.shape[dimh1id]-Ngrid2+1): percendone(i,(dh1-Ngrid), 1, vn + ' filtered') varslice = [] varslicesum = [] for dimn in vardims: if dimn == dimhn1: varslicesum.append(slice(i-Ngrid2,i+Ngrid2+1)) else: varslicesum.append(slice(0, len(ncobj.dimensions[dimn]))) varvalssum = varv[tuple(varslicesum)] if varvalssum.mask.all(): varfilt[tuple(varslice)] = fillValue else: numberValues = len(varvalssum.compressed()) varfilt[tuple(varslice)] = np.sum(varvalssum, \ axis=dimh1id)/numberValues*1. print ' ' + fname + ': 1D Finished!!' elif dimh2id != -1 and dimh1id == -1: print ' ' + fname + ': 1D filtered variable' for j in range(Ngrid2,varvobj.shape[dimh2id]-Ngrid2+1): percendone(j,(dh2-Ngrid), 1, vn + ' filtered') varslice = [] varslicesum = [] for dimn in vardims: if dimn == dimhn2: varslicesum.append(slice(j-Ngrid2,j+Ngrid2+1)) else: varslicesum.append(slice(0, len(ncobj.dimensions[dimn]))) varvalssum = varv[tuple(varslicesum)] if varvalssum.mask.all(): varfilt[tuple(varslice)] = fillValue else: numberValues = len(varvalssum.compressed()) varfilt[tuple(varslice)] = np.sum(varvalssum, \ axis=dimh2id)/numberValues print ' ' + fname + ': 1D Finished!!' newvar[:] = varfilt else: newvar[:] = varvobj[:] vattr = varvobj.ncattrs() for nattr in vattr: attrv = varvobj.getncattr(nattr) if nattr != '_FillValue': newattr = set_attribute(newvar, nattr, attrv) gattr = ncobj.ncattrs() for nattr in gattr: attrv = ncobj.getncattr(nattr) newattr = set_attribute(newnc, nattr, attrv) ncobj.close() newnc.sync() newnc.close() print fname + ": successfull creation of '" + ofile + "' !!!" return #filter_2dim('80,y,x,lon,lat', 'tahiti_5m_ll.grd', 'z') def ncreplace(values, ncfile, varn): """ Function to replace something from a netCDF file values= [kind],[kindn],[netcdffileref] kind: element to substitute 'var': variable 'vattr': variable attributes 'gattr': global attributes kindn: name of the 'something' to replace ('all', for all possible values, [varn]@[attrn] for attribute of a given variable) netcdffileref: netCDF file from which replace values will be taken ncfile= netCDF file to replace something varn= name to use (accordingly to '[kind]') ncreplace('var,HGT_M,Polynesie_filtered_nn.nc', 'geo_em.d04.nc', 'HGT_M') """ fname = 'ncreplace' if values == 'h': print fname + '_____________________________________________________________' print ncreplace.__doc__ quit() expectargs = '[kind],[kindn],[netcdffileref]' check_arguments(fname,values,expectargs,',') kind = values.split(',')[0] kindn = values.split(',')[1] netcdffileref = values.split(',')[2] print ' ' + fname + ": replacing '" + kind + "' in '" + ncfile + "' using '" + \ netcdffileref + "' ..." if not os.path.isfile(netcdffileref): print errormsg print ' ' + fname + ": reference file '" + netcdffileref + \ "' does not exist !!" quit(-1) ncrefobj = NetCDFFile(netcdffileref, 'r') ncobj = NetCDFFile(ncfile, 'a') if kind == 'var' and kindn != 'all': if not ncrefobj.variables.has_key(kindn): print errormsg print ' ' + fname + ": reference file does not have variable '" + kindn \ + "' !!" print ' it has: ',ncrefobj.variables.keys() quit(-1) elif kind == 'vattr' and kindn != 'all': if kindn.index('@') == -1: print errormsg print ' ' + fname + ": an pair [var]@[attrn] is not given! '" + kindn + \ + "' !!" quit(-1) varn = kindn.split('@')[0] attrn = kindn.split('@')[1] if not ncrefobj.variables.has_key(varn): print errormsg print ' ' + fname + ": reference file does not have variable '" + varn +\ "' !!" quit(-1) varobj = ncrefobj.variables[varn] if not searchInlist(varobj.ncattrs(), attrn): print errormsg print ' ' + fname + ": reference variable '" + varn + \ "' does not have attribute '" + attrn + "' !!" quit(-1) else: print errormsg print ' ' + fname + ": kind to replace '" + kind + "' not ready !!" quit(-1) if kind == 'var': if kindn == 'all': varrefns = ncrefobj.variables varns = ncobj.variables else: varrefns = [kindn] varns = [varn] # On kindn= 'all' & varn= 'all' one has to assume same variables' names for vnref in varrefns: if not ncrefobj.variables.has_key(vnref): print errormsg print ' ' + fname + ": reference file does not have variable '" + \ vnref + "' !!" quit(-1) for vn in varns: if not ncobj.variables.has_key(vn): print errormsg print ' ' + fname + ": file does not have variable '" + vn + \ "' !!" quit(-1) if kindn == 'all' and vnref == vn: print ' ' + fname + ": replacing all now '" + vn + "' with '"+\ vnref + "' ..." varv = ncobj.variables[vn] varrefv = ncrefobj.variables[vnref][:] varv[:] = varrefv ncobj.sync() break else: vdimns = numVector_String(ncobj.variables[vn].shape,':') vrefdimns = numVector_String(ncrefobj.variables[vnref].shape,':') if vdimns != vrefdimns: print errormsg print ' ' + fname + ': values can not be replaced!' print ' different shapes! original:', \ ncobj.variables[vn].shape,'reference:', \ ncrefobj.variables[vnref].shape # In case of HGT_M... # quit(-1) quit(-1) print ' ' + fname + ": replacing '" + vn + "' with '" + \ vnref + "' ..." varv = ncobj.variables[vn] varrefv = ncrefobj.variables[vnref][:] # In case of HGT_M... varv[0,:,:] = varrefv varv[:] = varrefv ncobj.sync() else: print errormsg print ' ' + fname + ": kind to replace '" + kind + "' not exactly ready !!" quit(-1) ncobj.sync() ncobj.close() ncrefobj.close() return def ncstepdiff(values, ncfile, varn): """ Function to compute differencies between time-steps (deacumulate) a netCDF file new variables labelled [varn] + 'de'will be added to the file values= [tdimn],[tvarn] tdimn: name of the dimension time tvarn: name of the variable time ncfile= netCDF file to replace something varn= ',' list of variables to deaccumulate ('all' for all) ncstepdiff('Time,Times', '/home/lluis/PY/wrfout_d01_2001-11-11_00:00:00.tests', 'Q2') """ fname = 'ncstepdiff' if values == 'h': print fname + '_____________________________________________________________' print ncstepdiff.__doc__ quit() expectargs = '[tdimn],[tvarn]' check_arguments(fname,values,expectargs,',') tdimn = values.split(',')[0] tvarn = values.split(',')[1] onc = NetCDFFile(ncfile, 'a') if varn.find(',') != -1: if varn != 'all': varns = varn.split(',') else: varns = onc.variables else: varns = [varn] otdim = onc.dimensions[tdimn] otvar = onc.variables[tvarn] timeinf = cls_time_information(onc, tvarn) dimt = len(otdim) print ' ' + fname + ': deaccumulating _______' for vn in varns: print " '" + vn + "'" ovar = onc.variables[vn] varinf = variable_inf(ovar) if not searchInlist(varinf.dimns, tdimn): print warnmsg print ' ' + fname + ": variable '" + vn + "' has not time dimension '"+\ tdimn + "' !!" print ' skipping it' else: slicefwd, dfwd = slice_variable(ovar, tdimn + ':1@'+str(dimt)) slicebck, dbck = slice_variable(ovar, tdimn + ':0@'+str(dimt-1)) deaccum = np.zeros((varinf.dims), dtype=varinf.dtype) ddeaccum = [] for idv in range(varinf.Ndims): if varinf.dimns[idv] == tdimn: ddeaccum.append(slice(1,varinf.dims[idv])) else: ddeaccum.append(slice(0,varinf.dims[idv])) deaccum[tuple(ddeaccum)] = slicefwd - slicebck if not searchInlist(onc.variables, vn + 'de'): if varinf.FillValue is not None: newvar = onc.createVariable(vn + 'de', varinf.dtype, \ tuple(varinf.dimns), fill_value = varinf.FillValue) else: newvar = onc.createVariable(vn + 'de', varinf.dtype, \ tuple(varinf.dimns)) else: print warnmsg print ' ' + fname + ": variable '" + vn + "de' already exist " + \ 'replacing values !!' newvar = onc.variables[vn + 'de'] if timeinf.dt is not None: # Not sure of this... # newvar[:] = deaccum / timeinf.dt newvar[:] = deaccum else: newvar[:] = deaccum newattr = set_attribute(newvar, 'deaccumulated', 'True') if timeinf.dt is not None: newattr = set_attributek(newvar, 'time_step_diff', timeinf.dt, \ 'npfloat') newattr = set_attribute(newvar, 'values', 'divided by time_step_diff') for attrn in varinf.attributes: attrv = ovar.getncattr(attrn) newattr = set_attribute(newvar, attrn, attrv) onc.sync() onc.close() return #ncstepdiff('Time,Times', '/home/lluis/PY/wrfout_d01_2001-11-11_00:00:00.tests', 'Q2,T2') def file_creation(values, ncfile, varn): """ Operation to create a file with one variable with a given set of dimensions values= [dimensions]|[varattributes]|[kind] [dimensions]: [dimn1]:[dsize1],...,[dimnN]:[dsizeN], ',' pairs of variable name [dimn] and [size] if [dsize] = 'None' (for UNLIMITED), give a third value with the real size [attributes]: [std_name]@[long_name]@[units], standard name, long name and units of the variable [kind]: type of variable (standard netCDF4/C-like values, 'c', 'i', 'f', 'f8',...) ncfile= name of the file varn= name of the variables """ fname = 'file_creation' if values == 'h': print fname + '_____________________________________________________________' print file_creation.__doc__ quit() expectargs = '[dimensions]|[varattributes]|[kind]' check_arguments(fname,values,expectargs,'|') dimensions = values.split('|')[0].split(',') attributes = values.split('|')[1] kind = values.split('|')[2] onc = NetCDFFile(ncfile, 'w') # Dimensions dnames = [] dsize = [] for dim in dimensions: # print " Adding: '" + dim + "' ..." dimn = dim.split(':')[0] dimv = dim.split(':')[1] if dimv == 'None': if len(dim.split(':')) != 3: print errormsg print ' ' + fname + ": dimension '" + dimn + "' is None but the " + \ 'size is requried!!' quit(-1) else: dv = None dsize.append(int(dim.split(':')[2])) else: dv = int(dimv) dsize.append(dv) dnames.append(dimn) newdim = onc.createDimension(dimn, dv) onc.sync() # Variable if kind == 'c': newvar = onc.createVariable(varn, 'c', tuple(dnames)) # newvar[:] = np.zeros(tuple(dsize), dtype=np.float) if kind == 'f' or kind == 'f4': newvar = onc.createVariable(varn, 'f4', tuple(dnames), fill_value=fillValue) newvar[:] = np.zeros(tuple(dsize), dtype=np.float) elif kind == 'f8': newvar = onc.createVariable(varn, 'f8', tuple(dnames), \ fill_value= np.float64(fillValue)) newvar[:] = np.zeros(tuple(dsize), dtype=np.float64) elif kind == 'i': newvar = onc.createVariable(varn, 'i', tuple(dnames), fill_value=-999999) newvar[:] = np.zeros(tuple(dsize), dtype=int) else: print errormsg print ' ' + fname + ": variable kind '" + kind + "' not ready!!" quit(-1) sname = attributes.split('@')[0] lname = attributes.split('@')[1] u = attributes.split('@')[2] newattr = basicvardef(newvar, sname, lname, u) onc.sync() # Global attributes newattr = set_attribute(onc, 'description', "file creation using " + fname ) onc.sync() onc.close() #file_creation('time_counter:12,sizes:24|time@time@seconds since 1949-12-01 00:00:00|f8', 'test.nc', 't_instant') def var_creation(values, ncfile, varn): """ Operation to create a new variable in a file with a given set of dimensions values= [dimensions]|[varattributes]|[kind] [dimensions]: [dimn1]:[dsize1],...,[dimnN]:[dsizeN], ',' pairs of variable name [dimn] and [size] if [dsize] = 'None', give a third value with the real size [attributes]: [std_name]@[long_name]@[units], standard name, long name and units of the variable ('!' for spaces) [kind]: type of variable (standard netCDF4/C-like values, 'c', 'i', 'f', 'f8',...) ncfile= name of the file varn= name of the variables """ fname = 'var_creation' if values == 'h': print fname + '_____________________________________________________________' print var_creation.__doc__ quit() expectargs = '[dimensions]|[varattributes]|[kind]' gen.check_arguments(fname,values,expectargs,'|') dimensions = values.split('|')[0].split(',') attributes = values.split('|')[1].replace('!',' ') kind = values.split('|')[2] onc = NetCDFFile(ncfile, 'a') # Dimensions dnames = [] dsize = [] for dim in dimensions: # print " Adding: '" + dim + "' ..." dimn = dim.split(':')[0] dimv = dim.split(':')[1] if dimv == 'None': if len(dim.split(':')) != 3: print errormsg print ' ' + fname + ": dimension '" + dimn + "' is None but the " + \ 'size is requried!!' quit(-1) else: dv = None dsize.append(int(dim.split(':')[2])) else: dv = int(dimv) dsize.append(dv) dnames.append(dimn) if not gen.searchInlist(onc.dimensions, dimn): newdim = onc.createDimension(dimn, dv) onc.sync() # Variable if kind == 'c': newvar = onc.createVariable(varn, 'c', tuple(dnames)) # newvar[:] = np.zeros(tuple(dsize), dtype=np.float) elif kind == 'f' or kind == 'f4': newvar = onc.createVariable(varn, 'f4', tuple(dnames), fill_value=fillValue) newvar[:] = np.zeros(tuple(dsize), dtype=np.float) elif kind == 'f8': newvar = onc.createVariable(varn, 'f8', tuple(dnames), \ fill_value=np.float64(fillValue)) newvar[:] = np.zeros(tuple(dsize), dtype=np.float64) elif kind == 'i': newvar = onc.createVariable(varn, 'i', tuple(dnames), fill_value=-99999) newvar[:] = np.zeros(tuple(dsize), dtype=int) else: print errormsg print ' ' + fname + ": variable kind '" + kind + "' not ready!!" quit(-1) sname = attributes.split('@')[0] lname = attributes.split('@')[1] u = attributes.split('@')[2] newattr = basicvardef(newvar, sname, lname, u) onc.sync() onc.close() return #var_creation('dim1:12,sizes:24|time@time@seconds since 1949-12-01 00:00:00|f4', 'test.nc', 'var2') #quit() def dimToUnlimited(values, ncfile): """ Operation to create an unlimited dimension from an existing one values= Name of the dimension to transform ncfile= name of the file """ fname = 'dimToUnlimited' if values == 'h': print fname + '_____________________________________________________________' print dimToUnlimited.__doc__ quit() dimn = values ncobj = NetCDFFile(ncfile, 'r') dimnames = ncobj.dimensions.keys() variables = ncobj.variables.keys() if not searchInlist(dimnames,dimn): print errormsg print ' ' + fname + ": dimension '" + dimn + "' does not exist in file '" + \ ncfile + "' !!" print ' existing ones are:',ncobj.dimensions.keys() quit(-1) # Creation of a new file newnc = NetCDFFile('new_' + ncfile, 'w') # Creation of dimensions for dn in dimnames: print 'adding dim:',dn if ncobj.dimensions[dn].isunlimited() or dn == dimn: newdim = newnc.createDimension(dn, None) else: newdim = newnc.createDimension(dn, len(ncobj.dimensions[dn])) # ncobj.close() newnc.sync() # newnc.close() # Adding variables for vn in variables: print 'adding variable:',vn vno = ncobj.variables[vn] vdim = vno.dimensions vtype = vno.dtype varatts = vno.ncattrs() if searchInlist(varatts, '_FillValue'): fillv = vno.getncattr('_FillValue') newvar = newnc.createVariable(vn, vtype, vdim, fill_value=fillv) else: newvar = newnc.createVariable(vn, vtype, vdim) newvar[:] = vno[:] for nattr in vno.ncattrs(): if not nattr == '_FillValue': nattrv = vno.getncattr(nattr) newattr = newvar.setncattr(nattr, nattrv) # Adding attributes fgaddattr(ncfile, 'new_' + ncfile) sub.call(['mv', 'new_'+ncfile, ncfile]) return #dimToUnlimited('lon', 'test.nc') def increaseDimvar(values, ncfile, varn): """ Function to increase with 1 dimension an existing variable within a netcdf file. Values of the variable will be repeated along the new dimension values='[dimname]:[size]:[position]:[unlimited]' dimname: name of the new dimension size: size of the new dimension position: position for the dimensions (starting from 0, outter most [left]) unlimited: y/n value for an unlimited dimension ncfile= netCDF file to use varn= variable to change its dimensions """ import subprocess as sub fname = 'increaseDimvar' if values == 'h': print fname + '_____________________________________________________________' print increaseDimvar.__doc__ quit() expectargs = '[dimname]:[size]:[position]:[unlimited]' check_arguments(fname,values,expectargs,':') dimname = values.split(':')[0] size = int(values.split(':')[1]) position = int(values.split(':')[2]) unlimited = values.split(':')[3] ofile = ncfile + '_new.nc' onc = NetCDFFile(ncfile, 'r') dims = onc.dimensions if searchInlist(dims, dimname): print errormsg print ' ' + fname + ": file '" + ncfile + "' already has dimension '" + \ dimname + "' !!" quit(-1) varns = onc.variables if not searchInlist(varns, varn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not variable '" + varn + \ "' !!" quit(-1) varobj = onc.variables[varn] vardims = varobj.dimensions varkind = varobj.dtype Norigdims = len(vardims) if position > Norigdims - 1: print errormsg print ' ' + fname + ": wrong position ", position," for the new dimension ",\ "on variable '" + varn + "' has only", Norigdims," dimensions !!" quit(-1) newdimns = [] newdimvs = [] idorig = 0 for idn in range(Norigdims + 1): if idn == position: newdimns.append(dimname) newdimvs.append(size) else: newdimns.append(vardims[idorig]) newdimvs.append(varobj.shape[idorig]) idorig = idorig + 1 # New variable newvarv = np.zeros(tuple(newdimvs), dtype=varkind) for inewdim in range(size): slicev = [] idorig = 0 for idn in range(Norigdims + 1): if idn == position: slicev.append(inewdim) else: slicev.append(slice(0,varobj.shape[idorig])) idorig = idorig + 1 newvarv[tuple(slicev)] = varobj[:] # Creation of a new file with the new variable dimensions newnc = NetCDFFile(ofile, 'w') # Creation of dimensions for dn in dims: print 'adding dim:',dn,' ...' if onc.dimensions[dn].isunlimited(): newdim = newnc.createDimension(dn, None) else: newdim = newnc.createDimension(dn, len(onc.dimensions[dn])) if unlimited == 'y': newdim = newnc.createDimension(dimname, None) else: newdim = newnc.createDimension(dimname, size) newnc.sync() # Adding variables for vn in varns: print 'adding variable:',vn,'...' vno = onc.variables[vn] if vn == varn: vdim = tuple(newdimns) else: vdim = vno.dimensions vtype = vno.dtype varatts = vno.ncattrs() if searchInlist(varatts, '_FillValue'): fillv = vno.getncattr('_FillValue') newvar = newnc.createVariable(vn, vtype, vdim, fill_value=fillv) else: newvar = newnc.createVariable(vn, vtype, vdim) for nattr in vno.ncattrs(): if not nattr == '_FillValue': nattrv = vno.getncattr(nattr) newattr = newvar.setncattr(nattr, nattrv) if vn == varn: newvar[:] = newvarv[:] else: newvar[:] = vno[:] onc.close() newnc.sync() newnc.close() # Adding attributes fgaddattr(ncfile, ofile) sub.call(['mv',ofile,ncfile]) return #increaseDimvar('shan:29:1:y', 'test.nc', 'var') def changevartype(values, ncfile, varn): """ Function to change the type of a variable (when possible) values=[newtype] type to use ('c', character; 'i', integer, 'f', float, 'd', double) ncfile= netCDF file to use varn= variable to change its type """ import subprocess as sub fname = 'changevartype' if values == 'h': print fname + '_____________________________________________________________' print changevartype.__doc__ quit() newtype = values ofile = ncfile + '_new.nc' onc = NetCDFFile(ncfile, 'r') if not searchInlist(onc.variables, varn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not variable '" + varn + \ "' !!" quit(-1) dims = onc.dimensions varns = onc.variables # Creation of a new file with the new variable dimensions newnc = NetCDFFile(ofile, 'w') # Creation of dimensions for dn in dims: print 'adding dim:',dn,' ...' if onc.dimensions[dn].isunlimited(): newdim = newnc.createDimension(dn, None) else: newdim = newnc.createDimension(dn, len(onc.dimensions[dn])) newnc.sync() # Transforming variable varo = onc.variables[varn] vartype = varo.dtype vardims = varo.dimensions varatts = varo.ncattrs() varshape = varo.shape varv = varo[:] if newtype == 'c': print errormsg print ' ' + fname + ": new type '" + newtype + "' not ready!!'" elif newtype == 'i': newvarv = varv.astype(int) elif newtype == 'f': newvarv = varv.astype(np.float) elif newtype == 'd': newvarv = varv.astype(np.float64) else: print errormsg print ' ' + fname + ": type '" + newtpe + "' not ready !!" print ' availables: i, f, d' quit(-1) if searchInlist(varatts, '_FillValue'): fillv = varo.getncattr('_FillValue') newvar = newnc.createVariable(varn, newtype, vardims, fill_value=fillv) else: newvar = newnc.createVariable(varn, newtype, vardims) for nattr in varo.ncattrs(): if not nattr == '_FillValue': nattrv = varo.getncattr(nattr) newattr = newvar.setncattr(nattr, nattrv) newvar[:] = newvarv # Adding variables for vn in varns: print 'adding variable:',vn,'...' vno = onc.variables[vn] if vn != varn: vdim = vno.dimensions vtype = vno.dtype varatts = vno.ncattrs() if searchInlist(varatts, '_FillValue'): fillv = vno.getncattr('_FillValue') newvar = newnc.createVariable(vn, vtype, vdim, fill_value=fillv) else: newvar = newnc.createVariable(vn, vtype, vdim) for nattr in vno.ncattrs(): if not nattr == '_FillValue': nattrv = vno.getncattr(nattr) newattr = newvar.setncattr(nattr, nattrv) newvar[:] = vno[:] onc.close() newnc.sync() newnc.close() # Adding attributes fgaddattr(ncfile, ofile) sub.call(['mv',ofile,ncfile]) return #changevartype('i', 'test.nc', 'var') def selvar(values, ncfile, varn): """ Function to select a series of variables from a netcdf file. Variables with the values of the associated dimensions will be also retrieved values= ',' list of couples [dimname]@[vardimname] [dimname]: name of the dimension [vardimname]: name of the variable with values for the dimension ('int', for counter of integers with the dimension length) ncfile= netCDF file to use varnames= ',' list of variables to retrieve """ fname = 'selvar' if values == 'h': print fname + '_____________________________________________________________' print selvar.__doc__ quit() varnames = varn.split(',') dimvals = values.split(',') ofile = fname + '_new.nc' onc = NetCDFFile(ncfile, 'r') for varn in varnames: if not gen.searchInlist(onc.variables, varn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not variable '" + \ varn + "' !!" quit(-1) dimsv = {} vardims = [] for dimv in dimvals: dimn = dimv.split('@')[0] if not gen.searchInlist(onc.dimensions, dimn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not dimension '" + \ dimn + "' !!" quit(-1) varn = dimv.split('@')[1] if not varn == 'int' and not gen.searchInlist(onc.variables, varn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not variable '" + \ varn + "' !!" quit(-1) dimsv[dimn] = varn if varn != 'int': vardims.append(varn) newnc = NetCDFFile(ofile, 'w') # Creation of dimensions add_dims(onc,newnc,dimsv.keys()) # Getting variables from dimensions add_vars(onc,newnc,vardims) # Getting variables for varn in varnames: varobj = onc.variables[varn] vardims = varobj.dimensions for dimn in vardims: if not gen.searchInlist(vardims,dimn): print errormsg print ' ' + fname + ": no assigned variable to dimension '" + \ dimn + "' !!" quit(-1) add_vars(onc,newnc,varnames) # Getting global attributes add_globattrs(onc,newnc,'all') onc.close() newnc.close() print "Successfull written of file '" + ofile + "' !!" return #selvar('x@lon,y@lat,time@time,shan@int', '/home/lluis/PY/test.nc', 'var') def compute_tevolboxtraj_radialsec(values, ncfile, varn): """ Function to compute tevolboxtraj_radialsec: temporal evolution at a given point along a number of radii at a given frequency following a trajectory values= [trajfile]@[Tbeg],[lonname],[latname],[zname],[timename],[timekind],[Nangle],[radii] [trajfile]: ASCII file with the trajectory ('#' not readed) [time] [xpoint] [ypoint] [Tbeg]: equivalent first time-step of the traqjectory within the netCDF file [lonname],[latname],[zname],[timename]: longitude, latitude, z and time variables names [timekind]: kind of time 'cf': cf-compilant 'wrf': WRF kind [Nangle]: Number of angles [radii]: length in grid points of the radius ncfile= netCDF file to use varn= ',' list of variables' name ('all', for all variables) """ import numpy.ma as ma import subprocess as sub fname='compute_tevolboxtraj_radialsec' if values == 'h': print fname + '_____________________________________________________________' print compute_tevolboxtraj_radialsec.__doc__ quit() arguments = '[trajfile]@[Tbeg],[lonname],[latname],[zname],[timename],' + \ '[timekind],[Nangle],[radii]]' check_arguments(fname,values,arguments,',') trajfile = values.split(',')[0].split('@')[0] Tbeg = int(values.split(',')[0].split('@')[1]) lonn = values.split(',')[1] latn = values.split(',')[2] zn = values.split(',')[3] timn = values.split(',')[4] timekind = values.split(',')[5] Nangle = int(values.split(',')[6]) radii = int(values.split(',')[7]) if not os.path.isfile(ncfile): print errormsg print ' ' + fname + " netCDF file: '" + ncfile + "' does not exist !!" quit(-1) if not os.path.isfile(trajfile): print errormsg print ' ' + fname + " trajectory file: '" + trajfile + "' does not exist !!" quit(-1) objfile = NetCDFFile(ncfile, 'r') lonobj = objfile.variables[lonn] latobj = objfile.variables[latn] timobj = objfile.variables[timn] if varn.find(',') != -1: varns = varn.split(',') else: if varn == 'all': varns = objfile.variables else: varns = [varn] lonv, latv = lonlat2D(lonobj[:],latobj[:]) dimx = lonv.shape[1] dimy = lonv.shape[0] timv = timobj[:] # Selecting accordingly a trajectory ## Ttraj = file_nlines(trajfile,'#') if timekind == 'wrf': dimt = objfile.variables[timn].shape[0] else: dimt = objfile.variables[timn].shape # Removing this checking # if Tbeg + Ttraj > dimt: # print errormsg # print ' ' + fname + ': trajectory has ', Ttraj, ' time steps and starts ' + \ # ' at',Tbeg,' and data',varobj.shape[0], '" !!!!!' # quit(-1) print ' ' + fname + ': Number of time-steps in trajectory file: ',Ttraj trajobj = open(trajfile,'r') # Trajectory values/slices ## # This is how we're going to proceed: # Creation of 1 matrix and 1 dictionay with the grid points where values have to be taken # trjradii(dimt,Nangle,Nrad,2): x,y pairs at each radii, angle and time step. # Ditance interpolation might be required! # SQtrjradii(dimt,2): x,y pairs at each radii, number of pairs (Npair) and time # step. Flip/rotation of points might be required! trjradii = np.zeros((dimt, Nangle, radii, 2), dtype = int) trjSQradii = {} TOTradii = [] iline = 0 it = 0 trajpos = np.zeros((Ttraj,Nangle,radii), dtype=np.float) gtrajvals = np.zeros((Ttraj,3), dtype=int) trajvals = np.zeros((Ttraj,3), dtype=np.float) # Grid position at each time-step and angle # varvalst = np.ones((Ttraj, Nangle, radii, 2), dtype=np.float)*fillValue # varvalstSQ = np.zeros((Ttraj, radii, radii), dtype=bool) it = 0 iline = 0 for line in trajobj: ## Skipping first line ## if not iline == 0: ## it = iline - 1 ## it = iline # Slicing brings to reduce 1 time-step.... ??? if line[0:1] != '#': gtrajvals[it,0] = Tbeg + iline gtrajvals[it,1] = int(line.split(' ')[1]) gtrajvals[it,2] = int(line.split(' ')[2]) print it,'t:',gtrajvals[it,0],'x y:', gtrajvals[it,1], gtrajvals[it,2] if timekind == 'wrf': gdate = datetimeStr_conversion(timv[gtrajvals[it,0]],'WRFdatetime', \ 'matYmdHMS') trajvals[it,0] = realdatetime1_CFcompilant(gdate, '19491201000000', \ 'hours') tunits = 'hours since 1949-12-01 00:00:00' elif timekind == 'cf': trajvals[it,0] = timv[gtrajvals[it,0]] tunits = timobj.getncattr('units') else: print errormsg print ' ' + fname + ' time kind: "' + timekind + '" not ready !!' quit(-1) # print iline, it,'time:',trajvals[it,0],'lon lat:', trajvals[it,1], \ # trajvals[it,2] # Assuming time as the first dimension in a fortran like way (t=0 as 1) # trajcut[0] = tstept - 1 # Assuming time as the first dimension in a C/python like way (t=0) posangle = np.zeros((radii,2), dtype=np.float) trjradii[it,:,:,:], trjSQradii[it] = radii_points(gtrajvals[it,1], \ gtrajvals[it,2],radii,Nangle,dimx,dimy) if it == 0: TOTradii = list(trjSQradii[it].keys()) else: for ir in trjSQradii[it].keys(): if searchInlist(TOTradii,ir): TOTradii.append(ir) it = it + 1 iline = iline + 1 trajobj.close() # Creation of the netCDF file ## NtotSQradii = len(TOTradii) TOTSQradii = sorted(TOTradii) print ' ' + fname + ':',NtotSQradii,' squared radii:',TOTSQradii if varn.find(',') != -1: varnS = 'multi-var' else: varnS = varn.replace(',','-') ofile = 'tevolradiitraj_' + varnS + '.nc' objofile = NetCDFFile(ofile, 'w') boxs = radii*2+1 # Dimensions newdim = objofile.createDimension('x', boxs) newdim = objofile.createDimension('y', boxs) newdim = objofile.createDimension('time', None) newdim = objofile.createDimension('radii', radii) newdim = objofile.createDimension('SQradii', NtotSQradii) stsn = ['min', 'max', 'mean', 'mean2', 'stdev', 'ac'] vstlname = ['minimum value within', 'maximum value within', 'mean value within', \ 'squared mean value within', 'standard deviation value within', \ 'accumulated value within'] Nsts = len(stsn) statnames = [] cstatnames = [] for i in range(Nsts): statnames.append(stsn[i] + 'radii') cstatnames.append(stsn[i] + 'SQradii') # Getting values ## ivar = 0 box2=0 for vn in varns: vnst = variables_values(vn)[0] if not searchInlist(objfile.variables, vn): print errormsg print ' ' + fname + ": variable name '" + vn + "' is not in file " + \ ncfile + '" !!!!!' quit(-1) varobj = objfile.variables[vn] Nvardims = len(varobj.shape) print ' ' + fname + ": getting '" + vn + "' ... .. ." slicev = [] slice2D = [] slicevnoT = [] cslicev = [] cslice2D = [] cslicevnoT = [] if Nvardims == 4: # Too optimistic, but at this stage... if not objofile.dimensions.has_key('z'): varzobj = objfile.variables[zn] if len(varzobj.shape) == 1: dimz = varzobj.shape zvals = varzobj[:] elif len(varzobj.shape) == 2: dimz = varzobj.shape[1] zvals = varzobj[0,:] elif len(varzobj.shape) == 3: dimz = varzobj.shape[2] zvals = varzobj[0,0,:] objofile.createDimension('z', dimz) newvar = objofile.createVariable(zn, 'f4', ('z'),fill_value=fillValue) if searchInlist(varzobj.ncattrs(),'standard_name'): vsname = varzobj.getncattr('standard_name') else: vsname = variables_values(zn)[1] if searchInlist(varzobj.ncattrs(),'long_name'): vlname = varzobj.getncattr('long_name') else: vlname = variables_values(zn)[4].replace('|',' ') if searchInlist(varzobj.ncattrs(),'units'): vunits = varzobj.getncattr('units') else: vunits = variables_values(zn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newvar[:] = zvals varvals = np.ones(tuple([Ttraj,dimz,boxs,boxs]), dtype=np.float) lonvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) latvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) radiivarvals = np.ones(tuple([Ttraj,dimz,radii]), dtype=np.float)*fillValueF SQradiivarvals = np.ones(tuple([Ttraj,dimz,NtotSQradii]), dtype=np.float)*fillValueF # One dimension plus for the values at the center of the trajectory statvarvals = np.ones(tuple([Ttraj,dimz,Nsts+1]), dtype=np.float) rstatvarvals = np.ones(tuple([Ttraj,dimz,Nsts+1]), dtype=np.float) for it in range(Ttraj): it0 = Tbeg + it # Radii values # trjradii[it,:,:,:], trjSQradii[it] for ir in range(radii): for ia in range(Nangle): xp = trjradii[it,ia,ir,0] yp = trjradii[it,ia,ir,1] xpI = int(xp*1.) ypI = int(yp*1.) print 'xp yp:',xp,yp,'xpI ypI:',xpI,ypI quit() mean = 0. for iz in range(dimz): SWv = varobj[it0,iz,ypI,xpI] SEv = varobj[it0,iz,ypI,xpI+1] NWv = varobj[it0,iz,ypI+1,xpI] NEv = varobj[it0,iz,ypI+1,xpI+1] val = hor_weight_int(SEv, SWv, NEv, NWv, xp-xpI, yp-ypI) # print 'Lluis hor_weight_int(',SEv,',', SWv,',', NEv,',', NWv,',', xp-xpI,',', yp-ypI,')' # print val mean = mean + val print 'mean:',mean quit() slicev = [] slice2D = [] slicevnoT = [] cslicev = [] cslice2D = [] cslice2Dhor = [] cslicevnoT = [] cslicevnoThor = [] slicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-box2 < 0 or gtrajvals[it,2]+box2 + 1 > dimy + 1 \ or gtrajvals[it,1]-box2 < 0 or gtrajvals[it,1]+box2 + 1 > dimx +1: # box values slicev.append(slice(0,dimz)) slicev.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicev.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slicevnoT.append(slice(0,dimz)) slicevnoT.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicevnoT.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slice2D.append(slice(0,dimz)) slice2D.append(slice(0,yrangeslice[it][1]-yrangeslice[it][0])) slice2D.append(slice(0,xrangeslice[it][1]-xrangeslice[it][0])) rvarvalst = np.ones((dimz, Nrad*2+1, Nrad*2+1),dtype=np.float)* \ fillValue varvalst[tuple(slice2D)] = varobj[tuple(slicev)] varvals[it,:,:,:] = varvalst # box stats values maskedvals = ma.masked_values (varvalst, fillValue) maskedvals2 = maskedvals*maskedvals for iz in range(dimz): statvarvals[it,iz,0] = varvalst[iz,box2,box2] statvarvals[it,iz,1] = np.min(varvalst[iz,:,:]) statvarvals[it,iz,2] = np.max(varvalst[iz,:,:]) statvarvals[it,iz,3] = np.mean(varvalst[iz,:,:]) statvarvals[it,iz,4] = maskedvals2[iz,:,:].mean() statvarvals[it,iz,5] = np.sqrt(statvarvals[it,iz,4] - \ statvarvals[it,iz,3]*statvarvals[it,iz,3]) statvarvals[it,iz,6] = np.sum(varvalst[iz,:,:]) else: slicev.append(slice(0,dimz)) slicev.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2+1)) slicev.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2+1)) slicevnoT.append(slice(0,dimz)) slicevnoT.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+ \ box2+1)) slicevnoT.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+ \ box2+1)) slice2D.append(slice(0,dimz)) slice2D.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2] + \ box2 + 1)) slice2D.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1] + \ box2 + 1)) varvalst = varobj[tuple(slicev)] # box values varvals[it,:,:,:] = varvalst # print 'values at time t______' # print varvalst # box stats values for iz in range(dimz): statvarvals[it,:,0] = varvalst[:,box2,box2] statvarvals[it,iz,1] = np.min(varvalst[iz,:,:]) statvarvals[it,iz,2] = np.max(varvalst[iz,:,:]) statvarvals[it,iz,3] = np.mean(varvalst[iz,:,:]) statvarvals[it,iz,4] = np.mean(varvalst[iz,:,:]* \ varvalst[iz,:,:]) statvarvals[it,iz,5] = np.sqrt(statvarvals[it,iz,4] - \ statvarvals[it,iz,3]*statvarvals[it,iz,3]) statvarvals[it,iz,6] = np.sum(varvalst[iz,:,:]) # Circle values cslicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-Nrad < 0 or gtrajvals[it,2]+Nrad + 1 >= dimy \ or gtrajvals[it,1]-Nrad < 0 or gtrajvals[it,1]+Nrad + 1 >= dimx: maxx = np.min([cxrangeslice[it][1], dimx]) maxy = np.min([cyrangeslice[it][1], dimy]) cslicev.append(slice(0,dimz)) cslicev.append(slice(cyrangeslice[it][0], maxy)) cslicev.append(slice(cxrangeslice[it][0], maxx)) cslicevnoT.append(slice(0,dimz)) cslicevnoT.append(slice(cyrangeslice[it][0], cyrangeslice[it][1])) cslicevnoT.append(slice(cxrangeslice[it][0], cxrangeslice[it][1])) cslice2D.append(slice(0,dimz)) cslice2D.append(slice(0,maxy-cyrangeslice[it][0])) cslice2D.append(slice(0,maxx-cxrangeslice[it][0])) cslice2Dhor.append(slice(0, maxy - cyrangeslice[it][0])) cslice2Dhor.append(slice(0, maxx -cxrangeslice[it][0])) rvarvalst = np.ones((dimz,Nrad*2+1,Nrad*2+1),dtype=np.float)* \ fillValue rvarvalst[tuple(cslice2D)] = varobj[tuple(cslicev)] for iz in range(dimz): tslice = [slice(it,it)]+cslice2Dhor zslice = [slice(iz,iz)]+cslice2Dhor rvarvalst[tuple(zslice)] = np.where(circdist[tuple(tslice)] > \ np.float(Nrad), fillValue, rvarvalst[tuple(zslice)]) rvarvals[it,:,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) maskedvals2 = maskedvals*maskedvals for iz in range(dimz): rstatvarvals[it,iz,0] = varvalst[iz,box2,box2] rstatvarvals[it,iz,1] = np.min(varvalst[iz,:,:]) rstatvarvals[it,iz,2] = np.max(varvalst[iz,:,:]) rstatvarvals[it,iz,3] = np.mean(varvalst[iz,:,:]) rstatvarvals[it,iz,4] = maskedvals2[iz,:,:].mean() rstatvarvals[it,iz,5] = np.sqrt(rstatvarvals[it,iz,4] - \ rstatvarvals[it,iz,3]*rstatvarvals[it,iz,3]) rstatvarvals[it,iz,6] = np.sum(varvalst[iz,:,:]) else: cslicev.append(slice(0,dimz)) cslicev.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslicev.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) cslicevnoT.append(slice(0,dimz)) cslicevnoT.append(slice(gtrajvals[it,2]-Nrad, gtrajvals[it,2]+ \ Nrad+1)) cslicevnoT.append(slice(gtrajvals[it,1]-Nrad, gtrajvals[it,1]+ \ Nrad+1)) cslicevnoThor.append(slice(gtrajvals[it,2]-Nrad, \ gtrajvals[it,2] + Nrad+1)) cslicevnoThor.append(slice(gtrajvals[it,1]-Nrad, \ gtrajvals[it,1] + Nrad+1)) cslice2D.append(slice(0,dimz)) cslice2D.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2] + \ Nrad+1)) cslice2D.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1] + \ Nrad+1)) rvarvalst = varobj[tuple(cslicev)] # circle values for iz in range(dimz): tslice = [it]+cslicevnoThor rvarvalst[iz,:,:] = np.where(circdist[tuple(tslice)] > \ np.float(Nrad), fillValue, rvarvalst[iz,:,:]) rvarvals[it,:,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) maskedvals2 = maskedvals*maskedvals for iz in range(dimz): rstatvarvals[it,iz,0] = rvarvalst[iz,Nrad,Nrad] rstatvarvals[it,iz,1] = maskedvals[iz,:,:].min() rstatvarvals[it,iz,2] = maskedvals[iz,:,:].max() rstatvarvals[it,iz,3] = maskedvals[iz,:,:].mean() rstatvarvals[it,iz,4] = maskedvals2[iz,:,:].mean() rstatvarvals[it,iz,5] = np.sqrt(rstatvarvals[it,iz,4] - \ rstatvarvals[it,iz,3]*rstatvarvals[it,iz,3]) rstatvarvals[it,iz,6] = maskedvals[iz,:,:].sum() # print 'statistics:',rstatvarvals[it,:] # variable box values newvar = objofile.createVariable(vnst + 'box', 'f4', ('time','z','y','x'),\ fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = varvals # center of the trajectory newvar = objofile.createVariable('trj_' + vnst, 'f', ('time','z'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'trj_' + vsname, 'value along the ' + \ 'trajectory of '+ vn, vunits) newvar[:] = statvarvals[:,:,0] # variable box statistics ist = 0 for statn in statnames: newvar = objofile.createVariable(statn + '_' + vnst,'f',('time','z'),\ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the box ('+str(boxs)+'x'+str(boxs)+') of ' + vnst, vunits) newvar[:] = statvarvals[:,:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 # variable circle values newvar = objofile.createVariable(vnst + 'circle','f4',('time','z','yr', \ 'xr'), fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = rvarvals # variable circle statistics ist = 0 for statn in cstatnames: newvar = objofile.createVariable(statn + '_' + vnst,'f',('time','z'),\ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the circle of radius ('+ str(circler)+') of ' + vnst, vunits) newvar[:] = rstatvarvals[:,:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 elif Nvardims == 3: # Too optimistic, but at this stage... dimt = varobj.shape[0] Nrad=1 varvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) lonvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) latvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) rvarvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) rlonvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) rlatvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) # One dimension plus for the values at the center of the trajectory statvarvals = np.ones(tuple([Ttraj,Nsts+1]), dtype=np.float) rstatvarvals = np.ones(tuple([Ttraj,Nsts+1]), dtype=np.float) for it in range(Ttraj): it0 = Tbeg + it slicev = [] slice2D = [] slicevnoT = [] cslicev = [] cslice2D = [] cslicevnoT = [] cslicevC = [] slicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-box2 < 0 or gtrajvals[it,2]+box2 + 1 > dimy + 1 \ or gtrajvals[it,1]-box2 < 0 or gtrajvals[it,1]+box2 + 1 > dimx + 1: # box values slicev.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicev.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slicevnoT.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicevnoT.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slice2D.append(slice(0,yrangeslice[it][1]-yrangeslice[it][0])) slice2D.append(slice(0,xrangeslice[it][1]-xrangeslice[it][0])) rvarvalst = np.ones((Nrad*2+1, Nrad*2+1),dtype=np.float)*fillValue varvalst[tuple(slice2D)] = varobj[tuple(slicev)] varvals[it,:,:] = varvalst # box stats values maskedvals = ma.masked_values (varvalst, fillValue) statvarvals[it,0] = varvalst[box2,box2] statvarvals[it,1] = maskedvals.min() statvarvals[it,2] = maskedvals.max() statvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals statvarvals[it,4] = maskedvals2.mean() statvarvals[it,5] = np.sqrt(statvarvals[it,4] - \ statvarvals[it,3]*statvarvals[it,3]) statvarvals[it,6] = maskedvals.sum() varvalst[tuple(slice2D)] = lonv[tuple(slicevnoT)] lonvals[it,:,:] = varvalst varvalst[tuple(slice2D)] = latv[tuple(slicevnoT)] latvals[it,:,:] = varvalst else: slicev.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2+1)) slicev.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2+1)) slicevnoT.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+ \ box2+1)) slicevnoT.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+ \ box2+1)) slice2D.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2 +\ 1)) slice2D.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2 +\ 1)) varvalst = varobj[tuple(slicev)] # box values varvals[it,:,:] = varvalst # print 'values at time t______' # print varvalst # box stats values statvarvals[it,0] = varvalst[box2,box2] statvarvals[it,1] = np.min(varvalst) statvarvals[it,2] = np.max(varvalst) statvarvals[it,3] = np.mean(varvalst) statvarvals[it,4] = np.mean(varvalst*varvalst) statvarvals[it,5] = np.sqrt(statvarvals[it,4] - \ statvarvals[it,3]*statvarvals[it,3]) statvarvals[it,6] = np.sum(varvalst) varvalst = lonv[tuple(slicevnoT)] lonvals[it,:,:] = varvalst varvalst = latv[tuple(slicevnoT)] latvals[it,:,:] = varvalst # Circle values cslicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-Nrad < 0 or gtrajvals[it,2]+Nrad + 1 >= dimy \ or gtrajvals[it,1]-Nrad < 0 or gtrajvals[it,1]+Nrad + 1 >= dimx: maxx = np.min([cxrangeslice[it][1], dimx]) maxy = np.min([cyrangeslice[it][1], dimy]) cslicev.append(slice(cyrangeslice[it][0],maxy)) cslicev.append(slice(cxrangeslice[it][0],maxx)) cslicevnoT.append(slice(cyrangeslice[it][0],cyrangeslice[it][1])) cslicevnoT.append(slice(cxrangeslice[it][0],cxrangeslice[it][1])) cslice2D.append(slice(0,maxy - cyrangeslice[it][0])) cslice2D.append(slice(0,maxx - cxrangeslice[it][0])) rvarvalst = np.ones((Nrad*2+1,Nrad*2+1),dtype=np.float)*fillValue rvarvalst[tuple(cslice2D)] = varobj[tuple(cslicev)] rvarvalst[tuple(cslice2D)] = np.where(circdist[tuple(cslicev)] >\ np.float(Nrad), fillValue, rvarvalst[tuple(cslice2D)]) rvarvals[it,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) rstatvarvals[it,0] = rvarvalst[Nrad,Nrad] rstatvarvals[it,1] = maskedvals.min() rstatvarvals[it,2] = maskedvals.max() rstatvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals rstatvarvals[it,4] = maskedvals2.mean() rstatvarvals[it,5] = np.sqrt(rstatvarvals[it,4] - \ rstatvarvals[it,3]*rstatvarvals[it,3]) rstatvarvals[it,6] = maskedvals2.sum() else: cslicev.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslicev.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) cslicevnoT.append(slice(gtrajvals[it,2]-Nrad, gtrajvals[it,2]+ \ Nrad+1)) cslicevnoT.append(slice(gtrajvals[it,1]-Nrad, gtrajvals[it,1]+ \ Nrad+1)) cslice2D.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslice2D.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) cslicevC.append(it) cslicevC.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslicevC.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) rvarvalst = varobj[tuple(cslicev)] cdist = circdist[tuple(cslicevC)] # circle values rvarvalst = np.where(cdist > np.float(Nrad),fillValue,rvarvalst) rvarvals[it,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) rstatvarvals[it,0] = rvarvalst[Nrad,Nrad] rstatvarvals[it,1] = maskedvals.min() rstatvarvals[it,2] = maskedvals.max() rstatvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals rstatvarvals[it,4] = maskedvals2.mean() rstatvarvals[it,5] = np.sqrt(rstatvarvals[it,4] - \ rstatvarvals[it,3]*rstatvarvals[it,3]) rstatvarvals[it,6] = maskedvals.sum() # print 'statistics:',rstatvarvals[it,:] # variable box values newvar = objofile.createVariable(vnst + 'box', 'f4', ('time', 'y', 'x'), \ fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = varvals # center of the trajectory newvar = objofile.createVariable('trj_' + vnst, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'trj_' + vsname, 'value along the ' + \ 'trajectory of '+ vnst, vunits) newvar[:] = statvarvals[:,0] # variable box statistics ist = 0 for statn in statnames: newvar = objofile.createVariable(statn + '_' + vnst, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the box ('+str(boxs)+'x'+str(boxs)+') of ' + vnst, vunits) newvar[:] = statvarvals[:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 # variable circle values newvar = objofile.createVariable(vnst + 'circle','f4',('time','yr','xr'),\ fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = rvarvals # variable circle statistics ist = 0 for statn in cstatnames: newvar = objofile.createVariable(statn + '_' + vnst, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the circle of radius ('+ str(circler)+') of ' + vnst, vunits) newvar[:] = rstatvarvals[:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 else: # Other variables print warnmsg print ' ' + fname + ": variable '" + vn + "' shape:",varobj.shape,' not'\ + ' ready!!' print ' skipping variable' if len(varns) == 1: objofile.close() sub.call(['rm', ofile]) print ' uniq variable! removing file and finishing program' quit() if not objofile.variables.has_key('trlon') and Nvardims == 3: # var dimensions newvar = objofile.createVariable('trlon', 'f8', ('time')) newattr = basicvardef(newvar,'trlon','trajectory longitude', \ 'degrees west_east') newvar[:] = trajvals[:,1] newvar = objofile.createVariable('trlat', 'f8', ('time')) newattr = basicvardef(newvar,'trlat','trajectory latitude', \ 'degrees north_south') newvar[:] = trajvals[:,2] newvar = objofile.createVariable('time', 'f8', ('time')) newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = trajvals[:,0] newvar = objofile.createVariable('lon', 'f8', ('time', 'y', 'x'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'longitude', 'longitude', \ 'degrees west_east') newvar[:] = lonvals newvar = objofile.createVariable('lat', 'f8', ('time', 'y', 'x'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'latitude', 'latitude', \ 'degrees north_south') newvar[:] = latvals ivar = ivar + 1 # global attributes objofile.setncattr('author', 'L. Fita') objofile.setncattr('institution', 'Laboratire de Meteorologie Dynamique') objofile.setncattr('university', 'Pierre Marie Curie - Jussieu') objofile.setncattr('center', 'Centre National de Recherches Scientifiques') objofile.setncattr('city', 'Paris') objofile.setncattr('country', 'France') objofile.setncattr('script', 'nc_var_tools.py') objofile.setncattr('function', 'compute_tevolboxtraj') objofile.setncattr('version', '1.0') objofile.setncattr('data_file',ncfile) objfile.close() objofile.sync() objofile.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return # [trajfile]: ASCII file with the trajectory ('#' not readed) # [time] [xpoint] [ypoint] # [Tbeg]: equivalent first time-step of the trajectory within the netCDF file # [lonname],[latname],[zname],[timename]: longitude, latitude, z and time variables names # [timekind]: kind of time # 'cf': cf-compilant # 'wrf': WRF kind # [Nangle]: Number of angles # [radii]: length in grid points of the radius #compute_tevolboxtraj_radialsec('/bdd/PCER/workspace/lfita/etudes/FF/medic051215_2km/run/trajectory_shorten.dat@0,XLONG,XLAT,ZNU,Times,wrf,4,2', \ # '/bdd/PCER/workspace/lfita/etudes/FF/medic051215_2km/run/wrfout/wrfout_d02_2005-12-13_00:00:00', 'QVAPOR') #quit() def DatesFiles(values, ncfile, varn): """ Function to find different time values on a series of WRF files in a folder [values]= [dates],[folder] [dates]: ':' list of [YYYY][MM][DD][HH][MI][SS] dates [fold]: folder with the location of the netCDF files [ncfile]= header of the name of the files to concatenate [ncfile]* [varn]= name of the variable Time ('WRFtime', for WRF time) """ import subprocess as sub import datetime as dt fname='DatesFiles' if values == 'h': print fname + '_____________________________________________________________' print DatesFiles.__doc__ quit() arguments = '[dates],[fold]' check_arguments(fname, values, arguments, ',') dates = values.split(',')[0].split(':') fold = values.split(',')[1] confiles = files_folder(fold,ncfile) Nfiles = len(confiles) print ' ' + fname +': looking on:',Nfiles,'files' # Opening all files and getting times ncobjs = {} for filen in confiles: filename = fold + '/' + filen ncf = NetCDFFile(filename, 'r') datesv = [] if varn == 'WRFtime': fdates = ncf.variables['Times'][:] for it in range(fdates.shape[0]): datesv.append(datetimeStr_conversion(fdates[it,:], 'WRFdatetime', \ 'YmdHMS')) else: tunits = ncf.variables[varn].getncattr('units') fdates = ncf.variables[varn][:] for it in range(fdates.shape[0]): datesv.append(datetimeStr_conversion(fdates[it], 'cfTime,' + tunits ,\ 'YmdHMS')) ncobjs[filename] = datesv ncf.close() # Computing time-step inside files (assuming regular) if filen == confiles[0]: ini=datesv[0] end=datesv[1] iniT = dt.datetime.strptime(ini,'%Y%m%d%H%M%S') endT = dt.datetime.strptime(end,'%Y%m%d%H%M%S') DT = endT - iniT # Python version depending if searchInlist(dir(DT), 'total_seconds'): diffT = DT.total_seconds() else: diffT = DT.days*24*3600. + DT.seconds print ' ' + fname + ': seconds between time-steps in files:', diffT Efounddates = {} Cfounddates = {} for dtv in dates: foundclosest = False for fn in ncobjs.keys(): if searchInlist(ncobjs[fn], dtv): istep = ncobjs[fn].index(dtv) print fname + 'Exact:',dtv,fn,istep,'(date, filname, time-step)' Efounddates[dtv] = [dtv,fn,istep] for it in ncobjs[fn]: iniT = dt.datetime.strptime(it,'%Y%m%d%H%M%S') endT = dt.datetime.strptime(dtv,'%Y%m%d%H%M%S') DT = endT - iniT # Python version depending if searchInlist(dir(DT), 'total_seconds'): diffitv = np.abs(DT.total_seconds()) else: diffitv = np.abs(DT.days*24*3600. + DT.seconds) if diffT/2. >= diffitv: istep = ncobjs[fn].index(it) print fname + 'Closest:',dtv,fn,istep,diffitv,it,'(date, ' + \ 'filname, time-step, distance in seconds, closest date)' Cfounddates[dtv] = [dtv,fn,istep,diffitv] foundclosest = True break if foundclosest: break return def TimeSplitmat(values, ncfile, vn): """ Function to transform a file with CFtimes to a matrix [Nyear,Nmonth,Nday,Nhour,Nminute,Nsecond] values= [tdim]:[tvdim]:[seltimes] tdim= name of the dimension time tvdim= name of the variable with the time values seltimes= ',' list of which times ('y', 'm', 'd', 'H','M', 'S') one wants to split the values desired final order ncfile= netcdf file to use vn= name of the variable, 'all' for all variables """ import datetime as dt fname = 'TimeSplitmat' ofile = 'TimeSplitmat.nc' if values == 'h': print fname + '_____________________________________________________________' print TimeSplitmat.__doc__ quit() arguments = '[tdim]:[tvdim]:[seltimes]' check_arguments(fname, values, arguments, ':') tdim = values.split(':')[0] tvdim = values.split(':')[1] seltimes = values.split(':')[2] of = NetCDFFile(ncfile, 'r') stimes = seltimes.split(',') if len(stimes) != 2: print errormsg print fname + ': number of splitting times',len(stimes),'not ready!!' quit(-1) if vn == 'all': vns = of.variables.keys() else: vns = [vn] otime = of.variables[tvdim] dt = otime.shape[0] if not searchInlist(otime.ncattrs(), 'units'): print errormsg print ' ' + fname + ": varible time '" + dimtv + "' has not units!!" quit(-1) tunits = otime.getncattr('units') mattimes = CFtimes_datetime(of,tvdim) Nyears = len(np.unique(mattimes[:,0])) Nmonths = len(np.unique(mattimes[:,1])) Ndays = len(np.unique(mattimes[:,2])) Nhours = len(np.unique(mattimes[:,3])) Nminutes = len(np.unique(mattimes[:,4])) Nseconds = len(np.unique(mattimes[:,5])) newdimns = [] newdimvs = [] firstmat = -1 secondmat = -1 for st in stimes: if st == 'y': newdimns.append('years') newdimvs.append(Nyears) if firstmat == -1: firstmat = 0 else: secondmat = 0 elif st == 'm': newdimns.append('months') newdimvs.append(Nmonths) if firstmat == -1: firstmat = 1 else: secondmat = 1 elif st == 'd': newdimns.append('days') newdimvs.append(Ndays) if firstmat == -1: firstmat = 2 else: secondmat = 2 elif st == 'H': newdimns.append('hours') newdimvs.append(Nhours) if firstmat == -1: firstmat = 3 else: secondmat = 3 elif st == 'M': newdimns.append('minutes') newdimvs.append(Nminutes) if firstmat == -1: firstmat = 4 else: secondmat = 4 elif st == 'S': newdimns.append('seconds') newdimvs.append(Nseconds) if firstmat == -1: firstmat = 5 else: secondmat = 5 else: print errormsg print ' ' + fname + ": selected time '" + st + "' not ready!!" quit(-1) print ' ' + fname + ':initial selection of:',newdimvs print ' ' + newdimns[0],':',np.unique(mattimes[:,firstmat]) print ' ' + newdimns[1],':',np.unique(mattimes[:,secondmat]) # We can have still multiple values (for given days, multiple months...). Assuming # 30days months newdimvs[1] = dt/newdimvs[0] print ' re-arranged to:',newdimvs newtime = np.zeros(tuple(newdimvs), dtype=np.float) # Openning new file onewfile = NetCDFFile(ofile, 'w') # Dimensions newdim = onewfile.createDimension(newdimns[0], newdimvs[0]) newdim = onewfile.createDimension(newdimns[1], newdimvs[1]) newdim = onewfile.createDimension('time', None) # Variable time newvar = onewfile.createVariable('time', 'f8', tuple(newdimns)) newvar[:] = newtime basicvardef(newvar, 'time', 'time', tunits) splitS = seltimes.replace(',','') for Vn in vns: print Vn + '...' oVn = of.variables[Vn] # Removing time dimension from the variable dimensions Vndims = oVn.dimensions if searchInlist(Vndims, tdim) and Vn != tvdim: varNOtdimns = [] varNOtdimvs = [] for idn in Vndims: if idn != tdim: varNOtdimns.append(idn) varNOtdimvs.append(len(of.dimensions[idn])) if not searchInlist(onewfile.dimensions.keys(), idn): newdim = onewfile.createDimension(idn, \ len(of.dimensions[idn])) newvarv = np.ones(tuple(newdimvs+varNOtdimvs), dtype=np.float)*fillValueF d1 = 0 d2 = -1 d1val = mattimes[0,firstmat] d2val = mattimes[0,secondmat] secondvals = [] for ddt in range(dt): if mattimes[ddt,firstmat] > d1val: d1 = d1 + 1 d1val = mattimes[ddt,firstmat] d2 = 0 else: d2 = d2 + 1 if d1 == 0: secondvals.append(mattimes[ddt,secondmat]) slicevar = [] slicenewvar = [] slicenewvar.append(d1) slicenewvar.append(d2) for idn in Vndims: if idn != tdim: slicevar.append(slice(0,len(of.dimensions[idn]))) slicenewvar.append(slice(0,len(of.dimensions[idn]))) else: slicevar.append(ddt) # print ddt,d1,d2,mattimes[ddt,:],'var:',slicevar,'new:',slicenewvar newvarv[tuple(slicenewvar)] = oVn[tuple(slicevar)] newtime[d1,d2] = otime[ddt] newvar = onewfile.createVariable(Vn + '_' + splitS, 'f', \ tuple(newdimns + varNOtdimns), fill_value = fillValueF) newattr = newvar.setncattr('desc', Vn + 'splitted by:' + newdimns[0] + \ ' & ' + newdimns[1]) newvar[:] = newvarv onewfile.sync() elif Vn != tvdim: for dvn in Vndims: if not searchInlist(onewfile.dimensions,dvn): dsize = len(of.dimensions[dvn]) newdim = onewfile.createDimension(dvn, dsize) # From http://guziy.blogspot.fr/2014/01/netcdf4-python-copying-variables-from.html vartype = oVn.dtype newvar = onewfile.createVariable(Vn, vartype, Vndims) for attrn in oVn.ncattrs(): attrv = oVn.getncattr(attrn) newattr = set_attribute(newvar, attrn, attrv) newvar[:] = oVn[:] # Variable time newvar = onewfile.variables['time'] newvar[:] = newtime # Variables selection idn=0 for newd in newdimns: print newd, onewfile.dimensions.keys() newvar = onewfile.createVariable(newd, 'f', (newd)) if idn == 0: newvar[:] = np.unique(mattimes[:,firstmat]) else: newvar[:] = secondvals set_attribute(newvar, 'desc', 'Selected ' + newdimns[idn]) idn = idn + 1 # Global attributes onewfile.setncattr('script', fname) onewfile.setncattr('version', '1.0') onewfile.setncattr('author', 'L. Fita') newattr = set_attributek(onewnfile, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') onewfile.setncattr('university', 'Pierre et Marie Curie') onewfile.setncattr('country', 'France') onewfile.setncattr('description', 'transform a file with CFtimes to a matrix ' +\ '[Nyear,Nmonth,Nday,Nhour,Nminute,Nsecond]') onewfile.sync() onewfile.close() print fname + ": Successful writting of file '" + ofile + "' !!" return #fileobj = NetCDFFile('/home/lluis/PY/ERAI_pl199501_130-133.nc', 'r') #TimeSplitmat(fileobj, 'time', 'time', 'all', 'd,H') def getvalues_lonlat(values, ncfile): """ Function to retrieve the values from the closest grid point to a set of longitude, latitude values values=[longitude]:[latitude]:[dnames]:[vdnames] [longitude]: value for the longitude [latitude]: value for the latitude [dnames]: [xdimname],[ydimname] names of the dimensions on the 'x' and 'y' axis [dvnames]: [xvdimname],[yvdimname] names of the variables with the values for the dimensions on the 'x' and 'y' axis ncfile= netcdf file to use """ import subprocess as sub fname = 'getvalues_lonlat' ofile = fname + '.nc' if values == 'h': print fname + '_____________________________________________________________' print getvalues_lonlat.__doc__ quit() arguments = '[longitude]:[latitude]:[dnames]:[vdnames]' check_arguments(fname, values, arguments, ':') longitude = np.float(values.split(':')[0]) latitude = np.float(values.split(':')[1]) dnames = values.split(':')[2].split(',') dvnames = values.split(':')[3].split(',') onc = NetCDFFile(ncfile, 'r') for dn in dnames: if not searchInlist(onc.dimensions, dn): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have dimension '"+\ dn + "' !!" quit(-1) for vdn in dvnames: if not searchInlist(onc.variables, vdn): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have variable '"+ \ vdn + "' !!" quit(-1) # Getting grid point localization ## lon2d, lat2d = lonlat2D(onc.variables[dvnames[0]], onc.variables[dvnames[1]]) dimx = lon2d.shape[1] dimy = lon2d.shape[0] mappairs = np.ones((2), dtype=int) dist=np.sqrt((longitude - lon2d)**2. + (latitude - lat2d)**2.) mindist = np.min(dist) minydist, minxdist = index_mat(dist,mindist) mappairs[:] = [minydist, minxdist] print ' ' + fname + ': nearest point: ',mappairs, 'distance:', mindist onc.close() vals = dnames[1] + ',' + str(mappairs[1]) + ',' + str(mappairs[1]) + ',1@' vals = vals + dnames[0] + ',' + str(mappairs[0]) + ',' + str(mappairs[0]) + ',1' DataSetSection_multidims(vals, ncfile) of = ncfile.split('.')[0] + '_' + dnames[1] + '_B' + str(mappairs[1]) + '-E' + \ str(mappairs[1]) + '-I1_' + dnames[0] + '_B' + str(mappairs[0]) + '-E' + \ str(mappairs[0]) + '-I1.nc' sub.call(['mv',of,ofile]) onc = NetCDFFile(ofile, 'a') newdim = onc.createDimension('lLd',1) newvar = onc.createVariable('lon_point','f',('lLd')) newattr = basicvardef(newvar,'lon_point','longitude of selection of the values', \ 'degrees_East') newvar[:] = longitude newvar = onc.createVariable('lat_point','f',('lLd')) newattr = basicvardef(newvar,'lat_point','latitude of selection of the values', \ 'degrees_North') newvar[:] = latitude newvar = onc.createVariable('flon_point','f',('lLd')) newattr = basicvardef(newvar,'flon_point','closest longitude of selection of ' + \ 'the values', 'degrees_East') newvar[:] = lon2d[tuple(mappairs)] newvar = onc.createVariable('flat_point','f',('lLd')) newattr = basicvardef(newvar,'flat_point','closest latitude of selection of ' + \ 'the values', 'degrees_North') newvar[:] = lat2d[tuple(mappairs)] newvar = onc.createVariable('fdist_point','f',('lLd')) newattr = basicvardef(newvar,'fdist_point','distance to the point of selection '+\ 'of the values', 'degrees') newvar[:] = mindist onc.sync() onc.close() print fname + ": successful writing of file '" + ofile + "' !!" return #vals='20.5:36.0:west_east,south_north:XLONG_M,XLAT_M' #ncf='/home/lluis/etudes/domains/medic051215_2km/geo_em.d01.nc' def put_variable_slice(inc, onc, vn, sliced): """ Function to add a variable from a netcdf object to a new one following a slice inc= input netcdf object onc= output netcdf object vn= variable name to include sliced= dictionary with the dimension name as key and [beg, end, int] as values if beg=-9, means that values for the dimension corresponds for a series of individual values """ fname = 'put_variable_slice' if not gen.searchInlist(inc.variables,vn): print errormsg print ' ' + fname + ": variable '" + vn + "' is not in file!!" quit(-1) ov = inc.variables[vn] # dnvs = ov.dimensions dnvs = [] varslice = [] # Dictionary with the size of the sliced dimensions newdimsizes = {} # Getting variable's dimensions Nslices = 1 for dnv in ov.dimensions: if gen.searchInlist(sliced.keys(),dnv): if sliced[dnv][0] == -9: newdimsize = len(sliced[dnv])-1 Nslices = newdimsize else: if sliced[dnv][2] != 0: newdimsize = len(range(sliced[dnv][0], sliced[dnv][1], \ sliced[dnv][2])) else: newdimsize = 0 newdimsizes[dnv] = newdimsize # Creation of dimensions according to the slice if not gen.searchInlist(onc.dimensions, dnv): od = inc.dimensions[dnv] if od.isunlimited(): newdim = onc.createDimension(dnv, None) else: if newdimsize > 0: newdim = onc.createDimension(dnv, newdimsize) else: #if not gen.searchInlist(onc.dimensions,dnv): print ' ' + fname + ": adding not sliced dimension '" + dnv + "'" # Avoiding monotone dimensions! od = inc.dimensions[dnv] if od.isunlimited(): dsize = None dvsize = len(od) else: dsize = len(od) dvsize = len(od) newdim = onc.createDimension(dnv, dsize) newdimsizes[dnv] = dvsize sliced[dnv] = [0, dvsize, 1] # Getting variable varattrs = ov.ncattrs() if gen.searchInlist(varattrs, '_FillValue'): varfil = ov._FillValue else: varfil = False vartype = ov.dtype # List of slices for idind in range(Nslices): newvarslice = [] for dnv in ov.dimensions: if gen.searchInlist(sliced.keys(),dnv): if sliced[dnv][0] != -9: if newdimsizes[dnv] > 0: varslice.append(slice(sliced[dnv][0], sliced[dnv][1], \ sliced[dnv][2])) dnvs.append(dnv) newvarslice.append(slice(sliced[dnv][0], sliced[dnv][1], \ sliced[dnv][2])) else: varslice.append(sliced[dnv][0]) else: varslice.append(sliced[dnv][idind]) newvarslice.append(idind) else: varslice.append(slice(0,len(inc.dimensions[dnv]))) dnvs.append(dnv) newvarslice.append(slice(0,len(inc.dimensions[dnv]))) if idind == 0: newvar = onc.createVariable(vn, vartype, tuple(dnvs), fill_value=varfil) finalvals = np.squeeze(ov[tuple(varslice)]) newvar[tuple(newvarslice)] = finalvals for attrs in varattrs: if not attrs == '_FillValue': attrv = ov.getncattr(attrs) attr = set_attribute(newvar, attrs, attrv) return def DataSetSection_multivars(values, filen, varn): """ Function to get a section (values along multiple variables) of a given data-set values= [varn1],[beg1],[end1],[int1]@[...[[varnM],[begM],[endM],[intM]]] [varni]: name of the variable with exception of: 'dist;[var1],[val1]|[...,[varN],val2]] distance computed from sqrt[sum_i([vari]-[vali])^2)] variables have to be of the same shape and share dimensions [begi],[endi],[inti]: beginning, end and interval along the variable [endi] = -1, maximum value [inti] = -1, all the values within the range (default) = 0, single value. [begi] will be taken as the reference value NOTE: variables without section by the variables are taken allover their size NOTE2: if inti = 'str', it is assumed that the variable is a string-list of values then [begi] must be the string to be used for the search ('!', for spaces) filen= netCDF with the data-set varn= ',' list of variables ('all', for all variables) """ # Not working 'WRFtime', for WRF time varibale import numpy.ma as ma fname = 'DataSetSection_multivars' # Variables not to check in the file NOcheck = ['WRFtime','dist'] if values == 'h': print fname + '_____________________________________________________________' print DataSetSection_multivars.__doc__ quit() ofile = 'DataSetSection_multivars.nc' Nvars = len(values.split('@')) nciobj = NetCDFFile(filen,'r') # Slicing variables varns = [] slicevalS = '' begvs = np.zeros((Nvars), dtype=int) endvs = np.zeros((Nvars), dtype=int) intvs = np.zeros((Nvars), dtype=int) fbegvs = np.zeros((Nvars), dtype=np.float) fendvs = np.zeros((Nvars), dtype=np.float) fintvs = np.zeros((Nvars), dtype=np.float) # Dictionary with the indices with coincident values dimvals = {} for ivar in range(Nvars): val = values.split('@')[ivar] vn = val.split(',')[0] if not gen.searchInlist(nciobj.variables,vn) and \ not gen.searchInlist(NOcheck,vn) and vn[0:4] != 'dist': print errormsg print ' ' + fname + ": file '" + filen + "' does not have variable '" + \ vn + "' !!" print ' it has:', list(nciobj.variables) quit(-1) if val[0:4] == 'dist': # Selection by distance varns.append('dist') vardist = val.split(';')[1].split('|') varsdist = {} iiv = 0 distv = 0. # Computing distance and looking for the minimal distance for vdn in vardist: dvarn = vdn.split(',')[0] dvalS = vdn.split(',')[1] if not gen.searchInlist(nciobj.variables,dvarn): print errormsg print ' '+fname+": file '"+filen+"' does not have variable '" + \ dvarn + "' !!" print ' it has:', list(nciobj.variables) quit(-1) oval = nciobj.variables[dvarn] Vvals = oval[:] dval = gen.retype(dvalS,oval.dtype) if iiv == 0: valsfirst = Vvals valsdims = oval.dimensions if dvalS[0:1] == '-': LdvalS = len(dvalS) distS = '('+ dvarn + '+' + dvalS[1:LdvalS+1] + ')^2' else: distS = '('+ dvarn + '-' + dvalS + ')^2' distv = (Vvals - dval)**2 else: # Checking for consistency among gen.same_shape(valsfirst, Vvals) if dvalS[0:1] == '-': LdvalS = len(dvalS) distS = distS + ' + ('+ dvarn + '+' + dvalS[1:LdvalS+1] + ')^2' else: distS = distS + ' + ('+ dvarn + '-' + dvalS + ')^2' distv = distv + (Vvals - dval)**2 iiv = iiv + 1 mindist = np.min(distv) iDistmindist = gen.multi_index_mat(distv, mindist) print infmsg print ' ' + fname + ": found a distance searching as dist=sqrt(" + \ distS + ")" varsdist[dvarn] = dval print ' found:', len(iDistmindist),'values at distance:', mindist iid = 0 for dimn in valsdims: dvvs = [] for ifound in range(len(iDistmindist)): foundvals = iDistmindist[ifound] if len(dvvs) == 0: dvvs = [int(foundvals[iid])] # getting indices for a given dimension including non-repeated if not gen.searchInlist(dvvs, foundvals[iid]): dvvs.append(int(foundvals[iid])) iid = iid + 1 # Including only that indices coincident with previous founds if dimvals.has_key(dimn): foundvals = dimvals[dimn] coincvals = list(set(foundvals).intersection(set(dvvs))) if len(coincvals) == 0: print errormsg print ' ' + fname + ': no coincidencies among searches ' + \ "found for dimension '" + dimn + "' !!" print ' revise criteria of selection of values' quit(-1) else: dimvals[dimn] = coincvals else: dimvals[dimn] = dvvs elif val.split(',')[3] == 'str': # String search ostrvar = nciobj.variables[vn] ostrvardims = ostrvar.dimensions varns.append(vn) istrvalue = val.split(',')[1].replace('!',' ') estrvalue = val.split(',')[2].replace('!',' ') print warnmsg print ' ' + fname + ": String value for variable '" + val + "' !!" print ' assuming string variable as [Nvalues, Lstring]' if istrvalue != estrvalue: print " from which is required from '" + istrvalue + "' to '" + \ estrvalue + "'" else: print " from which only is required '" + istrvalue + "'" strvals = get_str_nc(ostrvar, ostrvar.shape[1]) if not gen.searchInlist(strvals, istrvalue): print errormsg print ' ' + fname + ": variable '" + val + "' does not have " + \ "value= '" + istrvalue + "' !!" print ' values:', strvals quit(-1) if istrvalue != estrvalue: if not gen.searchInlist(strvals, estrvalue): print errormsg print ' ' + fname + ": variable '" + val + "' does not have " + \ "value= '" + estrvalue + "' !!" print ' values:', strvals quit(-1) dimvals[ostrvardims[1]] = range(ostrvar.shape[1]) dimn = ostrvardims[0] if istrvalue == estrvalue: Nfound = strvals.count(istrvalue) print infmsg print ' '+fname+ ": variable '" + vn + "' has:", Nfound, "times " + \ "value '" + istrvalue + "'" # Values found for each dimension # Assuming shape as said copyvals = list(strvals) dvvs = [] for iif in range(Nfound): iidstr = copyvals.index(istrvalue) dvvs.append(iidstr+iif) copyvals.pop(iidstr) else: ibeg = strvals.index(istrvalue) iend = strvals.index(estrvalue) dvvs = range(ibeg,iend) # Including only that indices coincident with previous founds if dimvals.has_key(dimn): foundvals = dimvals[dimn] coincvals = list(set(foundvals).intersection(set(dvvs))) if len(coincvals) == 0: print errormsg print ' ' + fname + ': no coincidencies among searches found '+ \ "for dimension '" + dimn + "' !!" print ' revise criteria of selection of values' quit(-1) else: dimvals[dimn] = coincvals else: dimvals[dimn] = dvvs indstrvalue = strvals.index(istrvalue) endstrvalue = strvals.index(estrvalue) begvs[ivar] = np.float(indstrvalue) endvs[ivar] = np.float(indstrvalue) intvs[ivar] = -99. else: # Range search ovar = nciobj.variables[vn] vals = ovar[:] valsdims = ovar.dimensions varns.append(vn) begvs = gen.retype(val.split(',')[1], ovar.dtype) endvs = gen.retype(val.split(',')[2], ovar.dtype) intvs = -1 fbegvs[ivar] = np.float(val.split(',')[1]) fendvs[ivar] = np.float(val.split(',')[2]) fintvs[ivar] = np.float(val.split(',')[3]) if endvs == -1: endvs = np.max(vals) maskinf = ma.masked_less(vals, begvs) masksup = ma.masked_greater(vals, endvs) # For all masked or not, ma.masked returns a single value... if len(maskinf.mask.flatten()) == 1: if maskinf.mask: maskinfv = np.ones((vals.shape), dtype=bool) else: maskinfv = np.zeros((vals.shape), dtype=bool) else: maskinfv = maskinf.mask if len(masksup.mask.flatten()) == 1: if masksup.mask: masksupv = np.ones((vals.shape), dtype=bool) else: masksupv = np.zeros((vals.shape), dtype=bool) else: masksupv = masksup.mask # if and len(masksup.mask.flatten()) == 1: # if not maskinf.mask*maskinf.mask: # finalmask = np.ones() # else: # finalmask = maskinf.mask + masksup.mask #if intvs[ivar] != -1: # print errormsg # print ' ' + fname + ': non-consecutive slices not ready!!' # quit(-1) # Indices of the found values finalmask = maskinfv + masksupv # Single value if val.split(',')[3] == '0': print infmsg print ' ' + fname + ": single value for variable '" + vn + "' =", \ begvs finalmask = np.where(vals == begvs, False, finalmask) iDistmindist = gen.multi_index_mat(finalmask, False) print ' found:', len(iDistmindist),"values in variable '" + vn + \ "' as; beginning:", begvs, 'ending:', endvs, 'interval:', intvs iid = 0 for dimn in valsdims: dvvs = [] for ifound in range(len(iDistmindist)): foundvals = iDistmindist[ifound] # getting indices for a given dimension dvvs.append(int(foundvals[iid])) iid = iid + 1 # Including only that indices coincident with previous founds if dimvals.has_key(dimn): foundvals = dimvals[dimn] coincvals = list(set(foundvals).intersection(set(dvvs))) if len(coincvals) == 0: print errormsg print ' ' + fname + ': no coincidencies among searches ' + \ "found for dimension '" + dimn + "' !!" print ' revise criteria of selection of values' quit(-1) else: dimvals[dimn] = coincvals else: dimvals[dimn] = dvvs if intvs != -1: slicevalS = slicevalS + varns[ivar] + ' (' + str(begvs) + ',' + \ str(fendvs) + ',' + str(fintvs) + '); ' else: slicevalS = slicevalS + varns[ivar] + ' (' + str(fbegvs) + ',' + \ str(fendvs) + ',1); ' print infmsg print ' ' + fname + ': Indices found for each dimensions ____' finalslice = {} for dimn in dimvals.keys(): ddvals = dimvals[dimn] print ' ' + dimn + ':', ddvals if len(dimvals[dimn]) > 1: Lddvals = len(ddvals) if np.all(np.array(ddvals[1:Lddvals])-np.array(ddvals[0:Lddvals-1])) == 1: finalslice[dimn] = [ddvals[0], ddvals[Lddvals-1], 1] else: finalslice[dimn] = [-9] + ddvals else: finalslice[dimn] = [ddvals[0], ddvals[0], 0] ncoobj = NetCDFFile(ofile,'w') # Output variables ## if varn == 'all': variables = nciobj.variables else: if varn.find(',') != -1: variables = varn.split(',') else: variables = [varn] for varn in variables: if not gen.searchInlist(nciobj.variables, varn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have variable '" + \ dvarn + "' !!" print ' it has:', list(nciobj.variables) quit(-1) ovar = nciobj.variables[varn] vardims = ovar.dimensions put_variable_slice(nciobj, ncoobj, varn, finalslice) # Global attributes ## for attrs in nciobj.ncattrs(): attrv = nciobj.getncattr(attrs) attr = set_attribute(ncoobj, attrs, attrv) attr = set_attribute(ncoobj, 'sliced_variables', slicevalS) add_global_PyNCplot(ncoobj, main, fname, '1.1') nciobj.close() ncoobj.sync() ncoobj.close() print ' ' + fname + ' succesfull creation of file "' + ofile + '" !!!' return #DataSetSection_multivars('XTIME,4380,4800,-1','/home/lluis/PY/wrfout_d01_2001-11-11_00:00:00','T2') #DataSetSection_multivars('WRFtime,20011111060000,20011111180000,-1','/home/lluis/PY/wrfout_d01_2001-11-11_00:00:00','all') #quit() def get_attribute(values, filen, varn): """ Function to get an attribute from a netCDF file values= [attrname] attrname: attribute name filen= name of the netCDF file varn= name of the variable ('global' for a global attribute) """ fname = 'get_attribute' attrn = values if values == 'h': print fname + '_____________________________________________________________' print get_attribute.__doc__ quit() onc = NetCDFFile(filen, 'r') if varn == 'global': if not gen.searchInlist(onc.ncattrs(), attrn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have atribute '" + \ attrn + "' !!" quit(-1) else: attrv = onc.getncattr(attrn) else: if not gen.searchInlist(onc.variables, varn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have variable '" + \ varn + "' !!" quit(-1) else: ovar = onc.variables[varn] if not gen.searchInlist(ovar.ncattrs(), attrn): print errormsg print ' ' + fname + ": variable '" + varn + "' does not have " + \ "atribute '" + attrn + "' !!" print ' it has:',ovar.ncattrs() quit(-1) else: attrv = ovar.getncattr(attrn) onc.close() return attrv def DimsLoop(ovar,seldims): """ Function to provide the shape of the dimensions which are not selected ova= variable object seldims= list of dimensions to select """ fname = 'DimsLoop' newshape = [] noloopdims = [] idim = 0 for dim in list(ovar.dimensions): if not gen.searchInlist(seldims,dim): newshape.append(ovar.shape[idim]) noloopdims.append(dim) idim = idim + 1 return newshape, noloopdims def SliceVar(ovar,dloop,dloopindex): """ Function to slice a given variable throughout a given list of dimensions ovar= variable object to slice dloop= list of dimensions to get a single value dloopindex= value of the loop dimension to get """ fname = 'SliceVar' varshape = ovar.shape slicevals = [] idim = 0 for dim in ovar.dimensions: if gen.searchInlist(dloop,dim): slicevals.append(dloopindex[idim]) else: slicevals.append(slice(0,varshape[idim])) idim = idim + 1 return slicevals def SliceVarDict(ovar,slicedict): """ Function to slice a given variable throughout a dictionary ovar= variable object to slice slicedict= dictionary with the values to slice slicedict[dimn] = [value] [value] = -1, all the values [value] = -9, last values [value] = int, a single value [value] = [beg, end, frq], from a beginning to an end with a given frequency returning: slicevals: list with the values of the slice slicefinaldims: final dimensions of the slice (without monotonic dimensions) """ fname = 'SliceVarDict' varshape = ovar.shape slicevals = [] slicefinaldims = [] idim = 0 for dim in ovar.dimensions: if gen.searchInlist(slicedict.keys(),dim): dictv = slicedict[dim] if type(dictv) == type([1]): slicevals.append( slice(dictv[0], dictv[1], dictv[2]) ) slicefinaldims.append(dim) elif type(dictv) == type(int(1)): if dictv == -1: slicevals.append(slice(0,varshape[idim])) slicefinaldims.append(dim) elif dictv == -9: slicevals.append(varshape[idim]-1) slicefinaldims.append(dim) else: slicevals.append(dictv) else: slicevals.append(slice(0,varshape[idim])) slicefinaldims.append(dim) idim = idim + 1 return slicevals, slicefinaldims def varDimension(oncf, dname): """ Function to find the variable with the values of a dimension. It is assumed, that given variable might have a shape with only the dimension, double precision oncf: netCDF object dname: name of the dimension """ fname = 'varDimension' if not oncf.dimensions.has_key(dname): print errormsg print ' ' + fname + ": NetCDF object does not have dimension '" + dname + "' !!" quit(-1) # 1D variables with the dimension name ## d1var = [] for vn in oncf.variables: ovn = oncf.variables[vn] if len(ovn.shape) == 1 and ovn.dimensions[0] == dname: d1var.append(vn) if len(d1var) != 0: print ' ' + fname + ": 1D variables with dimension '" + dname + "':",d1var else: print warnmsg print ' ' + fname + ": Any 1D variables with dimension '" + dname + "' !!" return # Dpouble precision ## if len(d1var) == 0: dimvarname = None if len(d1var) > 1: d1vardouble = [] for vn in d1var: ovn = oncf.variables[vn] if ovn.dtype == type(np.float64(1.)): d1vardouble.append(vn) if len(d1vardouble) > 1: print errormsg print ' ' + fname + ": Found more than 1D double variable with " + \ "dimension '" + dname + "' !!" print ' found:', d1vardouble quit(-1) elif len(d1vardouble) == 0: print warnmsg print ' ' + fname + ": Found any 1D double variable with dimension '" + \ dname + "' !!" dimvarname = None else: dimvarname = d1vardouble[0] else: dimvarname = d1var[0] return dimvarname def ovar_onc(incf, ovar, oncf): """ Function to copy an object variable to a nother netcdf object incf: input netCDF object ovar= variable object oncf= netCDF object """ fname = 'ovar_onc' varname = ovar._name if oncf.variables.has_key(varname): print warnmsg print ' ' + fname + ": netCDF object already has variable '" + varname + \ "' !!" print ' doing noting' return vardims = ovar.dimensions vartype = ovar.dtype for dn in vardims: if not gen.searchInlist(oncf.dimensions, dn): if incf.dimensions[dn].isunlimited(): newdim = oncf.createDimension(dn, None) else: newdim = oncf.createDimension(dn, len(incf.dimensions[dn])) newvar = oncf.createVariable(varname, vartype, vardims) newvar[:] = ovar[:] for attrn in ovar.ncattrs(): attrv = ovar.getncattr(attrn) set_attribute(newvar, attrn, attrv) oncf.sync() return def SpatialWeightedMean(values, filen, varn): """ Function to compute the spatial mean using weights from a netCDF file values= [weightskind],[xdimname],[ydimname],[addvars] [weightskind]: type of weights: * 'varnames',[varname],[oper]: using a variable [varname] with an operation [oper] as area size [oper]: available operations 'inv': inverse of the values 1/[varname] 'nothing': direct values 'prodinv': inverse of the product between 2 variables ([varname1]*[varname2]), NOTE: [varname] = [varname1]:[varname2] * 'reglonlat',[lonname],[latname]: regular lon/lat projection to compute the area size [xdimname]: name of the dimension for the x-axis [ydimname]: name of the dimension for the y-axis [addvars]: ':', separetd list of name of variables to add to the output file ('None' for any) filen= name of the netCDF file varn= name of the variable """ fname = 'SpatialWeightedMean' if values == 'h': print fname + '_____________________________________________________________' print SpatialWeightedMean.__doc__ quit() operations = ['inv','nothing','prodinv'] weightk = values.split(',')[0] xdimname = values.split(',')[3] ydimname = values.split(',')[4] addvars = values.split(',')[5] ofile = 'SpatialWeightedMean_' + weightk + '.nc' if weightk == 'varnames': arguments = '[weightk],[varname],[oper],[xdimname],[ydimname],[addvars]' gen.check_arguments(fname, values, arguments, ',') varname = values.split(',')[1] oper = values.split(',')[2] if not gen.searchInlist(operations, oper): print errormsg print ' ' + fname + ": operation '" + oper + "' not ready!!" print ' ready operations:',operations quit(-1) elif weightk == 'reglonlat': arguments = '[weightk],[lonname],[latname],[xdimname],[ydimname],[addvars]' gen.check_arguments(fname, values, arguments, ',') lonname = values.split(',')[1] latname = values.split(',')[2] else: print errormsg print ' ' + fname + ": wieght type '" + weightk + "' not ready!!" quit(-1) # Operations ## onc = NetCDFFile(filen, 'r') if not onc.variables.has_key(varn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have variable '" + \ varn + "' !!" quit(-1) iovar = onc.variables[varn] if not gen.searchInlist(list(iovar.dimensions),xdimname) and not \ gen.searchInlist(list(iovar.dimensions),ydimname): print errormsg print ' ' + fname + ": variable '" + varn + "' does not have dimension '" + \ xdimname + "' neither '" + ydimname + "' !!" quit(-1) # Variable attributes filemiss = False ivarattrs = iovar.ncattrs() if gen.searchInlist(ivarattrs, 'missing_value'): filemissv = iovar.getncattr('missing_value') filemiss = True print ' ' + fname + ' input data with missing value:',filemissv,'!!' filefill = False if gen.searchInlist(ivarattrs, '_FillValue'): filefillv = iovar.getncattr('_FillValue') filefill = True print ' ' + fname + ' input data with fill value:',filefillv,'!!' # Getting shape of output variables for future average loop ## loopshape, dimsloop = DimsLoop(iovar,[xdimname, ydimname]) if len(loopshape) > 3: print erromsg print ' ' + fname + ': output size not ready!!' quit(-1) ivarv = iovar[:] if weightk == 'varnames': if oper == 'inv': if not onc.variables.has_key(varname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varname + "' !!" quit(-1) iovarwgt = onc.variables[varname] ivarwgtv = iovarwgt[:] elif oper == 'nothing': if not onc.variables.has_key(varname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varname + "' !!" quit(-1) iovarwgt = onc.variables[varname] ivarwgtv = iovarwgt[:] elif oper == 'prodinv': varname1 = varname.split(':')[0] varname2 = varname.split(':')[1] if not onc.variables.has_key(varname1): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varname1 + "' !!" quit(-1) if not onc.variables.has_key(varname2): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varname2 + "' !!" quit(-1) iovarwgt1 = onc.variables[varname1] iovarwgt2 = onc.variables[varname2] ivarwgtv = iovarwgt1[:]*iovarwgt2[:] if oper == 'inv': longvarname = 'using inverse of variable ' + varname + ' as space weights' if len(loopshape) == 1: newvals = np.ones((loopshape[0]), dtype=np.float)*gen.fillValueF newsumvals = np.ones((loopshape[0]), dtype=np.float)*gen.fillValueF for id1 in range(loopshape[0]): slicevalues = SliceVar(iovar,dimsloop,[id1]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1]) TOTsumwgt = np.sum(1./ivarwgtv[tuple(slicewgt)]) newsumvals[id1] = np.sum(ivarv[tuple(slicevalues)] / \ ivarwgtv[tuple(slicewgt)]) newvals[id1] = np.sum(ivarv[tuple(slicevalues)] / \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt elif len(loopshape) == 2: newvals = np.ones((loopshape[0],loopshape[1]),dtype=np.float)* \ gen.fillValueF newsumvals = np.ones((loopshape[0],loopshape[1]),dtype=np.float)* \ gen.fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1,id2]) TOTsumwgt = np.sum(1./ivarwgtv[tuple(slicewgt)]) newsumvals[id1,id2] = np.sum(ivarv[tuple(slicevalues)] / \ ivarwgtv[tuple(slicewgt)]) newvals[id1,id2] = np.sum(ivarv[tuple(slicevalues)] / \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt elif len(loopshape) == 3: newvals = np.ones((loopshape[0],loopshape[1],loopshape[2]), \ dtype=np.float)*gen.fillValueF newsumvals = np.ones((loopshape[0],loopshape[1],loopshape[2]), \ dtype=np.float)*gen.fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): for id3 in range(loopshape[1]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2,id3]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1,id2,id3]) TOTsumwgt = np.sum(1./ivarwgtv[tuple(slicewgt)]) newsumvals[id1,id2,id3]=np.sum(ivarv[tuple(slicevalues)]/\ ivarwgtv[tuple(slicewgt)]) newvals[id1,id2,id3]= np.sum(ivarv[tuple(slicevalues)]/ \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt outweightvals = 1./ivarwgtv[tuple(slicewgt)] elif oper == 'nothing': longvarname = 'using variable ' + varname + ' as space weights' if len(loopshape) == 1: newvals = np.ones((loopshape[0]), dtype=np.float)*gen.fillValueF newsumvals = np.ones((loopshape[0]), dtype=np.float)*gen.fillValueF for id1 in range(loopshape[0]): slicevalues = SliceVar(iovar,dimsloop,[id1]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1]) TOTsumwgt = np.sum(ivarwgtv[tuple(slicewgt)]) newsumvals[id1] = np.sum(ivarv[tuple(slicevalues)]* \ ivarwgtv[tuple(slicewgt)]) newvals[id1] = np.sum(ivarv[tuple(slicevalues)]* \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt elif len(loopshape) == 2: newvals = np.ones((loopshape[0],loopshape[1]), dtype=np.float)* \ gen.fillValueF newsumvals = np.ones((loopshape[0],loopshape[1]), dtype=np.float)* \ gen.fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1,id2]) TOTsumwgt = np.sum(ivarwgtv[tuple(slicewgt)]) newsumvals[id1,id2] = np.sum(ivarv[tuple(slicevalues)]* \ ivarwgtv[tuple(slicewgt)]) newvals[id1,id2] = np.sum(ivarv[tuple(slicevalues)]* \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt elif len(loopshape) == 3: newvals = np.ones((loopshape[0],loopshape[1],loopshape[2]), \ dtype=np.float)*gen.fillValueF newsumvals = np.ones((loopshape[0],loopshape[1],loopshape[2]), \ dtype=np.float)*gen.fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): for id3 in range(loopshape[2]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2.id3]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1,id2,id3]) TOTsumwgt = np.sum(ivarwgtv[tuple(slicewgt)]) newsumvals[id1,id2,id3]=np.sum(ivarv[tuple(slicevalues)]*\ ivarwgtv[tuple(slicewgt)]) newvals[id1,id2,id3]=np.sum(ivarv[tuple(slicevalues)]* \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt outweightvals = ivarwgtv[tuple(slicewgt)] if oper == 'prodinv': longvarname = 'using inverse of product of variables ' + varname1 + '*' \ + varname2 + ' as space weights' if len(loopshape) == 1: newvals = np.ones((loopshape[0]), dtype=np.float)*gen.fillValueF newsumvals = np.ones((loopshape[0]), dtype=np.float)*gen.fillValueF for id1 in range(loopshape[0]): slicevalues = SliceVar(iovar,dimsloop,[id1]) slicewgt = SliceVar(iovarwgt1,dimsloop,[id1]) TOTsumwgt = np.sum(1./ivarwgtv[tuple(slicewgt)]) newsumvals[id1] = np.sum(ivarv[tuple(slicevalues)] / \ ivarwgtv[tuple(slicewgt)]) newvals[id1] = np.sum(ivarv[tuple(slicevalues)] / \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt elif len(loopshape) == 2: newvals = np.ones((loopshape[0],loopshape[1]),dtype=np.float)* \ gen.fillValueF newsumvals = np.ones((loopshape[0],loopshape[1]),dtype=np.float)* \ gen.fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1,id2]) TOTsumwgt = np.sum(1./ivarwgtv[tuple(slicewgt)]) newsumvals[id1,id2] = np.sum(ivarv[tuple(slicevalues)] / \ ivarwgtv[tuple(slicewgt)]) newvals[id1,id2] = np.sum(ivarv[tuple(slicevalues)] / \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt elif len(loopshape) == 3: newvals = np.ones((loopshape[0],loopshape[1],loopshape[2]), \ dtype=np.float)*gen.fillValueF newsumvals = np.ones((loopshape[0],loopshape[1],loopshape[2]), \ dtype=np.float)*gen.fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): for id3 in range(loopshape[1]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2,id3]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1,id2,id3]) TOTsumwgt = np.sum(1./ivarwgtv[tuple(slicewgt)]) newsumvals[id1,id2,id3]=np.sum(ivarv[tuple(slicevalues)]/\ ivarwgtv[tuple(slicewgt)]) newvals[id1,id2,id3]= np.sum(ivarv[tuple(slicevalues)]/ \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt outweightvals = 1./ivarwgtv[tuple(slicewgt)] elif weightk == 'reglonlat': longvarname = 'using shoelace method of normalized ds_x,ds_y as space weights' if not onc.variables.has_key(lonname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have variable '" + \ lonname + "' !!" quit(-1) if not onc.variables.has_key(latname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have variable '" + \ latname + "' !!" quit(-1) iolon = onc.variables[lonname] iolat = onc.variables[latname] ilonv = iolon[:] ilatv = iolat[:] lonv, latv = gen.lonlat2D(ilonv, ilatv) # Direct and wrong #ivarwgtv = np.abs(np.cos(latv*np.pi/180.)) # Using shoelace formula dx = lonv.shape[1] dy = lonv.shape[0] dlon = np.zeros((lonv.shape), dtype=np.float) dlat = np.zeros((latv.shape), dtype=np.float) ivarwgtv = np.zeros((latv.shape), dtype=np.float) dlon[:,0:dx-1] = lonv[:,1:dx] - lonv[:,0:dx-1] dlat[0:dy-1,:] = latv[1:dy,:] - latv[0:dy-1,:] dlon[:,dx-1] = lonv[:,dx-1] - lonv[:,dx-2] dlat[dy-1,:] = latv[dy-1,:] - latv[dy-2,:] coslat = np.abs(np.cos(latv*np.pi/180.)) dsx = EarthR*coslat*dlon dsy = EarthR*dlat for j in range(dy-1): for i in range(dx-1): # xvals = [dsx[j,i], dsx[j+1,i], dsx[j+1,i+1], dsx[j,i+1]] # yvals = [dsy[j,i], dsy[j+1,i], dsy[j+1,i+1], dsy[j,i+1]] xvals = [0., 0., dsx[j+1,i+1], dsx[j,i+1]] yvals = [0., dsy[j+1,i], dsy[j+1,i+1], 0.] ivarwgtv[j,i] = gen.PolyArea(xvals, yvals) j=dy-1 for i in range(dx-1): xvals = [0., 0., dsx[j-1,i+1], dsx[j,i-1]] yvals = [0., dsy[j-1,i], dsy[j-1,i+1], 0.] ivarwgtv[j,i] = gen.PolyArea(xvals, yvals) i=dx-1 for j in range(dy-1): xvals = [0., 0., dsx[j+1,i-1], dsx[j,i-1]] yvals = [0., dsy[j+1,i], dsy[j+1,i-1], 0.] ivarwgtv[j,i] = gen.PolyArea(xvals, yvals) i=dx-1 j=dy-1 xvals = [0., 0., dsx[j-1,i-1], dsx[j,i-1]] yvals = [0., dsy[j-1,i], dsy[j-1,i-1], 0.] ivarwgtv[j,i] = gen.PolyArea(xvals, yvals) TOTsumwgt = np.sum(ivarwgtv) if len(loopshape) == 1: newvals = np.ones((loopshape[0]), dtype=np.float)*gen.fillValueF newsumvals = np.ones((loopshape[0]), dtype=np.float)*gen.fillValueF for id1 in range(loopshape[0]): slicevalues = SliceVar(iovar,dimsloop,[id1]) vals = ivarv[tuple(slicevalues)] if filemiss: vals = np.where(vals == filemissv, 0., vals) if filefill: vals = np.where(vals == filefillv, 0., vals) newsumvals[id1] = np.sum(vals*ivarwgtv) newvals[id1] = newsumvals[id1]/TOTsumwgt elif len(loopshape) == 2: newvals = np.ones((loopshape[0],loopshape[1]), \ dtype=np.float)*gen.fillValueF newsumvals = np.ones((loopshape[0],loopshape[1]), \ dtype=np.float)*gen.fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2]) vals = ivarv[tuple(slicevalues)] if filemiss: vals = np.where(vals == filemissv, 0., vals) if filefill: vals = np.where(vals == filefillv, 0., vals) newsumvals[id1,id2] = np.sum(vals*ivarwgtv) newvals[id1,id2] = newsumvals[id1,id2] / TOTsumwgt elif len(loopshape) == 3: newvals = np.ones((loopshape[0],loopshape[1],loopshape[2]), \ dtype=np.float)*gen.fillValueF newsumvals = np.ones((loopshape[0],loopshape[1],loopshape[2]), \ dtype=np.float)*gen.fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): for id3 in range(loopshape[2]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2,id3]) vals = ivarv[tuple(slicevalues)] if filemiss: vals = np.where(vals == filemissv, 0., vals) if filefill: vals = np.where(vals == filefillv, 0., vals) newsumvals[id1,id2,id3] = np.sum(vals*ivarwgtv) newvals[id1,id2,id3] = newsumvals[id1,id2,id3] / TOTsumwgt outweightvals = ivarwgtv/TOTsumwgt # Writting output file ## print 'TOTAL sunm of weights:', np.sum(outweightvals) onewnc = NetCDFFile(ofile, 'w') # Dimensions creation ## for dim in dimsloop: newdim = onewnc.createDimension(dim, len(onc.dimensions[dim])) vardim = varDimension(onc, dim) if vardim is not None: ovar_onc(onc, onc.variables[vardim], onewnc) # Output variable ## newvar = onewnc.createVariable(varn + 'spaceweightsum', 'f4', tuple(dimsloop), \ fill_value=gen.fillValueF) basicvardef(newvar, varn + 'spaceweightsum', 'space summed ' + varn + ' ' + \ longvarname, iovar.getncattr('units')) newvar[:] = newsumvals onewnc.sync() newvar = onewnc.createVariable(varn + 'spaceweightmean', 'f4', tuple(dimsloop), \ fill_value=gen.fillValueF) basicvardef(newvar, varn + 'spaceweightmean', 'space weighted ' + varn + ' ' + \ longvarname, iovar.getncattr('units')) newvar[:] = newvals onewnc.sync() # Spatial weight ## newdim = onewnc.createDimension(ydimname, outweightvals.shape[0]) newdim = onewnc.createDimension(xdimname, outweightvals.shape[1]) newvar = onewnc.createVariable('spatialweight', 'f4',tuple([ydimname, xdimname]),\ fill_value=gen.fillValueF) basicvardef(newvar, 'spatialweight', 'space weight ' + longvarname, '-') newvar[:] = outweightvals onewnc.sync() # No needed, only for checks !! # newvar = onewnc.createVariable('dlon', 'f4', tuple([ydimname, xdimname]), \ # fill_value=gen.fillValueF) # basicvardef(newvar, 'dlon', 'dlon', 'degrees_East') # newvar[:] = dlon[:] # newvar = onewnc.createVariable('dlat', 'f4', tuple([ydimname, xdimname]), \ # fill_value=gen.fillValueF) # basicvardef(newvar, 'dlat', 'dlat', 'degrees_North') # newvar[:] = dlat[:] # newvar = onewnc.createVariable('coslat', 'f4', tuple([ydimname, xdimname]), \ # fill_value=gen.fillValueF) # basicvardef(newvar, 'coslat', 'latitude cosinus', '-') # newvar[:] = coslat[:] # newvar = onewnc.createVariable('dsx', 'f4', tuple([ydimname, xdimname]), \ # fill_value=gen.fillValueF) # basicvardef(newvar, 'dsx', 'x distance', 'm') # newvar[:] = dsx[:] # newvar = onewnc.createVariable('dsy', 'f4', tuple([ydimname, xdimname]), \ # fill_value=gen.fillValueF) # basicvardef(newvar, 'dsy', 'y distance', 'm') # newvar[:] = dsy[:] # onewnc.sync() # Additional variables ## if addvars != 'None': for vn in addvars.split(':'): if not onc.variables.has_key(vn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have variable '" + \ vn + "' !!" quit(-1) ovar_onc(onc, onc.variables[vn], onewnc) onewnc.sync() # Global attributes ## onewnc.setncattr('script', fname) onewnc.setncattr('version', '1.0') onewnc.setncattr('author', 'L. Fita') newattr = set_attributek(onewnc, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') onewnc.setncattr('university', 'Pierre et Marie Curie') onewnc.setncattr('country', 'France') for attrs in onc.ncattrs(): attrv = onc.getncattr(attrs) attr = set_attribute(onewnc, attrs, attrv) onc.close() onewnc.sync() onewnc.close() print fname + ": Successfull written of file: '" + ofile + "' !!" return #SpatialWeightedMean('varnames,MAPFAC_M,inv,west_east,south_north','wrfout_d01_2001-11-11_00:00:00','HGT') #SpatialWeightedMean('reglonlat,XLONG,XLAT,west_east,south_north','wrfout_d01_2001-11-11_00:00:00','HGT') def nctype(vartype): """ Function to provide the string for the variable creation in a netCDF file [vartype]= type of the variable """ fname = 'nctype' if vartype == type('c'): ncs = 'c' elif vartype == '|S1': ncs = 'c' elif vartype == type(int(1)): ncs = 'i' elif vartype == type(np.int16(1)): ncs = 'i' elif vartype == type(np.int32(1)): ncs = 'i4' elif vartype == type(np.int64(1)): ncs = 'i8' elif vartype == type(np.float(1.)): ncs = 'f' elif vartype == type(np.float32(1.)): ncs = 'f4' elif vartype == type(np.float64(1.)): ncs = 'f8' else: print errormsg print ' ' + fname + ': variable type', vartype, 'not ready !!' quit(-1) return ncs def VarVal_FillValue(values, filen, varn): """ Function to transform a given value from a given variable to _FillValue in a netCDF file values= [value],[oper],[fillVal] [value]: value to pass to '_FillValue' [oper]: operation to follow with [value] to transform to `_FillValue' 'eq': equal than [value] 'ge': greater equal than [value] 'gt': greater than [value] 'le': less equal than [value] 'lt': less than [value] [fillVal]: value for '_FillValue', 'std' for the standard value which means: Float = 1.e20 Character = '-' Integer = -99999 Float64 = np.float(Float) Integer32 = np.int32(Integer) filen= name of the netCDF file varn= name of the variable """ fname = 'VarVal_FillValue' availopers = ['eq', 'ge', 'gt', 'le', 'lt'] if values == 'h': print fname + '_____________________________________________________________' print VarVal_FillValue.__doc__ quit() arguments = '[value],[oper],[fillVal]' gen.check_arguments(fname, values, arguments, ',') value = values.split(',')[0] oper = values.split(',')[1] fval0 = values.split(',')[2] onc = NetCDFFile(filen, 'a') if not onc.variables.has_key(varn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varn + "' !!" quit(-1) varo = onc.variables[varn] varvals = varo[:] vartype = varo.dtype if vartype == '|S': valchk = value elif vartype == type(int(1)): valchk = int(value) elif vartype == type(np.int32(1)): valchk = np.int32(value) elif vartype == type(np.float(1.)): valchk = np.float(value) elif vartype == type(np.float32(1.)): valchk = np.float32(value) elif vartype == type(np.float64(1.)): valchk = np.float64(value) else: print errormsg print ' ' + fname + ': variable type', vartype, 'not ready !!' quit(-1) fval = gen.fillvalue_kind(vartype, fval0) if oper == 'eq': newvarvals = np.where(varvals == valchk, fval, varvals) elif oper == 'ge': newvarvals = np.where(varvals >= valchk, fval, varvals) elif oper == 'gt': newvarvals = np.where(varvals > valchk, fval, varvals) elif oper == 'le': newvarvals = np.where(varvals <= valchk, fval, varvals) elif oper == 'lt': newvarvals = np.where(varvals < valchk, fval, varvals) else: print errormsg print ' ' + fname + ": operation '" + oper + "' not ready !!" print ' available ones:', availopers quit(-1) varo[:] = newvarvals[:] varattrs = varo.ncattrs() print 'varattrs 1:', varattrs if gen.searchInlist(varattrs, 'missing_value'): missval = varo.getncattr('missing_value') difvals = fval*10.**(-np.log10(valchk)) - missval*10.**(-np.log10(missval)) if missval == valchk or np.abs(difvals) < 1.e-6: print warnmsg print ' ' + fname + ': same missing and checking Value!' print ' renaming missing_value' varo.delncattr('missing_value') if not gen.searchInlist(varattrs, '_FillValue'): set_attribute(varo, '_FillValue', fval) else: varo.delncattr('_FillValue') set_attribute(varo, '_FillValue', fval) onc.sync() onc.close() print fname +': Successfull change to _FillValue !!' return #VarVal_FillValue('0.,eq,std', '/home/lluis/etudes/DYNAMICO/ORCHIDEE/interpolation/data/new_lai2D.nc', 'LAI') def lonlatProj(dxres, dyres, project, wfile): """ Function to compute longitudes and latitudes for a given projection following subroutine calc_resolution_in from ORCHIDEE/src_global/module_InterpWeight dxres= resolution on the WE axis [m] dyres= resolution on the SN axis [m] project= projection 'lonlat': standard lon/lat projection tacking number of points from [dx/dy]res at the Equator 'lonlat_dxdyFix': projection with a fixed grid distance wfile= Should an output file be written (True/False) """ fname = 'lonlatProj' ofile = 'lonlatProj.nc' # Given constants RadDeg = 180./np.pi Npts = 1 if project == 'lonlat': Nlon = int(2. * np.pi * EarthR / dxres) + 1 Nlat = int(np.pi * EarthR / dyres) + 1 if np.mod(Nlon,2) == 0: Nlon = Nlon + 1 if np.mod(Nlat,2) == 0: Nlat = Nlat + 1 lonProj = np.zeros((Nlon), dtype=np.float) latProj = np.zeros((Nlat), dtype=np.float) lonProj = -180. + np.arange(Nlon) * 2. * np.pi * RadDeg / (Nlon - 1) latProj = -90. + np.arange(Nlat) * np.pi * RadDeg / (Nlat - 1) dimx = Nlon dimy = Nlat Npts = dimx*dimy Npoints = np.arange((Npts)).reshape(dimy,dimx) elif project == 'lonlat_dxdyFix': lonProj = [] latProj = [] lonProjcirc = [] latProjcirc = [] Nlonmaxcirc = -10. lonProj.append(0.) latProj.append(-90.) lonProjcirc.append(0.) latProjcirc.append(-90.) # Differential on latitudes Nlat = int(np.pi * EarthR / dyres) + 3 latarc = np.pi / (Nlat - 1) Nloncirc = np.zeros((Nlat), dtype=int) Nloncirc[0] = 1 lat=-np.pi/2. ilat = 1 while lat + latarc <= np.pi/2.: lon=0. lat = lat + latarc lonarc = dxres / (EarthR * np.cos(lat)) lonscirc = [] latProjcirc.append(lat*RadDeg) while lon <= 2.*np.pi: # print lonarc, latarc, lon, lat, [lon*RadDeg, lat*RadDeg] lonProj.append(-180. + lon*RadDeg) latProj.append(lat*RadDeg) lonscirc.append(-180. + lon*RadDeg) lon = lon + lonarc Npts = Npts + 1 lonProjcirc.append(lonscirc) Nloncirc[ilat] = len(lonscirc) if Nloncirc[ilat] > Nlonmaxcirc: Nlonmaxcirc = Nloncirc[ilat] ilat = ilat + 1 lonProj.append(0.) latProj.append(90.) lonProjcirc.append(0.) latProjcirc.append(90.) Nloncirc[ilat] = 1 Npts = Npts + 1 dimx = Npts dimy = Npts Npoints = np.arange(Npts) else: print errormsg print ' ' + fname + ": projection '" + project + "' not ready !!" quit(-1) if ofile: # netCDF creation onewnc = NetCDFFile(ofile, 'w') # Dimensions newdim = onewnc.createDimension('x', dimx) newdim = onewnc.createDimension('y', dimy) newdim = onewnc.createDimension('Npts', Npts) if project == 'lonlat_dxdyFix': newdim = onewnc.createDimension('xcirc', Nlonmaxcirc) newdim = onewnc.createDimension('ycirc', Nlat) # Variables newvar = onewnc.createVariable('lon', 'f8', ('x')) basicvardef(newvar, 'longitude', 'Longitude', 'degrees_East') newvar[:] = lonProj newvar.setncattr('axis', 'X') newvar.setncattr('_CoordinateAxisType', 'Lon') newvar = onewnc.createVariable('lat', 'f8', ('y')) basicvardef(newvar, 'latitude', 'Latitude', 'degrees_North') newvar[:] = latProj newvar.setncattr('axis', 'Y') newvar.setncattr('_CoordinateAxisType', 'Lat') # Specific variables if project == 'lonlat_dxdyFix': newvar = onewnc.createVariable('Nloncirc', 'i', ('ycirc')) newvar[:] = Nloncirc basicvardef(newvar, 'Nloncirc', 'Number of longitudes by latitude circs', '-') newvar = onewnc.createVariable('loncirc', 'f8', ('ycirc','xcirc')) basicvardef(newvar, 'longitude', 'Longitude by latitude circs', 'degrees_East') for iL in range(Nlat): newvar[iL,0:Nloncirc[iL]] = np.asarray(lonProjcirc[iL]) newvar = onewnc.createVariable('latcirc', 'f8', ('ycirc')) basicvardef(newvar, 'latitude', 'Latitude', 'degrees_North') newvar[:] = latProjcirc if len(Npoints.shape) == 2: newvar = onewnc.createVariable('points', 'i8', ('y', 'x')) else: newvar = onewnc.createVariable('points', 'i8', ('Npts')) basicvardef(newvar, 'points', '# point of the projection', '-') newvar[:] = Npoints newvar.setncattr('coordinates', 'lon lat') # Global attributes onewnc.setncattr('script', fname) onewnc.setncattr('version', '1.0') onewnc.setncattr('author', 'L. Fita') newattr = set_attributek(onewnc, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') onewnc.setncattr('university', 'Pierre et Marie Curie') onewnc.setncattr('country', 'France') onewnc.setncattr('Projection','project') onewnc.sync() onewnc.close() print fname + "Successfull written of '" + ofile + "' !!'" if project == 'lonlat': return lonProj, latProj elif project == 'lonlat_dxdyFix': return lonProjcirc, latProj #lonlatProj(100000.,100000.,'lonlat',True) #quit() def Partialmap_Entiremap(values, filen, varn): """ Function to transform from a partial global map (e.g.: only land points) to an entire one Coincidence of points is done throughout a first guess from fractions of the total domain of search values= [lonmame],[latname],[fillVal],[resolution],[kind],[lonlatProjfile],[fracd],[fracs] [lonname]: name of the longitude variable [latname]: name of the latitude variable [fillVal]: value for '_FillValue', 'std' for the standard value Float = 1.e20 Character = '-' Integer = -99999 Float64 = 1.e20 Integer32 = -99999 [resolution]: resolution of the map [kind]: kind of target projection 'lonlat': standard lon/lat projection tacking number of points from [dx/dy]res at the Equator 'lonlat_dxdyFix': projection with a fixed grid distance 'Goode': Goode projection [lonlatProjfile]: file with the lon,lat of the desired projection. 'None' to be computed and written on fly [fracd]: Percentage of the fractions within perform the first guess search [fracs]: frequency of points at which syncronize file with calculations filen= name of the netCDF file varn= name of the variable """ import numpy.ma as ma import subprocess as sub fname = 'Partialmap_Entiremap' if values == 'h': print fname + '_____________________________________________________________' print Partialmap_Entiremap.__doc__ quit() arguments = '[lonmame],[latname],[fillVal],[resolution],[kind],' + \ '[lonlatProjfile],[fracd],[fracs]' check_arguments(fname, values, arguments, ',') ofile = 'EntireGlobalMap.nc' onc = NetCDFFile(filen, 'r') if not onc.variables.has_key(varn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varn + "' !!" quit(-1) lonname = values.split(',')[0] latname = values.split(',')[1] fval0 = values.split(',')[2] resolution = np.float(values.split(',')[3]) kind = values.split(',')[4] Projfile = values.split(',')[5] fracd = np.float(values.split(',')[6]) fracs = int(values.split(',')[7]) if Projfile == 'None': lonlatProjfile = None else: lonlatProjfile = Projfile if not onc.variables.has_key(lonname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "longitude '" + lonname + "' !!" quit(-1) if not onc.variables.has_key(latname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "latitude '" + latname + "' !!" quit(-1) olon = onc.variables[lonname] olat = onc.variables[latname] lonvs = olon[:] latvs = olat[:] Ninpts = len(lonvs) nlat = np.min(latvs) nlon = np.min(lonvs) minlat = np.min(np.abs(latvs)) minlon = np.min(np.abs(lonvs)) # mindiff = np.min([minlat, minlon])/1. mindiff = np.min([minlat, minlon])*10.e6 print ' ' + fname + ': Closest latitude to Equator:', minlat print ' ' + fname + ': Closest longitude to Greenwich Meridian:', minlon print ' ' + fname + ': Minium distance for point coincidence:', mindiff # Existing longitudes along/closest to the Equator eqlons = [] for i in range(len(latvs)): if latvs[i] == minlat: eqlons.append(lonvs[i]) # Existing latitudes along/closest to the Greenwich Meridian grlats = [] for i in range(len(lonvs)): if lonvs[i] == minlon: grlats.append(latvs[i]) sorteqlons = np.sort(eqlons) sortgrlats = np.sort(grlats) Neqlons = len(sorteqlons) Ngrlats = len(sortgrlats) if Neqlons > 1: diffeqlons = sorteqlons[1:Neqlons-1] - sorteqlons[0:Neqlons-2] mindeqlon = np.min(diffeqlons) print 'N sorteqlons:',Neqlons,'min deqlon:', mindeqlon else: mindeqlon = None if Ngrlats > 1: mindgrlat = np.min(diffgrlats) diffgrlats = sortgrlats[1:Ngrlats-1] - sortgrlats[0:Ngrlats-2] print 'N sortgrlats:',Ngrlats,'min dgrlat:', mindgrlat latmap = np.range(0,360.,360./mindeqlon) else: mindgrlat = None # Fixing in case it has not worked if mindeqlon is not None and mindgrlat is None: mindgrlat = mindeqlon if mindeqlon is None and mindgrlat is not None: mindeqlon = mindgrlat if mindeqlon is None and mindgrlat is None: print errormsg print fname + ': Not enough values along the Equator and Greenwich!!' quit(-1) if lonlatProjfile is None: lonlatProjfile = kind + '.nc' print warnmsg print ' ' + fname + ": no reference map !!" print " creation of '" + lonlatProjfile + "'" print ' creating it via: lonlatProj(', resolution, ',', resolution, \ ', ' + kind + ', True)' lonmap, latmap = lonlatProj(resolution, resolution, kind, True) sub.call(['mv','lonlatProj.nc',lonlatProjfile]) oproj = NetCDFFile(lonlatProjfile, 'r') if kind == 'Goode': olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] projlon = olon[:] projlat = olat[:] omapvals = oproj.variables['ptid'] Ntotvals = omapvals.shape[0] * omapvals.shape[1] Nmaplon = olon.shape[1] Nmaplat = olat.shape[0] elif kind == 'lonlat': olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] projlon = olon[:] projlat = olat[:] omapvals = oproj.variables['points'] Ntotvals = omapvals.shape[0] * omapvals.shape[1] Nmaplon = olon.shape[0] Nmaplat = olat.shape[0] elif kind == 'lonlat_dxdyFix': oprojlon = oproj.variables['loncirc'] oprojlat = oproj.variables['latcirc'] oNprojlon = oproj.variables['Nloncirc'] projlat = oprojlat[:] Nprojlon = oNprojlon[:] Ntotvals = len(oproj.dimensions['Npts']) olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] omapvals = oproj.variables['points'] # Ntotvals = len(lonmap) Nmaplon = Ntotvals Nmaplat = Ntotvals else: print errormsg print ' ' + fname + ": projection kind '" + kind + "' not ready!!" quit(-1) # Matrix map creation ## ovar = onc.variables[varn] vartype = type(ovar[0]) varlongname = ovar.long_name varunits = ovar.units fval = fillvalue_kind(type(ovar[0]), fval0) # Final map creation ## if not os.path.isfile(ofile): print ' ' + fname + "File '" + ofile + "' does not exist !!" print ' cration of output file' newnc = NetCDFFile(ofile, 'w') # dimensions newdim = newnc.createDimension('lon',Nmaplon) newdim = newnc.createDimension('lat',Nmaplat) newdim = newnc.createDimension('inpts', Ninpts) newdim = newnc.createDimension('pts', Ntotvals) # Variables if kind == 'Goode': newvar = newnc.createVariable('lon','f8',('lat', 'lon')) else: newvar = newnc.createVariable('lon','f8',('lon')) basicvardef(newvar, 'lon', 'Longitudes','degrees_East') newvar[:] = lonmap newvar.setncattr('axis', 'X') newvar.setncattr('_CoordinateAxisType', 'Lon') if kind == 'Goode': newvar = newnc.createVariable('lat','f8',('lat','lon')) else: newvar = newnc.createVariable('lat','f8',('lat')) basicvardef(newvar, 'lat', 'Latitudes','degrees_North') newvar[:] = latmap newvar.setncattr('axis', 'Y') newvar.setncattr('_CoordinateAxisType', 'Lat') newvarinpt = newnc.createVariable('locinpt','i',('inpts')) basicvardef(newvarinpt, 'locinpt', 'input point located: 0: no, 1: yes','-') newvarindiff = newnc.createVariable('locindiff','f4',('inpts')) basicvardef(newvarindiff, 'locindiff', 'distance between input point and its final location','degree') # map variable Lmapvalsshape = len(omapvals.shape) if Lmapvalsshape == 1: newvar = newnc.createVariable(varn, nctype(vartype), ('pts'), fill_value=fval) else: newvar = newnc.createVariable(varn, nctype(vartype), ('lat','lon'), fill_value=fval) basicvardef(newvar, varn, varlongname, varunits) newvar.setncattr('coordinates', 'lon lat') if Lmapvalsshape == 1: newvarin = newnc.createVariable('inpts', 'i4', ('pts')) else: newvarin = newnc.createVariable('inpts', 'i4', ('lat','lon')) basicvardef(newvarin, 'inpts', 'Equivalent point from the input source', '-') newvar.setncattr('coordinates', 'lon lat') else: print ' ' + fname + "File '" + ofile + "' exists !!" print ' reading values from file' newnc = NetCDFFile(ofile, 'a') newvar = newnc.variables[varn] newvarin = newnc.variables['inpts'] newvarinpt = newnc.variables['locinpt'] newvarindiff = newnc.variables['locindiff'] amsk = np.arange(3) amsk = ma.masked_equal(amsk, 0) # fraclon = projlon[::Nmaplon*0.1] # fraclat = projlat[::Nmaplat*0.1] # print 'fraclon________________', fraclon.shape # print fraclon # print 'fraclat________________', fraclat.shape # print fraclat # Reducing the searching points newvarinvals = newvarinpt[:] maskpt = np.where(newvarinvals.mask == True, False, True) points = np.arange(Ninpts) mapoints = ma.array(points, mask=maskpt) ptsf = mapoints.compressed() Nptsf = len(ptsf) print Ninpts,'Npoints to find:', len(ptsf), ptsf[0:10], newvarindiff[ptsf[0:10]] # Error at 150024, 150025, 151709, 153421 print ' ' + fname + ': from:', Ninpts,'re-locating:',Nptsf,'points...' if kind == 'Goode': empty = CoarselonlatFindAll(newnc,projlon,projlat,lonvs[ptsf],latvs[ptsf], \ fracd,fracs,False) # for iv0 in range(Nptsf): # iv = mapoints[ptsf[iv0]] # if newvarinpt[iv] == 0: ## difflonlat = np.sqrt((projlon-lonvs[iv])**2. + (projlat-latvs[iv])**2.) ## mindiffLl = np.min(difflonlat) ## ilatlon = index_mat(difflonlat, mindiffLl) # ilatlon, mindiffLl = CoarselonlatFind(projlon,projlat,lonvs[iv],latvs[iv],fracd) ## if ilatlon[0] != ilatlon2[0] or ilatlon[1] != ilatlon2[1]: ## print ilatlon, '<', ilatlon2 ## print 'diff:', mindiffLl, '<', mindiffLl2 ## quit(-1) ## if mindiffLl != 0. or type(newvar[ilatlon[0],ilatlon[1]]) != type(amsk): ## print errormsg ## if mindiffLl !=0.: ## print ' ' + main + ': no zero', newvarindiff[iv], 'distance!!' ## else: ## print ' ' + main+': point',projlon[ilatlon[0],ilatlon[1]], \ ## ',', projlat[ilatlon[0],ilatlon[1]], 'already filled!!' ## print ' value:',newvar[ilatlon[0],ilatlon[1]],'distance:',\ ## newvarindiff[iv] ## ### print ' iv:', iv, 'lon:', lonvs[iv],'lat:',latvs[iv] ### print ' mindiffLl:',mindiffLl,'ilatlon:',ilatlon ### quit(-1) ### print ' Lluis; ' + fname + ' iv:', iv, 'lon:', lonvs[iv],'lat:',latvs[iv] ### print ' Lluis; ' + fname + ' mindiffLl:',mindiffLl,'ilatlon:',ilatlon ### print ' Lluis; ' + fname + ' A found lon:',projlon[ilatlon[0], ilatlon[1]], 'lat:', projlat[ilatlon[0], ilatlon[1]] ### quit() ## else: # if mindiffLl == 0. and type(newvar[ilatlon[0],ilatlon[1]]) == type(amsk): # percendone(iv,Ninpts,0.5,'done:') # if mindiffLl > mindiff: # print errormsg # print ' ' + fname + ': for point #', iv,'lon,lat in ' + \ # 'incomplet map:', lonvs[iv], ',', latvs[iv], 'there is ' + \ # 'not a set of lon,lat in the completed map closer than: ', \ # mindiff, '!!' # print ' minimum difference:', mindiffLl # quit(-1) # if ilatlon[0] >= 0 and ilatlon[1] >= 0: # newvar[ilatlon[0],ilatlon[1]] = ovar[iv] # newvarin[ilatlon[0],ilatlon[1]] = iv # newvarinpt[iv] = 1 # newvarindiff[iv] = mindiffLl ## print 'Lluis iv:', newvarin[ilatlon[0],ilatlon[1]], \ ## 'localized:', newvarinpt[iv], 'values:', \ ## newvar[ilatlon[0],ilatlon[1]], 'invalues:', ovar[iv], \ ## 'mindist:', newvarindiff[iv], 'point:',ilatlon # else: # print errormsg # print ' ' + fname + ': point iv:', iv, 'at', lonvs[iv], ',',\ # latvs[iv],' not relocated !!' # print ' mindiffl:', mindiffLl, 'ilon:', ilatlon[1], \ # 'ilatlon:', ilatlon[1] # quit(-1) # if np.mod(iv,1000) == 0: # newnc.sync() ## print ' ' + fname + 'values localized:', newvar[:].compressed() ## if iv > 10: ## newnc.sync() ## newnc.close() ## quit(-1) elif kind == 'lonlat': for iv in range(Ntotvals): difflat = np.abs(projlat - latvs[iv]) mindiffL = np.min(difflat) ilat = index_mat(difflat, mindiffL) difflon = np.abs(projlon - lonvs[iv]) mindiffl = difflon.min() ilon = index_mat(difflon, mindiffl) percendone(iv,Ntotvals,0.5,'done:') if mindiffl > mindiff or mindiffL > mindiff: print errormsg print ' ' + fname + ': for point #', iv,'lon,lat in incomplet map:', \ lonvs[iv], ',', latvs[iv], 'there is not a set of lon,lat in the ' + \ 'completed map closer than: ', mindiff, '!!' print ' minimum difflon:', np.min(difflon), 'difflat:', np.min(difflat) quit() if ilon >= 0 and ilat >= 0: newvar[ilat,ilon] = ovar[iv] newnc.sync() else: print errormsg print ' ' + fname + ': point iv:',iv,'at',lonvs[iv],',',latvs[iv],' not relocated !!' print ' mindiffl:',mindiffl,'mindiffL:',mindiffL,'ilon:',ilon,'ilat:',ilat quit(-1) newnc.sync() elif kind == 'lonlat_dxdyFix': for iv in range(Ntotvals): # for iv in range(15): # print 'Lluis:',iv,'lon lat:',lonvs[iv],',',latvs[iv] difflat = np.abs(projlat - latvs[iv]) mindiffL = np.min(difflat) ilat = index_mat(difflat, mindiffL) # print ' Lluis mindiffL:',mindiffL,'<> ilat:',ilat,'lat_ilat:',projlat[ilat],'Nprojlon:',Nprojlon[ilat],'shape oporjlon:',oprojlon[ilat,:].shape, type(oprojlon[ilat,:]) loncirc = np.zeros((Nprojlon[ilat]), dtype=np.float) loncirc[:] = np.asarray(oprojlon[ilat,0:int(Nprojlon[ilat])].flatten()) difflon = np.abs(loncirc - lonvs[iv]) mindiffl = difflon.min() ilon = index_mat(difflon, mindiffl) # print ' difflon:',type(difflon),'shape difflon',difflon.shape,'shape loncirc:', loncirc.shape # print ' Lluis mindiffl:',mindiffl,'<> ilon:',ilon,'oprojlon:', loncirc[ilon] percendone(iv,Ntotvals,0.5,'done:') if mindiffl > mindiff or mindiffL > mindiff: print errormsg print ' ' + fname + ': for point #', iv,'lon,lat in incomplet map:', \ lonvs[iv], ',', latvs[iv], 'there is not a set of lon,lat in the ' + \ 'completed map closer than: ', mindiff, '!!' print ' minimum difflon:', np.min(difflon), 'difflat:', np.min(difflat) quit() if ilon >= 0 and ilat >= 0: if Lmapvalsshape ==1: newvar[iv] = ovar[iv] newnc.sync() else: print errormsg print ' ' + fname + ': point iv:',iv,'at',lonvs[iv],',',latvs[iv],' not relocated !!' print ' mindiffl:',mindiffl,'mindiffL:',mindiffL,'ilon:',ilon,'ilat:',ilat quit(-1) newnc.sync() # Global attributes ## newnc.setncattr('script', fname) newnc.setncattr('version', '1.0') newnc.setncattr('author', 'L. Fita') newattr = set_attributek(newnc, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') newnc.setncattr('university', 'Pierre et Marie Curie') newnc.setncattr('country', 'France') for attrs in onc.ncattrs(): attrv = onc.getncattr(attrs) attr = set_attribute(newnc, attrs, attrv) newnc.sync() onc.close() newnc.close() print fname + ": Successfull written of file: '" + ofile + "' !!" return def Partialmap_EntiremapFor(values, filen, varn): """ Function to transform from a partial global map (e.g.: only land points) to an entire one usinf Fortran code Coincidence of points is done throughout a first guess from fractions of the total domain of search Using fortran codes: module_ForInterpolate.F90, module_generic.F90 foudre: f2py -m module_ForInterpolate --f90exec=/usr/bin/gfortran-4.7 -c module_generic.F90 module_ForInterpolate.F90 >& run_f2py.log ciclad: f2py --f90flags="-fPIC" -L/opt/canopy-1.3.0/Canopy_64bit/System/lib/ -L/usr/lib64/ -L/opt/canopy-1.3.0/Canopy_64bit/System/lib/ -m module_ForInterpolate -c module_generic.F90 module_ForInterpolate.F90 values= [lonmame],[latname],[fillVal],[resolution],[kind],[lonlatProjfile],[ipoint],[fracd],[fracs],[mindiff] [lonname]: name of the longitude variable [latname]: name of the latitude variable [fillVal]: value for '_FillValue', 'std' for the standard value Float = 1.e20 Character = '-' Integer = -99999 Float64 = 1.e20 Integer32 = -99999 [resolution]: resolution of the map [kind]: kind of target projection 'lonlat': standard lon/lat projection tacking number of points from [dx/dy]res at the Equator 'lonlatfrac': standard lon/lat projection tacking number of points from [dx/dy]res at the Equator with mnultiple values at the target interpolation grid [kind]@[Ntypes] [Ntypes]: Number of types at each grid-point 'lonlat_dxdyFix': projection with a fixed grid distance 'Goode': Goode projection [lonlatProjfile]: file with the lon,lat of the desired projection. 'None' to be computed and written on fly [ipoint]: initial point to use for the interpolation (0, the first) [fracd]: Percentage of the fractions within perform the first guess search [fracs]: Number of grid points to perform the syncronization with the file and the computed values [mindiff]: Authorized minium distance between input and final lon,lat point filen= name of the netCDF file varn= name of the variable """ import module_ForInt as fin import numpy.ma as ma import subprocess as sub fname = 'Partialmap_Entiremap' if values == 'h': print fname + '_____________________________________________________________' print Partialmap_Entiremap.__doc__ quit() arguments = '[lonmame],[latname],[fillVal],[resolution],[kind],' + \ '[lonlatProjfile],[ipoint],[fracd],[fracs],[mindiff]' gen.check_arguments(fname, values, arguments, ',') ofile = 'EntireGlobalMap.nc' onc = NetCDFFile(filen, 'r') if not onc.variables.has_key(varn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varn + "' !!" quit(-1) lonname = values.split(',')[0] latname = values.split(',')[1] fval0 = values.split(',')[2] resolution = np.float(values.split(',')[3]) kind0 = values.split(',')[4] Projfile = values.split(',')[5] ipoint = int(values.split(',')[6]) fracd = np.float64(values.split(',')[7]) fracs = int(values.split(',')[8]) mindiff = np.float64(values.split(',')[9]) if Projfile == 'None': lonlatProjfile = None else: lonlatProjfile = Projfile if kind0.find('@') != -1: kind = kind0.split('@')[0] Ntypes = np.int(kind0.split('@')[1]) else: kind = kind0 Lkind = len(kind) if kind[Lkind-4:Lkind] == 'frac' and kind0.find('@') == -1: print errormsg print ' ' + fname + ": there is a fractional kind of interpolation '" + \ kind + "' but no number of types has been privided!!" print " when [kind] is provided, should be '[kind]@[Ntypes]'" quit() if not onc.variables.has_key(lonname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "longitude '" + lonname + "' !!" quit(-1) if not onc.variables.has_key(latname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "latitude '" + latname + "' !!" quit(-1) olon = onc.variables[lonname] olat = onc.variables[latname] lonvs = olon[:] latvs = olat[:] Ninpts = len(lonvs) nlat = np.min(latvs) nlon = np.min(lonvs) minlat = np.min(np.abs(latvs)) minlon = np.min(np.abs(lonvs)) ivtest = 2198 # mindiff = np.min([minlat, minlon])/1. mindiffin = np.sqrt(minlat*minlat + minlon*minlon) print ' ' + fname + ': Longitudes minimum:', np.min(lonvs), 'max:', np.max(lonvs) print ' ' + fname + ': Latitudes minimum:', np.min(latvs), 'max:', np.max(latvs) # print 'Lluis lon/lat ivtest:',lonvs[ivtest],',',latvs[ivtest] print ' ' + fname + ': Closest latitude to Equator:', minlat print ' ' + fname + ': Closest longitude to Greenwich Meridian:', minlon print ' ' + fname + ': Minium distance for point coincidence:', mindiff # Existing longitudes along/closest to the Equator eqlons = [] for i in range(len(latvs)): if latvs[i] == minlat: eqlons.append(lonvs[i]) # Existing latitudes along/closest to the Greenwich Meridian grlats = [] for i in range(len(lonvs)): if lonvs[i] == minlon: grlats.append(latvs[i]) sorteqlons = np.sort(eqlons) sortgrlats = np.sort(grlats) Neqlons = len(sorteqlons) Ngrlats = len(sortgrlats) if Neqlons > 1: diffeqlons = sorteqlons[1:Neqlons-1] - sorteqlons[0:Neqlons-2] mindeqlon = np.min(diffeqlons) print 'N sorteqlons:',Neqlons,'min deqlon:', mindeqlon else: mindeqlon = None if Ngrlats > 1: mindgrlat = np.min(diffgrlats) diffgrlats = sortgrlats[1:Ngrlats-1] - sortgrlats[0:Ngrlats-2] print 'N sortgrlats:',Ngrlats,'min dgrlat:', mindgrlat latmap = np.range(0,360.,360./mindeqlon) else: mindgrlat = None # Fixing in case it has not worked if mindeqlon is not None and mindgrlat is None: mindgrlat = mindeqlon if mindeqlon is None and mindgrlat is not None: mindeqlon = mindgrlat if mindeqlon is None and mindgrlat is None: print errormsg print fname + ': Not enough values along the Equator and Greenwich!!' quit(-1) if lonlatProjfile is None or not os.path.isfile(lonlatProjfile): if lonlatProjfile is None: lonlatProjfile = kind + '.nc' print warnmsg print ' ' + fname + ": no reference map !!" print " creation of '" + lonlatProjfile + "'" print ' creating it via: lonlatProj(', resolution, ',', resolution, \ ', ' + kind + ', True)' if kind[Lkind-4:Lkind] == 'frac': lonmap, latmap = lonlatProj(resolution, resolution, kind[0:Lkind-4], True) else: lonmap, latmap = lonlatProj(resolution, resolution, kind, True) sub.call(['mv','lonlatProj.nc', lonlatProjfile]) oproj = NetCDFFile(lonlatProjfile, 'r') if kind == 'Goode': olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] projlon = olon[:].astype('float64') projlat = olat[:].astype('float64') omapvals = oproj.variables['ptid'] Ntotvals = omapvals.shape[0] * omapvals.shape[1] Nmaplon = olon.shape[1] Nmaplat = olat.shape[0] elif kind == 'lonlat': olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] projlon = olon[:] projlat = olat[:] omapvals = oproj.variables['points'] Ntotvals = omapvals.shape[0] * omapvals.shape[1] Nmaplon = olon.shape[0] Nmaplat = olat.shape[0] elif kind == 'lonlatfrac': olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] projlon = olon[:] projlat = olat[:] omapvals = oproj.variables['points'] Ntotvals = omapvals.shape[0] * omapvals.shape[1] Nmaplon = olon.shape[0] Nmaplat = olat.shape[0] elif kind == 'lonlat_dxdyFix': oprojlon = oproj.variables['loncirc'] oprojlat = oproj.variables['latcirc'] oNprojlon = oproj.variables['Nloncirc'] projlat = oprojlat[:] Nprojlon = oNprojlon[:] Ntotvals = len(oproj.dimensions['Npts']) olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] omapvals = oproj.variables['points'] # Ntotvals = len(lonmap) Nmaplon = Ntotvals Nmaplat = Ntotvals else: print errormsg print ' ' + fname + ": projection kind '" + kind + "' not ready!!" quit(-1) sortedlonmap = np.sort(lonmap) sortedlatmap = np.sort(latmap) Nsortlonm = len(sortedlonmap) Nsortlatm = len(sortedlatmap) diffsortlon = sortedlonmap[1:Nsortlonm-1]-sortedlonmap[0:Nsortlonm-2] diffsortlat = sortedlatmap[1:Nsortlatm-1]-sortedlatmap[0:Nsortlatm-2] mindifflonmap = np.min(diffsortlon) maxdifflonmap = np.max(diffsortlon) mindifflatmap = np.min(diffsortlat) maxdifflatmap = np.max(diffsortlat) mindiffmap = np.sqrt(mindifflonmap*mindifflonmap+mindifflatmap*mindifflatmap) maxdiffmap = np.sqrt(maxdifflonmap*maxdifflonmap+maxdifflatmap*maxdifflatmap) print ' ' + fname + ': minimum distance in target projection:', mindiffmap, \ 'maximum:', maxdiffmap # Matrix map creation ## ovar = onc.variables[varn] vartype = type(ovar[0]) varlongname = ovar.long_name varunits = ovar.units print 'Lluis ovar[ivtest]:',ovar[ivtest] quit() fval = gen.fillvalue_kind(type(ovar[0]), fval0) # Final map creation ## if not os.path.isfile(ofile): print ' ' + fname + "File '" + ofile + "' does not exist !!" print ' cration of output file' newnc = NetCDFFile(ofile, 'w') # dimensions newdim = newnc.createDimension('lon',Nmaplon) newdim = newnc.createDimension('lat',Nmaplat) newdim = newnc.createDimension('inpts', Ninpts) newdim = newnc.createDimension('pts', Ntotvals) # Variables if kind == 'Goode': newvar = newnc.createVariable('lon','f8',('lat', 'lon')) else: newvar = newnc.createVariable('lon','f8',('lon')) basicvardef(newvar, 'longitude', 'Longitude','degrees_East') newvar[:] = lonmap newvar.setncattr('axis', 'X') newvar.setncattr('_CoordinateAxisType', 'Lon') if kind == 'Goode': newvar = newnc.createVariable('lat','f8',('lat','lon')) else: newvar = newnc.createVariable('lat','f8',('lat')) basicvardef(newvar, 'latitude', 'Latitude','degrees_North') newvar[:] = latmap newvar.setncattr('axis', 'Y') newvar.setncattr('_CoordinateAxisType', 'Lat') newvarinpt = newnc.createVariable('locinpt','i',('inpts')) basicvardef(newvarinpt, 'locinpt', 'input point located: 0: no, 1: yes','-') newvarindiff = newnc.createVariable('locindiff','f4',('inpts')) basicvardef(newvarindiff, 'locindiff', 'distance between input point and ' + \ 'its final location','degree') set_attributek(newvarindiff,'authorized_minimum_distance',mindiff,'R') # map variable Lmapvalsshape = len(omapvals.shape) if Lmapvalsshape == 1: newvar = newnc.createVariable(varn, nctype(vartype), ('pts'), \ fill_value=fval) else: if kind[Lkind-4:Lkind] == 'frac': newdim = newnc.createDimension('Ntypes', Ntypes) newvar = newnc.createVariable(varn, 'f4', ('Ntypes','lat','lon'), \ fill_value=gen.fillvalue_kind(type(1.), fval0)) else: newvar = newnc.createVariable(varn, nctype(vartype), ('lat','lon'), \ fill_value=fval) basicvardef(newvar, varn, varlongname, varunits) newvar.setncattr('coordinates', 'lon lat') if Lmapvalsshape == 1: newvarin = newnc.createVariable('inpts', 'i4', ('pts')) else: if kind[Lkind-4:Lkind] == 'frac': Nmaxcoinpt = np.int((maxdiffmap/mindiffin)*(maxdiffmap/mindiffin)) print ' ' + fname + ': maximum theoretical number of coincident ' + \ 'grid points in the same target gid point:', Nmaxcoinpt newdim = newnc.createDimension('Ncoinpt', Nmaxcoinpt) newvarin = newnc.createVariable('inpts', 'i4', ('Ncoinpt','lat', \ 'lon')) basicvardef(newvarin, 'inpts', 'Equivalent points from the input ' + 'source', '-') else: newvarin = newnc.createVariable('inpts', 'i4', ('lat','lon')) basicvardef(newvarin, 'inpts', 'Equivalent point from the input ' + \ 'source', '-') newvar.setncattr('coordinates', 'lon lat') newvarin.setncattr('coordinates', 'lon lat') else: print ' ' + fname + "File '" + ofile + "' exists !!" print ' reading values from file' newnc = NetCDFFile(ofile, 'a') newvar = newnc.variables[varn] newvarin = newnc.variables['inpts'] newvarinpt = newnc.variables['locinpt'] newvarindiff = newnc.variables['locindiff'] amsk = np.arange(3) amsk = ma.masked_equal(amsk, 0) # fraclon = projlon[::Nmaplon*0.1] # fraclat = projlat[::Nmaplat*0.1] # print 'fraclon________________', fraclon.shape # print fraclon # print 'fraclat________________', fraclat.shape # print fraclat # Reducing the searching points Nsearchpt = 999999 Ninpts=np.min([Nsearchpt, Ninpts-ipoint]) newvarinvals = newvarinpt[ipoint:ipoint+Ninpts] maskpt = np.where(newvarinvals.mask == True, False, True) points = np.arange(ipoint,ipoint+Ninpts,1) mapoints = ma.array(points, mask=maskpt) ptsf = mapoints.compressed() Nptsf = len(ptsf) print Ninpts,'Npoints to find:', len(ptsf), ptsf[0:10], newvarindiff[ptsf[0:10]] ipoint = ptsf[0] Ninpts=np.min([Nsearchpt, Ninpts-ipoint]) # Error at 150024, 150025, 151709, 153421 print ' ' + fname + ': from:', Ninpts,'re-locating:',Nptsf,'points...' if kind == 'Goode': # newvar,newvarin,newvarinpt,newvarindiff = \ # fin.module_forinterpolate.coarseinterpolate(projlon, projlat, lonvs, \ # latvs, percen, mindiff, ivar, dimx, dimy, ninpts) for ir in range(ipoint,Ninpts,fracs): iri = ir ire = ir + fracs + 1 print iri,',',ire # percendone(iri,Ninpts,0.5,'done:') pts = np.arange(iri,ire,1, dtype='int32') lonvss = lonvs[iri:ire].astype('float64') latvss = latvs[iri:ire].astype('float64') ovars = ovar[iri:ire].astype('float64') newvar,newvarin,newvarinpt[pts],newvarindiff[pts] = \ fin.module_forinterpolate.coarseinterpolate(projlon, projlat, \ lonvss, latvss, np.float64(fracd), np.float64(mindiff), ovars) # Slow way # newvar,newvarin,newvarinpt[pts],newvarindiff[pts] = \ # fin.module_forinterpolate.interpolate(projlon, projlat, \ # lonvss, latvss, np.float64(mindiff), ovars) newnc.sync() elif kind == 'lonlat': for ir in range(ipoint,ipoint+Ninpts,fracs): iri = ir ire = ir + fracs print iri,',',ire # percendone(iri,Ninpts,0.5,'done:') lonvss = lonvs[iri:ire+1].astype('float64') latvss = latvs[iri:ire+1].astype('float64') inptss = newvarinpt[iri:ire+1].astype('int32') idiff, ilonlatv = fin.module_forinterpolate.interpolate1dll(projlon, \ projlat, lonvss, latvss, np.float64(mindiff), inptss) for i in range(np.min([len(idiff),fracs])): # print 'Lluis:',ilonlatv[i,1],',',ilonlatv[i,0] newvar[ilonlatv[i,1],ilonlatv[i,0]] = ovar[iri + i] newvarin[ilonlatv[i,1],ilonlatv[i,0]] = iri + i newvarinpt[iri+i] = 1 newvarindiff[iri+i] = idiff[i] newnc.sync() # newnc.close() # quit() elif kind == 'lonlatfrac': for ir in range(ipoint,ipoint+Ninpts,fracs): iri = ir ire = ir + fracs print iri,',',ire # percendone(iri,Ninpts,0.5,'done:') lonvss = lonvs[iri:ire+1].astype('float64') latvss = latvs[iri:ire+1].astype('float64') inptss = newvarinpt[iri:ire+1].astype('int32') idiff, ilonlatv = fin.module_forinterpolate.interpolate1dll(projlon, \ projlat, lonvss, latvss, np.float64(mindiff), inptss) # Fortran routine provides ranges from 1,dimx! ilonlatv = ilonlatv - 1 for i in range(np.min([len(idiff),fracs])): if type(newvar[ovar[iri+i]-1,ilonlatv[i,1],ilonlatv[i,0]]) == \ type(ma.array(1)) and newvar[ovar[iri+i]-1,ilonlatv[i,1], \ ilonlatv[i,0]].mask: newvar[ovar[iri+i]-1,ilonlatv[i,1],ilonlatv[i,0]].mask = False newvar[ovar[iri+i]-1,ilonlatv[i,1],ilonlatv[i,0]] = 1. newvarin[0,ilonlatv[i,1],ilonlatv[i,0]] = iri + i else: Npt = np.sum(newvarin[:,ilonlatv[i,1],ilonlatv[i,0]] > 0) print ilonlatv[i,1],ilonlatv[i,0],'Npt:',Npt print newvarin[:,ilonlatv[i,1],ilonlatv[i,0]] newvar[ovar[iri+i]-1,ilonlatv[i,1],ilonlatv[i,0]] = \ newvar[ovar[iri+i]-1,ilonlatv[i,1],ilonlatv[i,0]] + 1. newvarin[Npt+1,ilonlatv[i,1],ilonlatv[i,0]] = iri + i newvarinpt[iri+i] = 1 newvarindiff[iri+i] = idiff[i] newnc.sync() if ire > ivtest: print 'iv:',ivtest,'lon lat:',lonvs[ivtest],',',latvs[ivtest] print 'ilonlat:',ilonlatv[ivtest,:],'lonlat proj:', projlon[ilonlatv[ivtest,0]],',', \ projlat[ilonlatv[ivtest,1]],'diff:',newvarindiff[ivtest] newnc.close() quit() # newnc.close() # quit() elif kind == 'lonlat_dxdyFix': for iv in range(Ntotvals): # for iv in range(15): # print 'Lluis:',iv,'lon lat:',lonvs[iv],',',latvs[iv] difflat = np.abs(projlat - latvs[iv]) mindiffL = np.min(difflat) ilat = index_mat(difflat, mindiffL) # print ' Lluis mindiffL:',mindiffL,'<> ilat:',ilat,'lat_ilat:',projlat[ilat],'Nprojlon:',Nprojlon[ilat],'shape oporjlon:',oprojlon[ilat,:].shape, type(oprojlon[ilat,:]) loncirc = np.zeros((Nprojlon[ilat]), dtype=np.float) loncirc[:] = np.asarray(oprojlon[ilat,0:int(Nprojlon[ilat])].flatten()) difflon = np.abs(loncirc - lonvs[iv]) mindiffl = difflon.min() ilon = index_mat(difflon, mindiffl) # print ' difflon:',type(difflon),'shape difflon',difflon.shape,'shape loncirc:', loncirc.shape # print ' Lluis mindiffl:',mindiffl,'<> ilon:',ilon,'oprojlon:', loncirc[ilon] percendone(iv,Ntotvals,0.5,'done:') if mindiffl > mindiff or mindiffL > mindiff: print errormsg print ' ' + fname + ': for point #', iv,'lon,lat in incomplet map:', \ lonvs[iv], ',', latvs[iv], 'there is not a set of lon,lat in the ' + \ 'completed map closer than: ', mindiff, '!!' print ' minimum difflon:', np.min(difflon), 'difflat:', np.min(difflat) quit() if ilon >= 0 and ilat >= 0: if Lmapvalsshape ==1: newvar[iv] = ovar[iv] newnc.sync() else: print errormsg print ' ' + fname + ': point iv:',iv,'at',lonvs[iv],',',latvs[iv],' not relocated !!' print ' mindiffl:',mindiffl,'mindiffL:',mindiffL,'ilon:',ilon,'ilat:',ilat quit(-1) newnc.sync() # Global attributes ## newnc.setncattr('script', fname) newnc.setncattr('version', '1.0') newnc.setncattr('author', 'L. Fita') newattr = set_attributek(newnc, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') newnc.setncattr('university', 'Pierre et Marie Curie') newnc.setncattr('country', 'France') for attrs in onc.ncattrs(): attrv = onc.getncattr(attrs) attr = set_attribute(newnc, attrs, attrv) newnc.sync() onc.close() newnc.close() print fname + ": Successfull written of file: '" + ofile + "' !!" return def Partialmap_EntiremapForExact(values, filen, varn): """ Function to transform from a partial global map (e.g.: only land points) to an entire one using Fortran code with exact location Coincidence of points is done throughout a first guess from fractions of the total domain of search Using fortran codes: module_ForInterpolate.F90, module_generic.F90 foudre: f2py -m module_ForInterpolate --f90exec=/usr/bin/gfortran-4.7 -c module_generic.F90 module_ForInterpolate.F90 >& run_f2py.log ciclad: f2py --f90flags="-fPIC" -L/opt/canopy-1.3.0/Canopy_64bit/System/lib/ -L/usr/lib64/ -L/opt/canopy-1.3.0/Canopy_64bit/System/lib/ -m module_ForInterpolate -c module_generic.F90 module_ForInterpolate.F90 values= [lonmame],[latname],[fillVal],[resolution],[kind],[lonlatProjfile],[ipoint],[fracd],[fracs],[mindiff] [lonname]: name of the longitude variable [latname]: name of the latitude variable [fillVal]: value for '_FillValue', 'std' for the standard value Float = 1.e20 Character = '-' Integer = -99999 Float64 = 1.e20 Integer32 = -99999 [resolution]: resolution of the map [kind]: kind of target projection 'lonlat': standard lon/lat projection tacking number of points from [dx/dy]res at the Equator 'lonlat_dxdyFix': projection with a fixed grid distance 'Goode': Goode projection [lonlatProjfile]: file with the lon,lat of the desired projection. 'None' to be computed and written on fly [ipoint]: initial point to use for the interpolation (0, the first) [fracd]: Percentage of the fractions within perform the first guess search [fracs]: Number of grid points to perform the syncronization with the file and the computed values [mindiff]: Authorized minium distance between input and final lon,lat point filen= name of the netCDF file varn= name of the variable """ import module_ForInterpolate as fin import numpy.ma as ma import subprocess as sub fname = 'Partialmap_EntiremapFortran' if values == 'h': print fname + '_____________________________________________________________' print Partialmap_Entiremap.__doc__ quit() arguments = '[lonmame],[latname],[fillVal],[resolution],[kind],' + \ '[lonlatProjfile],[ipoint],[fracd],[fracs],[mindiff]' gen.check_arguments(fname, values, arguments, ',') ofile = 'EntireGlobalMap.nc' onc = NetCDFFile(filen, 'r') if not onc.variables.has_key(varn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varn + "' !!" quit(-1) lonname = values.split(',')[0] latname = values.split(',')[1] fval0 = values.split(',')[2] resolution = np.float(values.split(',')[3]) kind = values.split(',')[4] Projfile = values.split(',')[5] ipoint = int(values.split(',')[6]) fracd = np.float64(values.split(',')[7]) fracs = int(values.split(',')[8]) mindiff = np.float64(values.split(',')[9]) if Projfile == 'None': lonlatProjfile = None else: lonlatProjfile = Projfile if not onc.variables.has_key(lonname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "longitude '" + lonname + "' !!" quit(-1) if not onc.variables.has_key(latname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "latitude '" + latname + "' !!" quit(-1) olon = onc.variables[lonname] olat = onc.variables[latname] lonvs = olon[:] latvs = olat[:] Ninpts = len(lonvs) nlat = np.min(latvs) nlon = np.min(lonvs) minlat = np.min(np.abs(latvs)) minlon = np.min(np.abs(lonvs)) print ' ' + fname + ': Closest latitude to Equator:', minlat print ' ' + fname + ': Closest longitude to Greenwich Meridian:', minlon print ' ' + fname + ': Minium distance for point coincidence:', mindiff # Existing longitudes along/closest to the Equator eqlons = [] for i in range(len(latvs)): if latvs[i] == minlat: eqlons.append(lonvs[i]) # Existing latitudes along/closest to the Greenwich Meridian grlats = [] for i in range(len(lonvs)): if lonvs[i] == minlon: grlats.append(latvs[i]) sorteqlons = np.sort(eqlons) sortgrlats = np.sort(grlats) Neqlons = len(sorteqlons) Ngrlats = len(sortgrlats) if Neqlons > 1: diffeqlons = sorteqlons[1:Neqlons-1] - sorteqlons[0:Neqlons-2] mindeqlon = np.min(diffeqlons) print 'N sorteqlons:',Neqlons,'min deqlon:', mindeqlon else: mindeqlon = None if Ngrlats > 1: mindgrlat = np.min(diffgrlats) diffgrlats = sortgrlats[1:Ngrlats-1] - sortgrlats[0:Ngrlats-2] print 'N sortgrlats:',Ngrlats,'min dgrlat:', mindgrlat latmap = np.range(0,360.,360./mindeqlon) else: mindgrlat = None # Fixing in case it has not worked if mindeqlon is not None and mindgrlat is None: mindgrlat = mindeqlon if mindeqlon is None and mindgrlat is not None: mindeqlon = mindgrlat if mindeqlon is None and mindgrlat is None: print errormsg print fname + ': Not enough values along the Equator and Greenwich!!' quit(-1) if lonlatProjfile is None: lonlatProjfile = kind + '.nc' print warnmsg print ' ' + fname + ": no reference map !!" print " creation of '" + lonlatProjfile + "'" print ' creating it via: lonlatProj(', resolution, ',', resolution, \ ', ' + kind + ', True)' lonmap, latmap = lonlatProj(resolution, resolution, kind, True) sub.call(['mv','lonlatProj.nc',lonlatProjfile]) oproj = NetCDFFile(lonlatProjfile, 'r') if kind == 'Goode': olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] projlon = olon[:].astype('float64') projlat = olat[:].astype('float64') omapvals = oproj.variables['ptid'] Ntotvals = omapvals.shape[0] * omapvals.shape[1] Nmaplon = olon.shape[1] Nmaplat = olat.shape[0] elif kind == 'lonlat': olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] projlon = olon[:] projlat = olat[:] omapvals = oproj.variables['points'] Ntotvals = omapvals.shape[0] * omapvals.shape[1] Nmaplon = olon.shape[0] Nmaplat = olat.shape[0] elif kind == 'lonlat_dxdyFix': oprojlon = oproj.variables['loncirc'] oprojlat = oproj.variables['latcirc'] oNprojlon = oproj.variables['Nloncirc'] projlat = oprojlat[:] Nprojlon = oNprojlon[:] Ntotvals = len(oproj.dimensions['Npts']) olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] omapvals = oproj.variables['points'] # Ntotvals = len(lonmap) Nmaplon = Ntotvals Nmaplat = Ntotvals else: print errormsg print ' ' + fname + ": projection kind '" + kind + "' not ready!!" quit(-1) # Matrix map creation ## ovar = onc.variables[varn] vartype = type(ovar[0]) varlongname = ovar.long_name varunits = ovar.units fval = gen.fillvalue_kind(type(ovar[0]), fval0) # Final map creation ## if not os.path.isfile(ofile): print ' ' + fname + "File '" + ofile + "' does not exist !!" print ' cration of output file' newnc = NetCDFFile(ofile, 'w') # dimensions newdim = newnc.createDimension('lon',Nmaplon) newdim = newnc.createDimension('lat',Nmaplat) newdim = newnc.createDimension('inpts', Ninpts) newdim = newnc.createDimension('pts', Ntotvals) # Variables if kind == 'Goode': newvar = newnc.createVariable('lon','f8',('lat', 'lon')) else: newvar = newnc.createVariable('lon','f8',('lon')) basicvardef(newvar, 'longitude', 'Longitude','degrees_East') newvar[:] = lonmap newvar.setncattr('axis', 'X') newvar.setncattr('_CoordinateAxisType', 'Lon') if kind == 'Goode': newvar = newnc.createVariable('lat','f8',('lat','lon')) else: newvar = newnc.createVariable('lat','f8',('lat')) basicvardef(newvar, 'latitude', 'Latitude','degrees_North') newvar[:] = latmap newvar.setncattr('axis', 'Y') newvar.setncattr('_CoordinateAxisType', 'Lat') newvarinpt = newnc.createVariable('locinpt','i',('inpts')) basicvardef(newvarinpt, 'locinpt', 'input point located: 0: no, 1: yes','-') newvarindiff = newnc.createVariable('locindiff','f4',('inpts')) basicvardef(newvarindiff, 'locindiff', 'distance between input point and its final location','degree') set_attributek(newvarindiff,'authorized_minimum_distance',mindiff,'R') # map variable Lmapvalsshape = len(omapvals.shape) if Lmapvalsshape == 1: newvar = newnc.createVariable(varn, nctype(vartype), ('pts'), fill_value=fval) else: newvar = newnc.createVariable(varn, nctype(vartype), ('lat','lon'), fill_value=fval) basicvardef(newvar, varn, varlongname, varunits) newvar.setncattr('coordinates', 'lon lat') if Lmapvalsshape == 1: newvarin = newnc.createVariable('inpts', 'i4', ('pts')) else: newvarin = newnc.createVariable('inpts', 'i4', ('lat','lon')) basicvardef(newvarin, 'inpts', 'Equivalent point from the input source', '-') newvarin.setncattr('coordinates', 'lon lat') else: print ' ' + fname + "File '" + ofile + "' exists !!" print ' reading values from file' newnc = NetCDFFile(ofile, 'a') newvar = newnc.variables[varn] newvarin = newnc.variables['inpts'] newvarinpt = newnc.variables['locinpt'] newvarindiff = newnc.variables['locindiff'] amsk = np.arange(3) amsk = ma.masked_equal(amsk, 0) # fraclon = projlon[::Nmaplon*0.1] # fraclat = projlat[::Nmaplat*0.1] # print 'fraclon________________', fraclon.shape # print fraclon # print 'fraclat________________', fraclat.shape # print fraclat # Reducing the searching points Nsearchpt = 499999 Ninpts=np.min([Nsearchpt, Ninpts-ipoint]) newvarinvals = newvarinpt[ipoint:ipoint+Ninpts] maskpt = np.where(newvarinvals.mask == True, False, True) points = np.arange(ipoint,ipoint+Ninpts,1) mapoints = ma.array(points, mask=maskpt) ptsf = mapoints.compressed() Nptsf = len(ptsf) print Ninpts,'Npoints to find:', len(ptsf), ptsf[0:10], newvarindiff[ptsf[0:10]] ipoint = ptsf[0] print ' ' + fname + ': from:', Ninpts,'re-locating:',Nptsf,'points starting at',\ ipoint,'...' if kind == 'Goode': # newvar,newvarin,newvarinpt,newvarindiff = \ # fin.module_forinterpolate.coarseinterpolate(projlon, projlat, lonvs, \ # latvs, percen, mindiff, ivar, dimx, dimy, ninpts) for ir in range(ipoint,ipoint+Ninpts,fracs): iri = ir ire = ir + fracs print iri,',',ire # percendone(iri,Ninpts,0.5,'done:') lonvss = lonvs[iri:ire+1].astype('float64') latvss = latvs[iri:ire+1].astype('float64') inptss = newvarinpt[iri:ire+1].astype('int32') # newvar,newvarin,newvarinpt[pts],newvarindiff[pts] = \ # fin.module_forinterpolate.coarseinterpolateexact(projlon, projlat, \ # lonvss, latvss, fracd, mindiff, ovars) # Slow way idiff, ilonlatv = fin.module_forinterpolate.interpolate(projlon, projlat,\ lonvss, latvss, np.float64(mindiff), inptss) # Fortran routine provides ranges from 1,dimx! ilonlatv = ilonlatv - 1 for i in range(np.min([len(idiff),fracs])): newvar[ilonlatv[i,0],ilonlatv[i,1]] = ovar[iri + i] newvarin[ilonlatv[i,0],ilonlatv[i,1]] = iri + i newvarinpt[iri+i] = 1 newvarindiff[iri+i] = idiff[i] newnc.sync() # newnc.close() # quit() elif kind == 'lonlat': for iv in range(Ntotvals): difflat = np.abs(projlat - latvs[iv]) mindiffL = np.min(difflat) ilat = index_mat(difflat, mindiffL) difflon = np.abs(projlon - lonvs[iv]) mindiffl = difflon.min() ilon = index_mat(difflon, mindiffl) percendone(iv,Ntotvals,0.5,'done:') if mindiffl > mindiff or mindiffL > mindiff: print errormsg print ' ' + fname + ': for point #', iv,'lon,lat in incomplet map:', \ lonvs[iv], ',', latvs[iv], 'there is not a set of lon,lat in the ' + \ 'completed map closer than: ', mindiff, '!!' print ' minimum difflon:', np.min(difflon), 'difflat:', np.min(difflat) quit() if ilon >= 0 and ilat >= 0: newvar[ilat,ilon] = ovar[iv] newnc.sync() else: print errormsg print ' ' + fname + ': point iv:',iv,'at',lonvs[iv],',',latvs[iv],' not relocated !!' print ' mindiffl:',mindiffl,'mindiffL:',mindiffL,'ilon:',ilon,'ilat:',ilat quit(-1) newnc.sync() elif kind == 'lonlat_dxdyFix': for iv in range(Ntotvals): # for iv in range(15): # print 'Lluis:',iv,'lon lat:',lonvs[iv],',',latvs[iv] difflat = np.abs(projlat - latvs[iv]) mindiffL = np.min(difflat) ilat = index_mat(difflat, mindiffL) # print ' Lluis mindiffL:',mindiffL,'<> ilat:',ilat,'lat_ilat:',projlat[ilat],'Nprojlon:',Nprojlon[ilat],'shape oporjlon:',oprojlon[ilat,:].shape, type(oprojlon[ilat,:]) loncirc = np.zeros((Nprojlon[ilat]), dtype=np.float) loncirc[:] = np.asarray(oprojlon[ilat,0:int(Nprojlon[ilat])].flatten()) difflon = np.abs(loncirc - lonvs[iv]) mindiffl = difflon.min() ilon = index_mat(difflon, mindiffl) # print ' difflon:',type(difflon),'shape difflon',difflon.shape,'shape loncirc:', loncirc.shape # print ' Lluis mindiffl:',mindiffl,'<> ilon:',ilon,'oprojlon:', loncirc[ilon] percendone(iv,Ntotvals,0.5,'done:') if mindiffl > mindiff or mindiffL > mindiff: print errormsg print ' ' + fname + ': for point #', iv,'lon,lat in incomplet map:', \ lonvs[iv], ',', latvs[iv], 'there is not a set of lon,lat in the ' + \ 'completed map closer than: ', mindiff, '!!' print ' minimum difflon:', np.min(difflon), 'difflat:', np.min(difflat) quit() if ilon >= 0 and ilat >= 0: if Lmapvalsshape ==1: newvar[iv] = ovar[iv] newnc.sync() else: print errormsg print ' ' + fname + ': point iv:',iv,'at',lonvs[iv],',',latvs[iv],' not relocated !!' print ' mindiffl:',mindiffl,'mindiffL:',mindiffL,'ilon:',ilon,'ilat:',ilat quit(-1) newnc.sync() # Global attributes ## newnc.setncattr('script', fname) newnc.setncattr('version', '1.0') newnc.setncattr('author', 'L. Fita') newattr = set_attributek(newnc, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') newnc.setncattr('university', 'Pierre et Marie Curie') newnc.setncattr('country', 'France') for attrs in onc.ncattrs(): attrv = onc.getncattr(attrs) attr = set_attribute(newnc, attrs, attrv) newnc.sync() onc.close() newnc.close() print fname + ": Successfull written of file: '" + ofile + "' !!" return #Partialmap_Entiremap('longitude,latitude,std,5000.,Goode,Goode_5km.nc', '/home/lluis/etudes/DYNAMICO/ORCHIDEE/interpolation/data/#carteveg5km.nc', 'vegetation_map') #Partialmap_Entiremap('longitude,latitude,std,5000.,lonlat_dxdyFix', '/home/lluis/etudes/DYNAMICO/ORCHIDEE/interpolation/data/carteveg5km.nc', 'vegetation_map') #Partialmap_Entiremap('longitude,latitude,std,5000.,lonlat', '/home/lluis/etudes/DYNAMICO/ORCHIDEE/interpolation/data/carteveg5km.nc', 'vegetation_map') def reproject(values, filen, variable): """ Function to reproject values to another one reproject(values, filen, variable) values=[inlonn],[inlatn],[projfile],[projlonn],[projlatn],[kind],[dimvars] inlonn= name of the longitude values in the file to interpolate inlatn= name of the latitude values in the file to interpolate projfile= name of the file with the target projection projlonn= name of the longitude values in the file with the target projection projlatn= name of the latitude values in the file with the target projection kind= kind of interpolation 'npp': nearest point 'dis': distance weighted (4 closest) dimvars= ':', separated list of couples of dimension, and variable with its values [dimn]@[variabledim] from file to interpolate filen= name of file to interpolate variable = ',' list of variable to interpolate ('all' for all variables) """ import module_ForInt as fin fname = 'reproject' ofile = fname + '.nc' if values == 'h': print fname + '_____________________________________________________________' print reproject.__doc__ quit() arguments = '[inlonn],[inlatn],[projfile],[projlonn],[projlatn],[kind],[dimvars]' gen.check_arguments(fname, values, arguments, ',') inlonn = values.split(',')[0] inlatn = values.split(',')[1] projfile = values.split(',')[2] projlonn = values.split(',')[3] projlatn = values.split(',')[4] kind = values.split(',')[5] dimvars = gen.str_list(values.split(',')[6],':') varns = gen.str_list(variable,',') inf = NetCDFFile(filen, 'r') if not os.path.isfile(projfile): print errormsg print ' ' + fname + ": file with target projection '" + projfile + \ "' does not exist !!" quit(-1) projf = NetCDFFile(projfile, 'r') if not inf.variables.has_key(inlonn): print errormsg print ' ' +fname+ ": input file '" + filen + "' does not have longitude '" +\ inlonn + "' !!" quit(-1) if not inf.variables.has_key(inlatn): print errormsg print ' ' +fname+ ": input file '" + filen + "' does not have latitude '" + \ inlatn + "' !!" quit(-1) if not projf.variables.has_key(projlonn): print errormsg print ' ' + fname + ": file with target projection '" + projfile + \ "' does not have longitude '" + projlonn + "' !!" quit(-1) if not projf.variables.has_key(projlatn): print errormsg print ' ' + fname + ": file with target projection '" + projfile + \ "' does not have latitude '" + projlatn + "' !!" quit(-1) oinlon = inf.variables[inlonn] oinlat = inf.variables[inlatn] oprojlon = projf.variables[projlonn] oprojlat = projf.variables[projlatn] ilonvals0 = oinlon[:] ilatvals0 = oinlat[:] olonvals0 = oprojlon[:] olatvals0 = oprojlat[:] # ilonvals0 = np.arange(0.,360.,360./oinlon.shape[0]) # ilatvals0 = np.arange(-90.,90.,180./oinlat.shape[0]) # olonvals0 = np.arange(0.5,359.5,359./oprojlon.shape[0]) # olatvals0 = np.arange(-89.5,89.5,179./oprojlat.shape[0]) # Getting 2D longitudes and latitudes ilonvals, ilatvals = gen.lonlat2D(ilonvals0, ilatvals0) rlonvals, rlatvals = gen.lonlat2D(olonvals0, olatvals0) if len(ilonvals.shape) != 2: print errormsg print ' ' + fname + ': wrong input longitude rank:', ilonvals.shape, \ 'it must be 2!!' quit(-1) if len(ilatvals.shape) != 2: print errormsg print ' ' + fname + ': wrong input latitude rank:', ilatvals.shape, \ 'it must be 2!!' quit(-1) if len(rlonvals.shape) != 2: print errormsg print ' ' + fname + ': wrong output longitude rank:', rlonvals.shape, \ 'it must be 2!!' quit(-1) if len(rlatvals.shape) != 2: print errormsg print ' ' + fname + ': wrong output latitude rank:', rlatvals.shape, \ 'it must be 2!!' quit(-1) for i in range(2): if ilonvals.shape[i] != ilatvals.shape[i]: print errormsg print ' ' + fname + ': dimension',i,'of input longitude:', ilonvals[i], \ 'differs from latitude:', ilatvals[i], 'they must be the same!!' quit(-1) if rlonvals.shape[i] != rlatvals.shape[i]: print errormsg print ' ' + fname + ': dimension',i,'of output longitude:', ilonvals[i],\ 'differs from latitude:', ilatvals[i], 'they must be the same!!' quit(-1) # Checking right lon, lat ranges iminlon = np.min(ilonvals) imaxlon = np.max(ilonvals) rminlon = np.min(rlonvals) rmaxlon = np.max(rlonvals) iminlat = np.min(ilatvals) imaxlat = np.max(ilatvals) rminlat = np.min(rlatvals) rmaxlat = np.max(rlatvals) if np.abs(imaxlon - rmaxlon) > 100.: print warnmsg print ' ' + fname + ': reshaping longitudes!' print ' input maximum lon:', imaxlon print ' target maximum lon:', rmaxlon print ' shifting input longitudes...' if rmaxlon > 180.: ilonvals = np.where(ilonvals < 0., 360. + ilonvals, ilonvals) else: ilonvals = np.where(ilonvals > 180., ilonvals - 360., ilonvals) flipy = False if np.abs(ilatvals[0,0] - rlatvals[0,0]) > 90.: print warnmsg print ' ' + fname + ': reshaping latitudes!' print ' input minimum lat:', ilatvals[0,0] print ' target minimum lat:', rlatvals[0,0] print ' flip input latitudes...' ilatvals = ilatvals[::-1,:] flipy = True idx = ilonvals.shape[1] idy = ilonvals.shape[0] rdx = rlonvals.shape[1] rdy = rlonvals.shape[0] ilon = ilonvals.astype('float64') ilat = ilatvals.astype('float64') rlon = rlonvals.astype('float64') rlat = rlatvals.astype('float64') ilont = ilon.transpose() ilatt = ilat.transpose() rlont = rlon.transpose() rlatt = rlat.transpose() # dimension-vardims from input variable vardimns = {} dimvarns = {} for vardn in dimvars: dimn = vardn.split('@')[0] dimvn = vardn.split('@')[1] vardimns[dimn] = dimvn dimvarns[dimvn] = dimn # File creation onewnc = NetCDFFile(ofile, 'w') # Dimensions newdim = onewnc.createDimension('lon',rdx) newdim = onewnc.createDimension('lat',rdy) # Dimensions newvar = onewnc.createVariable('lon','f8',('lat','lon')) newvar[:] = rlon[:] newattr = basicvardef(newvar, 'longitude','Longitude','degrees_East') newattr = set_attribute(newvar, 'axis', 'X') newattr = set_attribute(newvar, '_CoordinateAxisType', 'Lon') newvar = onewnc.createVariable('lat','f8',('lat','lon')) newvar[:] = rlat[:] newattr = basicvardef(newvar, 'latitude','Latitude','degrees_North') newattr = set_attribute(newvar, 'axis', 'Y') newattr = set_attribute(newvar, '_CoordinateAxisType', 'Lat') # Which variables to re-project if varns[0] == 'all': varns = inf.variables.keys() # No interpolation of lon, lat varns.remove(inlonn) varns.remove(inlatn) for varn in varns: if not inf.variables.has_key(varn): print errormsg print ' ' + fname + ": input file '" + filen + "' does not have '" + \ varn + "' !!" quit(-1) ovarin = inf.variables[varn] if flipy: varin = ovarin[...,::-1,:] else: varin = ovarin[:] Nvarind = len(varin.shape) if Nvarind < 2: print errormsg print ' ' + fname + ': wrong input data rank:', varin.shape, \ 'it must be at least 2!!' quit(-1) varin = varin.astype('float64') varint = varin.transpose() # Variable's dimensions and their related variables vdims = list(ovarin.dimensions) for dimn in vdims: dimsfile = onewnc.dimensions.keys() if not gen.searchInlist(dimsfile, dimn): odimn = inf.dimensions[dimn] if odimn.isunlimited(): newdim = onewnc.createDimension(dimn, None) else: newdim = onewnc.createDimension(dimn, len(odimn)) if not vardimns.has_key(dimn): print errormsg print ' ' + fname + ": dimension '" + dimn + "' does not have "+\ 'assigned any variable !!' print ' assigned pairs (via [dimvars]) _______' gen.printing_dictionary(vardimns) quit(-1) else: add_vars(inf,onewnc,[vardimns[dimn]]) # Interpolated variable vardims = list(ovarin.dimensions) newvardims = gen.replace_list(vardims,dimvarns[inlonn],'lon') newvardims = gen.replace_list(newvardims,dimvarns[inlatn],'lat') varattrs = ovarin.ncattrs() if gen.searchInlist(varattrs,'_FillValue'): fillv = ovarin.getncattr('_FillValue') varint = np.where(varint == fillv, gen.fillValueF, varint) varattrs.remove('_FillValue') newvar = onewnc.createVariable(varn, 'f4', tuple(newvardims), \ fill_value=gen.fillValueF) onewnc.sync() # Do we have masked values? if type(varin) == type(ma.asarray([1,1])): inmask = np.where(varin.mask, 0, 1) print ' ' + fname + ": re-projecting masked '" + varn + "'", \ varin.shape, " ..." else: inmask = np.ones((varin.shape), dtype=int) print ' ' + fname + ": re-projecting '" +varn+ "'", varin.shape," ..." inmaskt = inmask.transpose() if Nvarind == 2: varoutt = fin.module_forinterpolate.var2d_intproj(var2din=varint, \ inlonv=ilont, inlatv=ilatt, projlon=rlont, projlat=rlatt, intkind=kind,\ mask=inmaskt, idimx=idx, idimy=idy, pdimx=rdx, pdimy=rdy) elif Nvarind == 3: varoutt = fin.module_forinterpolate.var3d_intproj(var3din=varint, \ inlonv=ilont, inlatv=ilatt, projlon=rlont, projlat=rlatt, intkind=kind,\ mask=inmaskt, idimx=idx, idimy=idy, pdimx=rdx, pdimy=rdy, \ d3=varin.shape[0]) elif Nvarind == 4: varoutt = fin.module_forinterpolate.var4d_intproj(var4din=varint, \ inlonv=ilont, inlatv=ilatt, projlon=rlont, projlat=rlatt, intkind=kind,\ mask=inmaskt, idimx=idx, idimy=idy, pdimx=rdx, pdimy=rdy, \ d3=varin.shape[1], d4=varin.shape[0]) elif Nvarind == 5: varoutt = fin.module_forinterpolate.var5d_intproj(var5din=varint, \ inlonv=ilont, inlatv=ilatt, projlon=rlont, projlat=rlatt, intkind=kind,\ mask=inmaskt, idimx=idx, idimy=idy, pdimx=rdx, pdimy=rdy, \ d3=varin.shape[2], d4=varin.shape[1], d5=varin.shape[0]) else: print errormsg print ' ' + fname + ': rank:', Nvarind, 'for input data not ready !!' quit(-1) # Recovering 'python' C-like space varout = varoutt.transpose() varout = np.where(varout >= 0.9*gen.fillValueF, gen.fillValueF, varout) newvar[:] = varout[:] for attrn in varattrs: attrv = ovarin.getncattr(attrn) set_attribute(newvar, attrn, attrv) set_attribute(newvar,'coordinates','lon lat') onewnc.sync() # Global attributes add_globattrs(inf,onewnc,'all') onewnc.sync() inf.close() projf.close() onewnc.close() print fname + ": successful re-projection saved in '" + ofile + "' !!" return def lonlatvarsFile(lonvals,latvals,dnx,dny,oLlnc): """ Function to CF-define the longitudes and latitudes variables within a file lonvals= 2D matrix with longitudes latvals= 2D matrix with latitudes dn[x/y]= names of the x and y dimensions oLlnc= netCDF file object where to write the lon,lats """ fname = 'lonlatvarsFile' if len(lonvals.shape) != 2: print errormsg print ' ' + fname +': wrong shape:',lonvals.shape," of 'lons' variable, " + \ 'must be 2D !!' quit(-1) if len(latvals.shape) != 2: print errormsg print ' ' + fname +': wrong shape:',latvals.shape," of 'lats' variable, " + \ 'must be 2D !!' quit(-1) dx = lonvals.shape[1] dy = lonvals.shape[0] if not gen.searchInlist(oLlnc.dimensions, dnx): print errormsg print ' ' + fname + ': netCDF object does not have x dimension: ', dnx,"' !!" quit(-1) if not gen.searchInlist(oLlnc.dimensions, dny): print errormsg print ' ' + fname + ': netCDF object does not have y dimension: ', dny,"' !!" quit(-1) # Creation of dimension variables newvar = oLlnc.createVariable('lon', 'f8', (dny,dnx)) basicvardef(newvar, 'longitude', 'Longitude','degrees_East') newvar[:] = lonvals newvar.setncattr('axis', 'X') newvar.setncattr('_CoordinateAxisType', 'Lon') newvar = oLlnc.createVariable('lat', 'f8', (dny,dnx)) basicvardef(newvar, 'latitude', 'Latitude','degrees_North') signlat = latvals[1,0] - latvals[0,0] newvar[:] = latvals newvar.setncattr('axis', 'Y') newvar.setncattr('_CoordinateAxisType', 'Lat') return def subbasin(values, ncfile): """ Function to retrieve the subbasin (all the sub-flows untila a given lon,lat) values= [subname],[sublon],[sublat] subname: name for the subbasin ('!' for spaces) sub[lon/lat]: longitude and latitude of the subbasin point searchdirection: searching direction river-up ('left': clockwise, 'right': anti-clockwise) ncfile= name of the file to use (commontly ORCHIDEE's 'routing.nc') """ fname = 'subbasin' if values == 'h': print fname + '_____________________________________________________________' print subbasin.__doc__ quit() arguments = '[subnmame],[sublon],[sublat],[searchdirection]' gen.check_arguments(fname, values, arguments, ',') ofile = 'subbasin.nc' onc = NetCDFFile(ncfile, 'r') subname = values.split(',')[0].replace('!',' ') sublon = np.float(values.split(',')[1]) sublat = np.float(values.split(',')[2]) searchdirection = values.split(',')[3] Lncfile = len(ncfile) if ncfile[Lncfile-10:Lncfile] == 'routing.nc': lonname = 'nav_lon' latname = 'nav_lat' tripname = 'trip' Soutflows = ['1: N', '2: NE', '3: E', '4: SE', '5: S', '6: SW', '7: W', \ '8: NW', '97: sub-basin', '98: small to sea', '99: large to sea'] filen = 'routing.nc' else: print errormsg print ' ' + fname + ": file '" + ncfile + "' not ready!!" quit(-1) Ndiroutflow = len(Soutflows) olons = onc.variables[lonname] olats = onc.variables[latname] ooutflows = onc.variables[tripname] # Flipping the routing file lons = olons[:] lats = olats[:] outflows = np.array(ooutflows[:], dtype=int) difflonlat = np.sqrt((lons - sublon)**2 + (lats - sublat)**2) mindifflonlat = np.min(difflonlat) if mindifflonlat != 0.: print errormsg print ' ' + fname + ': wrong initial lon,lat of the basin!!' print ' for lon,lat:', sublon,',',sublat,'there is not grid point ' + \ "coincidence in file '" + ncfile + "'" quit(-1) isub = gen.index_mat(difflonlat, mindifflonlat) print ' ' + fname + "initial point of subbasin '" + subname + "' in the " + \ 'matrix:', isub masksubbasin, subflows, subflowspt = gen.subbasin_point(outflows, isub, \ searchdirection) if len(subflows.keys()) == 1: print errormsg print ' ' + fname + ": Only 1 point for subbasin '" + subname + \ "' at lon,lat:", sublon,',',sublat print ' values around (3x3 centered at: ', isub, ')' lon3x3 = gen.vals_around(lons, isub) lat3x3 = gen.vals_around(lats, isub) out3x3 = gen.vals_around(outflows, isub) print ' lon lat out _______' for i in range(3): print lon3x3[i,:], lat3x3[i,:], out3x3[i,:] print 'out equivalencies:', Soutflows quit(-1) onewnc = NetCDFFile(ofile, 'w') dx = outflows.shape[1] dy = outflows.shape[0] # Number of sub-flows within the sub-basin Nsubflows = len(subflows.keys()) # Maximum number of grid points of a sub-flow Nmaxgridsubflow = -1 Ngridpoint = 0 for subflow in subflows.keys(): Ngridsubflow = len(subflows[subflow]) Ngridpoint = Ngridpoint + Ngridsubflow if Ngridsubflow > Nmaxgridsubflow: Nmaxgridsubflow = Ngridsubflow # Creation of dimensions Lstr = 50 newdim = onewnc.createDimension('x', dx) newdim = onewnc.createDimension('y', dy) newdim = onewnc.createDimension('Nsubflow', Nsubflows) newdim = onewnc.createDimension('Lstring', Lstr) newdim = onewnc.createDimension('Ngridsubflow', Nmaxgridsubflow) newdim = onewnc.createDimension('outflow', Ndiroutflow) if filen == 'routing.nc': lonlatvarsFile(lons, lats[::-1,:], 'x', 'y', onewnc) else: lonlatvarsFile(lons, lats, 'x', 'y', onewnc) # Creation of variables newvar = onewnc.createVariable('subbasin', 'c', ('Lstring')) for ic in range(len(subname)): newvar[ic] = subname[ic:ic+1] newattr = set_attributek(newvar,'Ngridpoint',Ngridpoint,'I') newattr = set_attributek(newvar,'Nsubflow',Nsubflows,'I') newattr = set_attributek(newvar,'lon',sublon,'R') newattr = set_attributek(newvar,'lat',sublat,'R') newattr = set_attributek(newvar,'icoord',isub[1],'I') newattr = set_attributek(newvar,'jcoord',isub[0],'I') onewnc.sync() newvar = onewnc.createVariable('subbasinmask','i',('y','x')) vardef = basicvardef(newvar, 'subbasinmask', 'mask of the subbasin','1') if filen == 'routing.nc': newvar[:] = masksubbasin[::-1,:] else: newvar[:] = masksubbasin newattr = set_attributek(newvar,'coordinates','lon lat','S') onewnc.sync() nsubflows = subflows.keys() nsubflows.sort newvar = onewnc.createVariable('subflow', 'c', ('Nsubflow', 'Lstring')) vardef = basicvardef(newvar, 'subflow', 'reference of the sub-flow','-') newvals = writing_str_nc(newvar, nsubflows, Lstr) newattr = set_attributek(newvar,'searching_direction',searchdirection,'S') onewnc.sync() # sub-flow grid point data newvar = onewnc.createVariable('Nsubflow', 'i', ('Nsubflow')) basicvardef(newvar, 'Nsubflow', 'Number of grid points of the sub-flow', '-') newvarlon = onewnc.createVariable('lonsubflow', 'f4', ('Nsubflow','Ngridsubflow')) vardef = basicvardef(newvarlon, 'lonsubflow', 'Longitude of the grid point of '+ \ 'the sub-flow','degrees_East') newvarlat = onewnc.createVariable('latsubflow', 'f4', ('Nsubflow','Ngridsubflow')) vardef = basicvardef(newvarlat, 'latsubflow', 'Latitude of the grid point of ' + \ 'the sub-flow','degrees_North') newvartrip = onewnc.createVariable('outsubflow', 'i', ('Nsubflow','Ngridsubflow')) vardef = basicvardef(newvartrip, 'outsubflow', 'out-flow of the grid point of '+ \ 'the sub-flow','outflow') isub = 0 for subf in nsubflows: grids = subflows[subf] Ngrid = len(grids) newvar[isub] = Ngrid # print ' ' + fname + ' Lluis; isub:', isub,'subf:',subf,'grids:',grids,'Ngrid',Ngrid for igrid in range(Ngrid): jigrid = subflowspt[grids[igrid]] lonv = lons[jigrid[0],jigrid[1]] latv = lats[jigrid[0],jigrid[1]] tripv = outflows[jigrid[0],jigrid[1]] # print ' jigrid:', jigrid,' lon:', lonv,'lat:',latv newvarlon[isub,igrid] = lonv newvarlat[isub,igrid] = latv newvartrip[isub,igrid] = tripv isub = isub + 1 onewnc.sync() newvar = onewnc.createVariable('outflow', 'c', ('outflow', 'Lstring')) vardef = basicvardef(newvar, 'outflow', 'out-flow direction','1') newvals = writing_str_nc(newvar, Soutflows, Lstr) onewnc.sync() # Global attributes onewnc.setncattr('script', fname) onewnc.setncattr('version', '1.0') onewnc.setncattr('author', 'L. Fita') newattr = set_attributek(onewnc, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') onewnc.setncattr('university', 'Pierre et Marie Curie') onewnc.setncattr('country', 'France') newattr = set_attributek(onewnc, 'file', ncfile ,'S') for attrs in onc.ncattrs(): attrv = onc.getncattr(attrs) attr = set_attribute(onewnc, attrs, attrv) onc.close() onewnc.sync() onewnc.close() print fname + ": succesfull written of '" + ofile + "' !!" return def var_model(varname, onc): """ Function to provide the way to compute a CF-variable providing its name varname= CF-compilant variable name onc= netCDF file object from which the variable should be retrieved printing= whether result should be printed in standard output or no ('true'/'false') NOTE: on output 'computevardiag', first value of the list is the 'diagnostics.py' name >>> onc = NetCDFFile('/home/lluis/PY/wrfout_d01_2001-11-11_00:00:00', 'r') >>> mod, diag = var_model('tas', onc) ['T2'], None >>> mod, diag = var_model('hurs', onc) None, [['WRFrhs', 'PSFC', 'T2', 'Q2']] """ fname = 'var_model' ncvars = onc.variables # Getting non-chceking variables folder = os.path.dirname(os.path.realpath(__file__)) infile = folder + '/diagnostics.inf' if not os.path.isfile(infile): print errormsg print ' ' + fname + ": File '" + infile + "' does not exist !!" quit(-1) ncf = open(infile, 'r') for line in ncf: if line[0:9] == '# NOcheck': NOchk = line.split('=')[1].replace(' ','').replace('\n','').split(':') ncf.close() break # Variables from which CF variable might be computed modelvars = gen.CFvar_MODvar(varname) diagvars = gen.CFvar_DIAGvar(varname) computevarmod = [] if modelvars is not None: for var in modelvars: if gen.searchInlist(ncvars,var): computevarmod.append(var) if len(computevarmod) < 1: computevarmod = None computevardiag = [] if diagvars is not None: for diagn in diagvars.keys(): combovars = diagvars[diagn] # Checking if there is more than one possible combination if type(combovars[0]) == type([1,2]): for combovs in combovars: alltrue = False if type(combovs) == type([1,2]): for cvar in combovs: if gen.searchInlist(NOchk,cvar): alltrue = True elif gen.searchInlist(ncvars,cvar): alltrue = True else: alltrue = False break else: if gen.searchInlist(ncvars,combovs): alltrue = True if alltrue: computevardiag.append([diagn] + combovs) break else: combovar = combovars alltrue = False if type(combovar) == type([1,2]): for cvar in combovar: if gen.searchInlist(NOchk,cvar): alltrue = True elif gen.searchInlist(ncvars,cvar): alltrue = True else: alltrue = False break else: if gen.searchInlist(ncvars,combovar): alltrue = True if alltrue: computevardiag.append([diagn] + combovar) break if alltrue: break if len(computevardiag) < 1: computevardiag = None return computevarmod, computevardiag def computevar_model(values, ncfile): """ Function to provide the way to compute a CF-variable providing its name values= CF-compilant variable name ncfile= netCDF file from which the variable should be retrieved """ fname = 'computevar_model' if values == 'h': print fname + '_____________________________________________________________' print computevar_model.__doc__ quit() varname = values ncobj = NetCDFFile(ncfile, 'r') varmod, vardiag = var_model(varname, ncobj) print ' ' + fname + ": CF variable '" + varname + "' can be computed ______" print ' model:', varmod print ' diagnostics:', vardiag if varmod is not None: varmodsingle = gen.numVector_String(varmod,':') else: varmodsingle = 'None' if vardiag is not None: for diag in vardiag: if diag == vardiag[0]: vardiagsingle = gen.numVector_String(diag,':') else: vardiagsingle = vardiagsingle + '#' + gen.numVector_String(diag,':') else: vardiagsingle = 'None' print ' |singleline@model=' + varmodsingle + '@diag=' + vardiagsingle ncobj.close() return varmod, vardiag #computevar_model('pr', '/home/lluis/PY/wrfout_d01_2001-11-11_00:00:00') #quit() class ModelChar(object): """ Class object to determine major characterisitcs of a given model output model= name of the model ncfile= name of the file to use to check """ def __init__(self,model,ncfile): fname = 'ModelChar' models = ['WRF', 'LMDZ', 'WRF_LMDZ', 'DYNAMICO', 'mthDYNAMICO'] if model == 'WRF' or model == 'WRF_LMDZ': if model == 'WRF': self.name = 'WRF' self.model = 'Weather Research and Forecasting' else: self.name = 'WRF_LMDZ' self.model= 'Laboratoire de M' + unichr(233) + 't' + unichr(233) + \ 'orologie Dynamique Zoom physics coupled to Weather Research ' + \ 'and Forecasting' self.dimxn = 'west_east' self.dimyn = 'south_north' self.dimzn = 'bottom_top' self.dimtn = 'Time' self.dimsn = 'soil_layers_stag' self.vardxn = 'XLONG' self.vardyn = 'XLAT' self.vardzn = 'ZNU' self.vardtn = 'Times' self.vardsn = 'ZS' if ncfile is not None and ncfile[0:2] == 'geo': self.vardxn = 'XLONG_M' self.vardyn = 'XLAT_M' if ncfile is not None: attrv = get_attribute('TITLE', ncfile, 'global') self.version = attrv.split(' ')[4] elif model == 'LMDZ': self.name = 'LMDZ' self.model = 'Laboratoire de M' + unichr(233) + 't' + unichr(233) + \ 'orologie Dynamique Zoom' self.dimxn = 'lon' self.dimyn = 'lat' self.dimzn = 'presnivs' self.dimtn = 'time_counter' self.dimsn = None self.vardxn = 'lon' self.vardyn = 'lat' self.vardzn = 'presnivs' self.vardtn = 'time_counter' self.vardsn = None if ncfile is not None: self.version = None elif model == 'DYNAMICO': self.name = 'DYNAMICO' self.model = 'DYNAMICO' self.dimxn = 'lon' self.dimyn = 'lat' self.dimzn = 'presnivs' self.dimtn = 'time_counter' self.dimsn = None self.vardxn = 'lon' self.vardyn = 'lat' self.vardzn = 'presnivs' self.vardtn = 'time_counter' self.vardsn = None if ncfile is not None: self.version = None elif model == 'mthDYNAMICO': self.name = 'mthDYNAMICO' self.model = 'monthly means from DYNAMICO' self.dimxn = 'lon' self.dimyn = 'lat' self.dimzn = 'presnivs' self.dimtn = 'time_counter' self.dimsn = None self.vardxn = 'lon' self.vardyn = 'lat' self.vardzn = 'presnivs' self.vardtn = 'time_centered' self.vardsn = None if ncfile is not None: self.version = None else: print errormsg print ' ' + fname + ": model '" + model + "' not ready !!" quit(-1) self.dimensions = [self.dimxn, self.dimyn, self.dimzn, self.dimtn, self.dimsn] self.vardimensions = [self.vardxn, self.vardyn, self.vardzn, self.vardtn, self.vardsn] def model_characteristics(values, ncfile, variables): """ Functino to provide major characterisitcs of a given model output values= [model] model: name of the model variables= i whether should the output be printed or not ('true/false') ncfile= name of the file to use to check """ fname = 'model_characteristics' if values == 'h': print fname + '_____________________________________________________________' print model_characteristics.__doc__ quit() model = values if ncfile == 'None': ncfile = None modchar = ModelChar(model,ncfile) if gen.Str_Bool(variables): gen.printing_class(modchar) gen.singleline_printing_class(modchar) return modchar def pinterp(values, ncfile, variables): """ Function to vertically interpolate using subroutines from the p_interp.F90 NCAR program Using fortran codes: module_generic.F90 foudre: f2py -m module_ForInterpolate --f90exec=/usr/bin/gfortran-4.7 -c module_generic.F90 module_ForInterpolate.F90 >& run_f2py.log ciclad: f2py --f90flags="-fPIC" -L/opt/canopy-1.3.0/Canopy_64bit/System/lib/ -L/usr/lib64/ -L/opt/canopy-1.3.0/Canopy_64bit/System/lib/ -m module_ForInterpolate -c module_generic.F90 module_ForInterpolate.F90 values= [interplevs],[linloginterp],[extrap] [interplevs]: ':' separated list of pressure values to interpolate [Pa] (decreassing in Pa) [linloginterp]: kind of linear interpolation 1: linear interp in pressure 2: linear interp in ln(pressure) [extrap]: whether to set to missing value below/above model ground and top (0), or extrapolate (1) ncfile= WRF file to use variables = ',' separated list of names of variables to interpolate ('all', fo all 4D-atmospheric variables) 'WRFght': for WRF geopotential height 'WRFrh': for WRF relative humidity 'WRFt': for WRF temperature 'WRFu': for WRF x-wind de-staggered 'WRFv': for WRF y-wind de-staggered """ import module_ForInt as fin fname = 'pinterp' if values == 'h': print fname + '_____________________________________________________________' print pinterp.__doc__ quit() arguments = '[interplevs],[linloginterp],[extrap]' gen.check_arguments(fname, values, arguments, ',') interplevs = np.array(values.split(',')[0].split(':'), dtype=np.float64) linloginterp = np.int32(values.split(',')[1]) extrap = np.int32(values.split(',')[2]) ofile = 'pinterp.nc' CFdims = ['time', 'pres', 'lat', 'lon'] # Just in case there are weird units, attribute value will be taken from input file CFattrs = ['standard_name', 'long_name'] onc = NetCDFFile(ncfile, 'r') # Gessing orgin of the file dimsinfile = onc.dimensions.keys() varsinfile = onc.variables.keys() if gen.searchInlist(dimsinfile,'bottom_top'): modname = 'WRF' print warnmsg print ' ' + fname + ": gessing that file '" + ncfile + "' comes from '" + \ modname + "' !!" # Variables to interpolate MODdims = ['Time', 'bottom_top', 'south_north', 'west_east'] newMODdims = ['Time', 'pres', 'south_north', 'west_east'] notCHK = ['WRFght', 'WRFrh', 'WRFt', 'WRFu', 'WRFv'] MODvarrequired = ['P', 'PB', 'PSFC', 'PH', 'PHB', 'HGT', 'T', 'QVAPOR', \ 'XLONG', 'XLAT', 'Times'] MODvardims = ['XLONG', 'XLAT', 'Times'] elif gen.searchInlist(dimsinfile,'presnivs'): modname = 'LMDZ' print warnmsg print ' ' + fname + ": gessing that file '" + ncfile + "' comes from '" + \ modname + "' !!" # Variables to interpolate MODdims = ['time_counter', 'presnivs', 'lat', 'lon'] newMODdims = ['time_counter', 'pres', 'lat', 'lon'] notCHK = [] MODvarrequired = ['pres', 'psol', 'geop', 'phis', 'temp', 'ovap', 'lon', \ 'lat', 'time_counter'] MODvardims = ['lon', 'lat', 'time_counter'] elif gen.searchInlist(dimsinfile,'pres') and gen.searchInlist(varsinfile, \ 'psol'): modname = 'cfLMDZ' print warnmsg print ' ' + fname + ": gessing that file '" + ncfile + "' comes from '" + \ modname + "' ('pres, time_counter' variables renamed as 'p, time') !!" # Variables to interpolate MODdims = ['time', 'pres', 'lat', 'lon'] newMODdims = ['time', 'pres', 'lat', 'lon'] notCHK = [] MODvarrequired = ['p', 'psol', 'geop', 'phis', 'temp', 'ovap', 'lon', \ 'lat', 'time'] MODvardims = ['lon', 'lat', 'time'] else: modname = 'CF' print warnmsg print ' ' + fname + ": gessing that file '" + ncfile + "' comes from '" + \ modname + "' !!" # Variables to interpolate MODdims = ['time', 'pres', 'lat', 'lon'] newMODdims = ['time', 'pres', 'lat', 'lon'] notCHK = [] MODvarrequired = ['p', 'ps', 'z', 'orog', 'ta', 'hus', 'lon', 'lat', 'time'] MODvardims = ['lon', 'lat', 'time'] varns = gen.str_list(variables, ',') if variables == 'all': varns = [] for vn in onc.variables: ovn = onc.variables[vn] coincdims = set(WRFdims) & set(ovn.dimensions) if len(MODdims) == len(ovn.dimensions) and len(coincdims) == len(MODdims): varns.append(vn) varns = varns + notCHK for vn in varns: if not gen.searchInlist(onc.variables, vn) and \ not gen.searchInlist(notCHK, vn): print errormsg print ' ' + fname + "': file '" + ncfile + "' does not have variable '"+\ vn + "' !!" quit(-1) # looking for model required variables for var in MODvarrequired: if not gen.searchInlist(onc.variables, var): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have required " + \ "variable '" + var + "' !!" quit(-1) # Computing necessary variables # pressure if modname == 'WRF': ovar1 = onc.variables['P'] ovar2 = onc.variables['PB'] pres0 = ovar1[:] + ovar2[:] pres = pres0.astype('float64') dimx = pres.shape[3] dimy = pres.shape[2] dimz = pres.shape[1] dimt = pres.shape[0] MODdimvs = {'Time':dimt, 'bottom_top':dimz, 'south_north':dimy, \ 'west_east': dimx} CFdimvs = {'time': dimt, 'bottom_top': dimz, 'lat': dimy, 'lon': dimx} unstaggerDIM = 'west_east' elif modname == 'LMDZ': ovar1 = onc.variables['pres'] pres = ovar1[:].astype('float64') dimx = pres.shape[3] dimy = pres.shape[2] dimz = pres.shape[1] dimt = pres.shape[0] MODdimvs = {'time_counter':dimt, 'presnivs':dimz, 'lat':dimy, 'lon': dimx} CFdimvs = {'time': dimt, 'presnivs': dimz, 'lat': dimy, 'lon': dimx} unstaggerDIM = 'lon' elif modname == 'cfLMDZ': ovar1 = onc.variables['p'] pres = ovar1[:].astype('float64') dimx = pres.shape[3] dimy = pres.shape[2] dimz = pres.shape[1] dimt = pres.shape[0] MODdimvs = {'time':dimt, 'pres':dimz, 'lat':dimy, 'lon': dimx} CFdimvs = {'time': dimt, 'pres': dimz, 'lat': dimy, 'lon': dimx} unstaggerDIM = 'lon' else: ovar1 = onc.variables['p'] pres = ovar1[:].astype('float64') dimx = pres.shape[3] dimy = pres.shape[2] dimz = pres.shape[1] dimt = pres.shape[0] MODdimvs = {'time':dimt, 'pres':dimz, 'lat':dimy, 'lon': dimx} CFdimvs = {'time': dimt, 'pres': dimz, 'lat': dimy, 'lon': dimx} unstaggerDIM = 'lon' print ' ' + fname + ': CF dimension lengths:', CFdimvs # sfc pressure if modname == 'WRF': ovar1 = onc.variables['PSFC'] psfc = ovar1[:].astype('float64') elif modname == 'LMDZ' or modname == 'cfLMDZ': ovar1 = onc.variables['psol'] psfc = ovar1[:].astype('float64') else: ovar1 = onc.variables['ps'] psfc = ovar1[:].astype('float64') # geopotential height if modname == 'WRF': ovar1 = onc.variables['PH'] ovar2 = onc.variables['PHB'] geop0 = ovar1[:] + ovar2[:] unstg = list(geop0.shape) unstg[1] = unstg[1] - 1 geop = np.zeros(tuple(unstg), dtype=np.float) geop = 0.5*(geop0[:,1:dimz+1,:,:] + geop0[:,0:dimz,:,:]).astype('float64') elif modname == 'LMDZ' or modname == 'cfLMDZ': ovar1 = onc.variables['geop'] geop = ovar1[:].astype('float64') else: ovar1 = onc.variables['z'] geop = ovar1[:].astype('float64') # terrain height if modname == 'WRF': ovar1 = onc.variables['HGT'] hgt = ovar1[0,:,:].astype('float64') elif modname == 'LMDZ' or modname == 'cfLMDZ': grav = 9.81 ovar1 = onc.variables['phis'] hgt = (ovar1[0,:,:]/grav).astype('float64') else: ovar1 = onc.variables['orog'] hgt = ovar1[:].astype('float64') # water vapour mixing ratio if modname == 'WRF': ovar1 = onc.variables['QVAPOR'] qv = ovar1[:].astype('float64') elif modname == 'LMDZ' or modname == 'cfLMDZ': ovar1 = onc.variables['ovap'] qv = ovar1[:].astype('float64') else: ovar1 = onc.variables['hus'] qv = ovar1[:].astype('float64') # temperature if modname == 'WRF': Rd = np.float64(287.04) Cp = np.float64(7.*Rd/2.) RCP = np.float64(Rd/Cp) p0 = np.float64(100000.) ovar10 = onc.variables['T'] var10 = ovar10[:] ovar1 = (var10).astype('float64') temp0 = (ovar1[:]+300.)*(pres[:]/p0)**RCP temp = temp0.astype('float64') elif modname == 'LMDZ' or modname == 'cfLMDZ': ovar1 = onc.variables['temp'] temp = ovar1[:].astype('float64') else: ovar1 = onc.variables['ta'] temp = ovar1[:].astype('float64') onewnc = NetCDFFile(ofile, 'w') # Creation of dimensions if modname == 'WRF': newdim = onewnc.createDimension('west_east', dimx) newdim = onewnc.createDimension('south_north', dimy) newdim = onewnc.createDimension('pres', len(interplevs)) newdim = onewnc.createDimension('Time', None) elif modname == 'LMDZ': newdim = onewnc.createDimension('lon', dimx) newdim = onewnc.createDimension('lat', dimy) newdim = onewnc.createDimension('pres', len(interplevs)) newdim = onewnc.createDimension('time_counter', None) else: newdim = onewnc.createDimension('lon', dimx) newdim = onewnc.createDimension('lat', dimy) newdim = onewnc.createDimension('pres', len(interplevs)) newdim = onewnc.createDimension('time', None) # Creation of variable dimensions for var in MODvardims: ovar = onc.variables[var] varinf = variable_inf(ovar) newvard = [] for vd in varinf.dimns: if not gen.searchInlist(onewnc.dimensions, vd) and \ not gen.searchInlist(CFdims, vd): newdim = onewnc.createDimension(vd, len(onc.dimensions[vd])) if gen.searchInlist(CFdims,vd): icfdim = CFdims.index(vd) newvard.append(MODdims[icfdim]) else: newvard.append(vd) newvar = onewnc.createVariable(var, nctype(varinf.dtype), \ tuple(newvard)) newvar[:] = ovar[:] for attrn in ovar.ncattrs(): attrv = ovar.getncattr(attrn) attr = set_attribute(newvar, attrn, attrv) # Creation of pressure variable dimension newvar = onewnc.createVariable('pres', 'f8', ('pres')) newvar[:] = interplevs[:] basicvardef(newvar, 'pressure', 'Pressure', 'Pa') attr = set_attribute(newvar, 'positive', 'down') attr = set_attribute(newvar, 'axis', 'Z') print ' ' + fname + ' interpolating at pressure levels _______' print ' ', interplevs for vn in varns: print " '" + vn + "' ..." newvarattr = {} varin = None if gen.searchInlist(notCHK, vn): if vn == 'WRFght': varin = geop isgeop = True varattrs = gen.variables_values('WRFght') CFvn = varattrs[0] newvarattr['standard_name'] = varattrs[1] newvarattr['long_name'] = varattrs[4].replace('|',' ') newvarattr['units'] = varattrs[5] elif vn == 'WRFrh': # relative humidity ovar1 = 10.*0.6112*np.exp(17.67*(temp-273.16)/(temp-29.65)) ovar2 = 0.622*ovar1/(0.01*pres-(1.-0.622)*ovar1) varin = 100.*qv/ovar2 isgeop = False varattrs = gen.variables_values('WRFrh') CFvn = varattrs[0] newvarattr['standard_name'] = varattrs[1] newvarattr['long_name'] = varattrs[4].replace('|',' ') newvarattr['units'] = varattrs[5] elif vn == 'WRFt': varin = temp isgeop = False varattrs = gen.variables_values('WRFt') CFvn = varattrs[0] newvarattr['standard_name'] = varattrs[1] newvarattr['long_name'] = varattrs[4].replace('|',' ') newvarattr['units'] = varattrs[5] elif vn == 'WRFu': ovarin = onc.variables['U'] print infmsg print ' ' + fname + ': De-staggering x-wind variable !!' print ' from:', ovarin.shape, 'to', (dimt, dimz, dimy, dimx) varin = np.zeros((dimt, dimz, dimy, dimx), dtype=np.float64) varin[:] = 0.5*(ovarin[:,:,:,0:dimx] + ovarin[:,:,:,1:dimx+1]) isgeop = False varattrs = gen.variables_values('ua') CFvn = varattrs[0] newvarattr['standard_name'] = varattrs[1] newvarattr['long_name'] = varattrs[4].replace('|',' ') newvarattr['units'] = varattrs[5] elif vn == 'WRFv': ovarin = onc.variables['V'] print infmsg print ' ' + fname + ': De-staggering y-wind variable !!' print ' from:', ovarin.shape, 'to', (dimt, dimz, dimy, dimx) varin = np.zeros((dimt, dimz, dimy, dimx), dtype=np.float64) varin[:] = 0.5*(ovarin[:,:,0:dimy,:] + ovarin[:,:,1:dimy+1,:]) isgeop = False varattrs = gen.variables_values('va') CFvn = varattrs[0] newvarattr['standard_name'] = varattrs[1] newvarattr['long_name'] = varattrs[4].replace('|',' ') newvarattr['units'] = varattrs[5] elif vn == 'pres': varin = pres isgeop = False varattrs = gen.variables_values('pres') CFvn = varattrs[0] newvarattr['standard_name'] = varattrs[1] newvarattr['long_name'] = varattrs[4].replace('|',' ') newvarattr['units'] = varattrs[5] # elif not gen.searchInlist(MODvarrequired, vn) or vn == 'QVAPOR': else: ovarin = onc.variables[vn] varinf = variable_inf(ovarin) varin = ovarin[:] # de-staggering if gen.searchInlist(ovarin.dimensions, unstaggerDIM): varin = gen.stagger_unstagger(varin,list(ovarin.dimensions),MODdimvs) else: varin = gen.stagger_unstagger(varin,list(ovarin.dimensions),CFdimvs) isgeop = False if vn == 'z': isgeop = True varattrs = gen.variables_values(vn) CFvn = varattrs[0] for attrn in ovarin.ncattrs(): attrv = ovarin.getncattr(attrn) newvarattr[attrn] = attrv newvarattr['standard_name'] = varattrs[1] newvarattr['long_name'] = varattrs[4].replace('|',' ') if varin is not None: newvar = onewnc.createVariable(CFvn, 'f4', tuple(newMODdims), \ fill_value=gen.fillValueF) # There is a memroy issue, thus is necessary to split the matrix... if np.prod(list(varin.shape)) > 150*150*39: dimtfrac = 5 varinterp = np.zeros(tuple([dimx,dimy,len(interplevs),dimtfrac]), dtype=np.float) print warnmsg print ' ' + fname + ': variable to interpolate:', varin.shape, \ 'too big!!' print ' p-interpolation will be done by time-slices of 5 time-steps' print range(0,dimtfrac*int(dimt/dimtfrac),dimtfrac) else: dimtfrac = dimt varinterp = np.zeros(tuple([dimx,dimy,len(interplevs),dimt]), dtype=np.float) for itt in range(0,dimtfrac*int(dimt/dimtfrac),dimtfrac): tini = itt tend = itt + dimtfrac varint = varin[tini:tend,:,:,:].transpose() prest = pres[tini:tend,:,:,:].transpose() psfct = psfc[tini:tend,:,:].transpose() hgtt = hgt.transpose() tempt = temp[tini:tend,:,:,:].transpose() qvt = qv[tini:tend,:,:,:].transpose() varinterp = fin.module_forinterpolate.interp( data_in=varint, \ pres_field=prest, interp_levels=interplevs, psfc=psfct, \ ter=hgtt, tk=tempt, qv=qvt, linlog=linloginterp, \ extrapolate=extrap, geopt=isgeop, missing=gen.fillValueF) newvar[tini:tend,:,:,:] = varinterp.transpose() if dimtfrac != dimt and tend != dimt-1: print ' finishing time-splitting:', tend, ', ', dimt varint = varin[tend:dimt,:,:,:].transpose() prest = pres[tend:dimt,:,:,:].transpose() psfct = psfc[tend:dimt,:,:].transpose() hgtt = hgt.transpose() tempt = temp[tend:dimt,:,:,:].transpose() qvt = qv[tend:dimt,:,:,:].transpose() varinterplast = fin.module_forinterpolate.interp( data_in=varint, \ pres_field=prest, interp_levels=interplevs, psfc=psfct, \ ter=hgtt, tk=tempt, qv=qvt, linlog=linloginterp, \ extrapolate=extrap, geopt=isgeop, missing=gen.fillValueF) newvar[tend:dimt,:,:,:] = varinterplast.transpose() for attrn in newvarattr.keys(): if attrn != '_FillValue': attr = set_attribute(newvar, attrn, newvarattr[attrn]) onewnc.sync() # Global attributes onewnc.setncattr('author', 'L. Fita') newattr = set_attributek(onewnc, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') onewnc.setncattr('university', 'Pierre Marie Curie - Jussieu') onewnc.setncattr('center', 'Centre National de Recherches Scientifiques') onewnc.setncattr('city', 'Paris') onewnc.setncattr('country', 'France') onewnc.setncattr('script', 'nc_var_tools.py') onewnc.setncattr('function', 'pinterp') onewnc.setncattr('version', '1.0') onewnc.setncattr('oringal', 'Interpolation using p_interp.F90 subroutines') newattr = set_attributek(onewnc, 'original_subroutines_author', unicode('Cindy '+\ 'Bruy' + unichr(232) + 're'), 'U') onewnc.setncattr('original_subroutines_date', 'November 2007') for attrn in onc.ncattrs(): attrv = onc.getncattr(attrn) attr = set_attribute(onewnc, attrn, attrv) onc.close() onewnc.sync() onewnc.close() print fname + ": successfull written of '" + ofile + "' !!" return def get_Variables(values, ncfile, variables): """ Function to get a list of variables from an existing file novar= behaviour when no variable is found 'error': an error messages arise and program stops 'warn': a warning messages arise and program continues ncfile= WRF file to use variables = ',' separated list of names of variables to get """ fname = 'get_Variables' if values == 'h': print fname + '_____________________________________________________________' print get_Variables.__doc__ quit() novar = values variables = variables.split(',') ofile = 'get_Variables.nc' onewnc = NetCDFFile(ofile, 'w') onc = NetCDFFile(ncfile, 'r') filevars = list(onc.variables.keys()) for var in variables: if not gen.searchInlist(filevars, var): if novar == 'error': print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have " + \ "variable '" + var + "' !!" print ' variables in file:', filevars quit(-1) elif novar == 'warn': print warnmsg print ' ' + fname + ": file '" + ncfile + "' does not have " + \ "variable '" + var + "' !!" print ' variables in file:', filevars continue else: print errormsg print ' ' + fname + ": novar '" + novar + "' not ready !!" quit(-1) print ' ' + fname + ": getting variable '" + var + "'..." ovar = onc.variables[var] varinf = variable_inf(ovar) vardims = varinf.dimns newfiledims = list(onewnc.dimensions.keys()) for vdim in vardims: if not gen.searchInlist(newfiledims, vdim): print ' ' + fname + ": including dimension '" + vdim + "'..." if onc.dimensions[vdim].isunlimited(): newdim = onewnc.createDimension(vdim, None) else: newdim = onewnc.createDimension(vdim, len(onc.dimensions[vdim])) if varinf.FillValue is not None: newvar = onewnc.createVariable(var, nctype(varinf.dtype), tuple(vardims),\ fill_value = varinf.FillValue) else: newvar = onewnc.createVariable(var, nctype(varinf.dtype), tuple(vardims)) newvar[:] = ovar[:] for attrn in varinf.attributes: if attrn != '_FillValue': attrv = ovar.getncattr(attrn) newattr = set_attribute(newvar, attrn, attrv) onewnc.sync() # Global attributes for attrn in onc.ncattrs(): attrv = onc.getncattr(attrn) newattr = set_attribute(onewnc, attrn, attrv) onewnc.sync() onc.close() onewnc.close() print fname + ": successful written of '" + ofile + "' !!" def WRF_to_newCF(values, ncfile, variables): """ Function to pass a WRF original file to new CF-conventions file values= [wlonn]:[wlatn]:[wtimen]:[refDateTime]:[Tunits] [wlonn]: WRF longitude var name (it might come from WPS!) [wlatn]: WRF latitude var name (it might come from WPS!) [wtimen]: WRF time var name [refDateTime]: date of reference for times (in [YYYY][MM][DD][HH][MI][SS] format) [Tunits]: units for time variable ncfile= file to transform variables= ',' list of variables to transform ('all', for all variables) """ fname = 'WRF_to_newCF' if values == 'h': print fname + '_____________________________________________________________' print WRF_to_newCF.__doc__ quit() arguments = '[wlonn]:[wlatn]:[wtimen]:[refDateTime]:[Tunits]' gen.check_arguments(fname, values, arguments, ':') ofile = 'CF_WRF.nc' # New names for certain dimensions dimsrename = {'west_east': 'lon', 'south_north': 'lat', 'Time':'time'} # Variables to not be included varsNO = ['XLONG', 'XLONG_M', 'XLAT', 'XLAT_M', 'Times'] onewnc = NetCDFFile(ofile, 'w') wlonn = values.split(':')[0] wlatn = values.split(':')[1] wtimen = values.split(':')[2] refDateTime = values.split(':')[3] Tunits = values.split(':')[4] wrfnc = NetCDFFile(ncfile, 'r') if variables == 'all': varns = wrfnc.variables.keys() else: varns = gen.str_list(variables, ',') # lon, lat if not gen.searchInlist(wrfnc.variables.keys(),wlonn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not lon variable '" + \ wlonn + "' !!!" quit(-1) if not gen.searchInlist(wrfnc.variables.keys(),wlatn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not lat variable '" + \ wlatn + "' !!!" quit(-1) if not gen.searchInlist(wrfnc.dimensions.keys(),'west_east'): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not 'west_east' dimension !!!" quit(-1) if not gen.searchInlist(wrfnc.dimensions.keys(),'south_north'): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not 'south_north' dimension !!!" quit(-1) lonobj = wrfnc.variables[wlonn] lonv = lonobj[:] latobj = wrfnc.variables[wlatn] latv = latobj[:] if len(lonv.shape) == 3: cflons = np.zeros((lonv.shape[1],lonv.shape[2]), dtype=np.float) cflons = np.where(lonv[0,:,:] < 0., lonv[0,:,:] + 360., lonv[0,:,:]) cflats = np.zeros((latv.shape[1],latv.shape[2]), dtype=np.float) cflats = latv[0,:,:] dimx = lonv.shape[2] dimy = lonv.shape[1] elif len(lonv.shape) == 2: cflons = np.zeros((lonv.shape), dtype=np.float) cflons = np.where(lonv < 0., lonv + 360., lonv) cflats = latv dimx = lonv.shape[1] dimy = lonv.shape[0] elif len(lonv.shape) == 1: cflons = np.zeros((lonv.shape), dtype=np.float) cflons = np.where(lonv < 0., lonv + 360., lonv) cflats = latv dimx = lonv.shape[0] dimy = latv.shape[0] else: print errormsg print ' ' + fname + ": original longitude variable '" + varn + "' shape:" + \ lonv.shape + "not ready !!!" quit(-1) # Creation of CF-dimensions newdim = onewnc.createDimension('lon', dimx) newdim = onewnc.createDimension('lat', dimy) newdim = onewnc.createDimension('time', None) # dimension-variables newvar = onewnc.createVariable('lon','f8',('lat','lon')) newattr = basicvardef(newvar, 'longitude','Longitude','degrees_East') newvar[:] = cflons newattr = set_attribute(newvar, 'axis', 'X') newattr = set_attribute(newvar, '_CoordinateAxisType', 'Lon') newvar = onewnc.createVariable('lat','f8',('lat','lon')) newattr = basicvardef(newvar, 'latitude','Latitude','degrees_North') newvar[:] = cflats newattr = set_attribute(newvar, 'axis', 'Y') newattr = set_attribute(newvar, '_CoordinateAxisType', 'Lat') onewnc.sync() # CFing time if not gen.searchInlist(wrfnc.variables.keys(), wtimen): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not time variable '" + \ wtimen + "' !!!" quit(-1) if not gen.searchInlist(wrfnc.dimensions.keys(), 'Time'): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not 'Time' dimension !!!" quit(-1) timeobj = wrfnc.variables[wtimen] timev = timeobj[:] yrref=refDateTime[0:4] monref=refDateTime[4:6] dayref=refDateTime[6:8] horref=refDateTime[8:10] minref=refDateTime[10:12] secref=refDateTime[12:14] tunit = Tunits + ' since ' + yrref + '-' + monref + '-' + dayref + ' ' + \ horref + ':' + minref + ':' + secref # WRF file might already have the right time-variable if wtimen == 'Times': fdates = wrfnc.variables['Times'][:] cftimes = [] for it in range(fdates.shape[0]): wrfdates = gen.datetimeStr_conversion(fdates[it,:], 'WRFdatetime', 'matYmdHMS') cftimes.append(gen.realdatetime1_CFcompilant(wrfdates, refDateTime, Tunits)) else: itimeunits = timeobj.getncattr('units').replace('/','-') if itimeunits != tunit: cftimes = gen.coincident_CFtimes(timev, tunit, itimeunits) else: cftimes = list(timev) newvar = onewnc.createVariable('time','f8',('time')) newattr = basicvardef(newvar, 'time','Time',tunit) newvar[:] = cftimes newattr = set_attribute(newvar, 'axis', 'T') newattr = set_attribute(newvar, '_CoordinateAxisType', 'Time') newattr = set_attribute(newvar, 'calendar', 'gregorian') onewnc.sync() # Removing not desired variables for vrmn in varsNO: if gen.searchInlist(varns, vrmn): varns.remove(vrmn) # Looping on variables for vn in varns: newvars = onewnc.variables.keys() newdims = list(onewnc.dimensions) if not gen.searchInlist(newvars,vn): varobj = wrfnc.variables[vn] varattrs = gen.variables_values(vn) vvatrs = varobj.ncattrs() newvard = [] for dimn in varobj.dimensions: if dimsrename.has_key(dimn): newvard.append(dimsrename[dimn]) else: newvard.append(str(dimn)) if not gen.searchInlist(newdims, dimn): add_dims(wrfnc,onewnc,[str(dimn)]) cfn = varattrs[0] stn = varattrs[1] lonn = varattrs[4].replace('|',' ') un = varattrs[5].replace('|',' ') newvar = onewnc.createVariable(cfn, nctype(varobj.dtype), tuple(newvard)) newvar[:] = varobj[:] newattr = set_attribute(newvar, 'standard_name', stn) newattr = set_attribute(newvar, 'long_name', lonn) # Original attributes for attrn in vvatrs: attrv = varobj.getncattr(attrn) if attrn == 'units': if un != attrv: print warnmsg print ' ' + fname + ": WRF variable '" + vn + "' has different " + \ " units '" + attrv + "' than CF '" + un + "' !!" print ' keeping WRF ones' newattr = set_attribute(newvar, 'units', attrv) else: newattr = set_attribute(newvar, 'units', un) newattr = set_attribute(newvar, 'coordinates', 'lon lat') onewnc.sync() onewnc.close() wrfnc.close() print fname + ": succesfull creation of file '" + ofile + "' !!" return def WRF_toCF(values, ncfile): """ Function to pass a WRF original file to CF-conventions values= [wrfvl]:[wrfvL]:[refDateTime]:[Tunits] [wrfvl]: WRF longitude var name (it might come from WPS!) [wrfvL]: WRF latitude var name (it might come from WPS!) [refDateTime]: date of reference for times (in [YYYY][MM][DD][HH][MI][SS] format) [Tunits]: units for time variable ncfile= file to transform variables= variables to transform """ fname = 'WRF_toCF' if values == 'h': print fname + '_____________________________________________________________' print WRF_toCF.__doc__ quit() arguments = '[wrfvl]:[wrfvL]:[refDateTime]:[Tunits]' gen.check_arguments(fname, values, arguments, ':') ofile = 'CF_WRF.nc' wrfvl = values.split(':')[0] wrfvL = values.split(':')[1] refDateTime = values.split(':')[2] Tunits = values.split(':')[3] ncf = NetCDFFile(ncfile, 'r') # CFing lon,lat if not ncf.variables.has_key('lon'): ncf.close() WRF_CFlonlat_creation('lon,lat,true', ncfile, wrfvl+','+wrfvL) else: print warnmsg print ' ' + fname + ": file already with variable 'lon,lat' skipping them!!" ncf.close() ncf = NetCDFFile(ncfile,'a') ncdims = ncf.dimensions CFdims = {'west_east': 'lon', 'south_north': 'lat', 'Time': 'time'} # CFing dimensions for dimn in CFdims.keys(): if gen.searchInlist(ncdims,dimn): # Making sure CF dimension is not in the file if not gen.searchInlist(ncdims,CFdims[dimn]): newname = ncf.renameDimension(dimn,CFdims[dimn]) ncf.sync() ncf.close() # CFing time ncf = NetCDFFile(ncfile, 'r') if not ncf.variables.has_key('time'): ncf.close() WRF_CFtime_creation(refDateTime+','+Tunits, ncfile, 'time') else: print warnmsg print ' ' + fname + ": file already with variable 'time' skipping them!!" ncf.close() ncf = NetCDFFile(ncfile,'a') # dimension variables which do not need to attach CF attributes, otherwise values # retrieved from 'variable_values.dat' dimvars = ['lon', 'lat', 'time', 'Times'] for vn in ncf.variables.keys(): if not gen.searchInlist(dimvars,vn): varobj = ncf.variables[vn] varattrs = gen.variables_values(vn) stn = varattrs[1] lonn = varattrs[4].replace('|',' ') un = varattrs[5].replace('|',' ') newattr = set_attribute(varobj, 'standard_name', stn) newattr = set_attribute(varobj, 'long_name', lonn) newattr = set_attribute(varobj, 'units', un) newattr = set_attribute(varobj, 'coordinates', 'lon lat') ncf.sync() ncf.sync() ncf.close() print fname + ": succesfull CFication of WRF file '" + ncfile + "' !!" return def LMDZ_toCF(values,ncfile): """ Function to pass a LMDZ original file to CF-conventions values= [CFtimeUnits] [CFtimeUnits]: [tunits] since [YYYYMMDDHHMISS], CF time-units to which variable time should be used ('!' for spaces) ncfile= file to transform """ fname = 'LMDZ_toCF' if ncfile == 'h': print fname + '_____________________________________________________________' print LMDZ_toCF.__doc__ quit() CFT = values.replace('!',' ') ofile = 'CF_LMDZ.nc' ncf = NetCDFFile(ncfile,'a') ncdims = ncf.dimensions.keys() ncvars = ncf.variables.keys() # CFing variable 'pres' if gen.searchInlist(ncvars,'pres'): print warnmsg print fname + ": renaming variable 'pres' as 'p' !!" newname = ncf.renameVariable('pres','p') ncf.sync() # Re-aranging time-axis tvaro = ncf.variables['time_counter'] tvals = tvaro[:] tunits = str(tvaro.getncattr('units')) # Fixing 0000 year issue... Stunits = tunits.split(' ') Utunits = Stunits[0] if len(Stunits) == 4: datetunits = Stunits[2] + ' ' + Stunits[3] else: datetunits = Stunits[2] + ' 00:00:00' if int(datetunits[0:4]) == 0: print warnmsg newdatetunits = datetunits.replace('0000', '1900') tunits = Utunits + ' since ' + newdatetunits print ' ' + fname + ": Imposing a new units year for dates on '1900' !!" print ' new time units reference: ' + tunits RefDate = CFT.split(' ')[2] RefDateS = RefDate[0:4] + '-' + RefDate[4:6] + '-' + RefDate[6:8] + ' ' + \ RefDate[8:10] + ':' + RefDate[10:12] + ':' + RefDate[12:14] CFtimeUnits = ' '.join(CFT.split(' ')[0:2]) + ' '+ RefDateS newtvals = gen.coincident_CFtimes(tvals, CFtimeUnits, tunits) tvaro[:] = newtvals newattr = set_attribute(tvaro, 'units', CFtimeUnits) newattr = set_attribute(tvaro, 'time_origin', CFtimeUnits) ncf.sync() # CFing dimensions CFdims = {'time_counter': 'time', 'presnivs': 'pres'} for dimn in CFdims.keys(): if gen.searchInlist(ncdims,dimn): # Making sure CF dimension is not in the file if not gen.searchInlist(ncdims,CFdims[dimn]): newname = ncf.renameDimension(dimn,CFdims[dimn]) ncf.sync() ncvars = ncf.variables.keys() # Re-arranging coordinates attribute for all the variables for varn in ncvars: varobj = ncf.variables[varn] varattrs = varobj.ncattrs() if gen.searchInlist(varattrs, unicode('coordinates')): attrv = varobj.getncattr('coordinates') newcoord = str(attrv) for dimn in CFdims.keys(): newcoord = newcoord.replace(dimn, CFdims[dimn]) newattr = set_attribute(varobj, 'coordinates', newcoord) ncf.sync() ncf.close() print fname + ": succesfull CFication of LMDZ file '" + ncfile + "' !!" return def DYNAMICO_toCF(values,ncfile): """ Function to pass a DYNAMICO original file to CF-conventions values= [CFtimeUnits] [CFtimeUnits]: [tunits] since [YYYYMMDDHHMISS], CF time-units to which variable time should be used ('!' for spaces) ncfile= file to transform """ fname = 'DYNAMICO_toCF' if ncfile == 'h': print fname + '_____________________________________________________________' print DYNAMICO_toCF.__doc__ quit() CFT = values.replace('!',' ') ofile = 'CF_DYNAMICO.nc' ncf = NetCDFFile(ncfile,'a') ncdims = ncf.dimensions.keys() ncvars = ncf.variables.keys() # CFing variable 'pres' if gen.searchInlist(ncvars,'pres'): print warnmsg print fname + ": renaming variable 'pres' as 'p' !!" newname = ncf.renameVariable('pres','p') ncf.sync() # Re-aranging time-axis tvaro = ncf.variables['time_counter'] tvals = tvaro[:] tunits = tvaro.getncattr('units') RefDate = CFT.split(' ')[2] RefDateS = RefDate[0:4] + '-' + RefDate[4:6] + '-' + RefDate[6:8] + ' ' + \ RefDate[8:10] + ':' + RefDate[10:12] + ':' + RefDate[12:14] CFtimeUnits = ' '.join(CFT.split(' ')[0:2]) + ' '+ RefDateS newtvals = gen.coincident_CFtimes(tvals, CFtimeUnits, tunits) tvaro[:] = newtvals newattr = set_attribute(tvaro, 'units', CFtimeUnits) newattr = set_attribute(tvaro, 'time_origin', CFtimeUnits) ncf.sync() # CFing dimensions CFdims = {'time_counter': 'time', 'presnivs': 'pres'} for dimn in CFdims.keys(): if gen.searchInlist(ncdims,dimn): # Making sure CF dimension is not in the file if not gen.searchInlist(ncdims,CFdims[dimn]): newname = ncf.renameDimension(dimn,CFdims[dimn]) ncf.sync() ncvars = ncf.variables.keys() # Re-arranging coordinates attribute for all the variables for varn in ncvars: varobj = ncf.variables[varn] varattrs = varobj.ncattrs() if gen.searchInlist(varattrs, unicode('coordinates')): attrv = varobj.getncattr('coordinates') newcoord = str(attrv) for dimn in CFdims.keys(): newcoord = newcoord.replace(dimn, CFdims[dimn]) newattr = set_attribute(varobj, 'coordinates', newcoord) ncf.sync() ncf.close() print fname + ": succesfull CFication of DYNAMICO file '" + ncfile + "' !!" return def mthDYNAMICO_toCF(values,ncfile): """ Function to pass a mthDYNAMICO original file to CF-conventions values= [CFtimeUnits] [CFtimeUnits]: [tunits] since [YYYYMMDDHHMISS], CF time-units to which variable time should be used ('!' for spaces) ncfile= file to transform """ fname = 'mthDYNAMICO_toCF' if ncfile == 'h': print fname + '_____________________________________________________________' print mthDYNAMICO_toCF.__doc__ quit() CFT = values.replace('!',' ') ofile = 'CF_mthDYNAMICO.nc' ncf = NetCDFFile(ncfile,'a') ncdims = ncf.dimensions.keys() ncvars = ncf.variables.keys() # CFing variable 'pres' if gen.searchInlist(ncvars,'pres'): print warnmsg print fname + ": renaming variable 'pres' as 'p' !!" newname = ncf.renameVariable('pres','p') ncf.sync() # Re-aranging time-axis tvaro = ncf.variables['time_centered'] tvals = tvaro[:] tunits = tvaro.getncattr('units') RefDate = CFT.split(' ')[2] RefDateS = RefDate[0:4] + '-' + RefDate[4:6] + '-' + RefDate[6:8] + ' ' + \ RefDate[8:10] + ':' + RefDate[10:12] + ':' + RefDate[12:14] CFtimeUnits = ' '.join(CFT.split(' ')[0:2]) + ' '+ RefDateS newtvals = gen.coincident_CFtimes(tvals, CFtimeUnits, tunits) tvaro[:] = newtvals newattr = set_attribute(tvaro, 'units', CFtimeUnits) newattr = set_attribute(tvaro, 'time_origin', CFtimeUnits) ncf.sync() # CFing dimensions CFdims = {'time_counter': 'time', 'presnivs': 'pres'} for dimn in CFdims.keys(): if gen.searchInlist(ncdims,dimn): # Making sure CF dimension is not in the file if not gen.searchInlist(ncdims,CFdims[dimn]): newname = ncf.renameDimension(dimn,CFdims[dimn]) ncf.sync() # CFing variable 'time_centerd' if gen.searchInlist(ncvars,'time'): print warnmsg print fname + ": readjusting variable 'time' !!" otime = ncf.variables['time'] otime[:] = newtvals timeattrs = otime.ncattrs() chgattrs= ['standard_name','long_name','units', 'time_origin', 'cell_method'] for chga in chgattrs: if gen.searchInlist(timeattrs,chga): otime.delncattr(chga) newattr = set_attribute(otime, 'standard_name', 'time') newattr = set_attribute(otime, 'long_name', 'Time') newattr = set_attribute(otime, 'units', CFtimeUnits) newattr = set_attribute(otime, 'cell_method', 'time mean') else: newvar = ncf.createVariable('time', 'f8', ('time')) newvar[:] = newtvals newattr = set_attribute(newvar, 'standard_name', 'time') newattr = set_attribute(newvar, 'long_name', 'Time') newattr = set_attribute(newvar, 'units', CFtimeUnits) newattr = set_attribute(newvar, 'cell_method', 'time mean') ncf.sync() ncvars = ncf.variables.keys() # Re-arranging coordinates attribute for all the variables for varn in ncvars: varobj = ncf.variables[varn] varattrs = varobj.ncattrs() if gen.searchInlist(varattrs, unicode('coordinates')): attrv = varobj.getncattr('coordinates') newcoord = str(attrv) for dimn in CFdims.keys(): newcoord = newcoord.replace(dimn, CFdims[dimn]) newattr = set_attribute(varobj, 'coordinates', newcoord) ncf.sync() ncf.close() print fname + ": succesfull CFication of mthDYNAMICO file '" + ncfile + "' !!" return def CDO_toCF(ncfile): """ Function to pass a CDO output file to CF-conventions ncfile= file to transform variables= variables to transform """ fname = 'CDO_toCF' if ncfile == 'h': print fname + '_____________________________________________________________' print CDO_toCF.__doc__ quit() ncf = NetCDFFile(ncfile,'a') ncdims = ncf.dimensions.keys() ncvars = ncf.variables.keys() # renaming variables 'lon, lat' CFvardims = {'lon': 'lon2', 'lat': 'lat2'} for dimvarn in CFvardims.keys(): if gen.searchInlist(ncvars,dimvarn): newn = CFvardims[dimvarn] print warnmsg print fname + ": renaming variable '" + dimvarn + "' as '" + newn + "' !!" newname = ncf.renameVariable(dimvarn,newn) ncf.sync() # CFing dimensions CFdims = {'x': 'lon', 'y': 'lat'} for dimn in CFdims.keys(): if gen.searchInlist(ncdims,dimn): # Making sure CF dimension is not in the file if not gen.searchInlist(ncdims,CFdims[dimn]): newname = ncf.renameDimension(dimn,CFdims[dimn]) ncf.sync() ncvars = ncf.variables.keys() olon2 = ncf.variables['lon2'] dimslon2 = olon2.dimensions newvar = ncf.createVariable('lon','f8',dimslon2) newvar[:] = olon2[:] olonattr = olon2.ncattrs() for attrn in olonattr: attrv = olon2.getncattr(attrn) set_attribute(newvar,attrn,attrv) ncf.sync() ncf.sync() ncf.sync() ncf.sync() ncvars = ncf.variables.keys() olat2 = ncf.variables['lat2'] dimslat2 = olat2.dimensions newvar = ncf.createVariable('lat','f8',dimslat2) newvar[:] = olat2[:] olatattr = olat2.ncattrs() for attrn in olatattr: attrv = olat2.getncattr(attrn) set_attribute(newvar,attrn,attrv) ncvars = ncf.variables.keys() # Re-arranging coordinates attribute for all the variables for varn in ncvars: varobj = ncf.variables[varn] varattrs = varobj.ncattrs() if gen.searchInlist(varattrs, unicode('coordinates')): attrv = varobj.getncattr('coordinates') newcoord = str(attrv) for dimn in CFdims.keys(): newcoord = newcoord.replace(dimn, CFdims[dimn]) newattr = set_attribute(varobj, 'coordinates', newcoord) ncf.sync() # We should removing spurious lon2, lat2, but this would make too much... ncf.close() print fname + ": succesfull CFication of CDO file '" + ncfile + "' !!" return def cleaning_varsfile(values, filen): """ Function to keep a list of varibales from a file values= ',' separated list of variables to keep filen= name of the file """ fname = 'cleaning_varsfile' if values == 'h': print fname + '_____________________________________________________________' print WRF_toCF.__doc__ quit() arguments = '[values]' gen.check_arguments(fname, values, arguments, ':') keepvars = values.split(',') ofile = 'cleaning_varsfile.nc' # opening source file onc = NetCDFFile(filen, 'r') # opening new file newonc = NetCDFFile(ofile, 'w') oldvars = onc.variables.keys() notinfile = list(set(keepvars).difference(set(oldvars))) if len(notinfile) > 0: print warnmsg print ' ' + fname + ': variables to keep:', notinfile, "are not in file '"+ \ filen + "' !!" print ' variables to keep:', keepvars print ' variables in file:', oldvars for oldv in keepvars: newdims = newonc.dimensions newvars = newonc.variables.keys() if not gen.searchInlist(newvars, oldv) and gen.searchInlist(oldvars, oldv): print ' ' + fname + ": keeping variable '" + oldv + "' ..." oldov = onc.variables[oldv] oldvdim = oldov.dimensions vinf = variable_inf(oldov) for oldd in oldvdim: if not gen.searchInlist(newdims,oldd): odim = onc.dimensions[oldd] if odim.isunlimited(): newdim = newonc.createDimension(oldd, None) else: newdim = newonc.createDimension(oldd, len(odim)) if vinf.FillValue is not None: newvar = newonc.createVariable(oldv, nctype(vinf.dtype), vinf.dimns, \ fill_value=vinf.FillValue) else: newvar = newonc.createVariable(oldv, nctype(vinf.dtype), vinf.dimns) # Avoiding segmentation fault in Curie... if len(newvar.shape) > 1: if newvar.shape[0] != 0: for ix in range(newvar.shape[0]): newvar[ix,...] = oldov[ix,...] else: for ix in range(newvar.shape[len(newvar.shape)-1]): newvar[...,ix] = oldov[...,ix] else: newvar[:] = oldov[:] for attrn in vinf.attributes: if attrn != '_FillValue': attrv = oldov.getncattr(attrn) newattr = set_attribute(newvar,attrn,attrv) newonc.sync() newonc.sync() # Global attributes for gattrn in onc.ncattrs(): gattrval = onc.getncattr(gattrn) newattr = set_attribute(newonc, gattrn, gattrval) newonc.sync() newonc.close() onc.close() sout = sub.call('mv ' + ofile + ' ' + filen, shell=True) print fname + ": succesfully cleaned '" + filen + "' !!" return def time_reset(values,ncfile,variable): """ Function to re-set the time axis of a file values= [kind],[value] [kind]: kind of time re-set: 'setorigin': make time-steps start at a given date [value]=[YYYYMMDDHHMISS] ncfile: netcdf file to re-set times variable: name of the variable time within the file """ import datetime as dt fname = 'time_reset' OpReady = ['setorigin'] if values == 'h': print fname + '_____________________________________________________________' print time_reset.__doc__ quit() arguments = '[kind],[value]' gen.check_arguments(fname, values, arguments, ',') kind = values.split(',')[0] value = values.split(',')[1] ncf = NetCDFFile(ncfile,'a') if not ncf.variables.has_key(variable): print errormsg print ' ' + fname + ": file '" +ncfile+ "' does not have time variable: '"+ \ variable + "' !!" quit(-1) otvar = ncf.variables[variable] tvals = otvar[:] dimt = tvals.shape[0] tunits = otvar.getncattr('units') tattrs = otvar.ncattrs() if kind == 'setorigin': RefDate = value RefDateS = RefDate[0:4] + '-' + RefDate[4:6] + '-' + RefDate[6:8] + ' ' + \ RefDate[8:10] + ':' + RefDate[10:12] + ':' + RefDate[12:14] dt = tvals[1] - tvals[0] newtvals = np.arange(dimt)*dt otvar[:] = newtvals newtunits = ' '.join(tunits.split(' ')[0:2]) + ' ' + RefDateS newattr = set_attribute(otvar, 'units', newtunits) if gen.searchInlist(tattrs, 'time_origin'): newattr = set_attribute(otvar, 'time_origin', newtunits) else: print errormsg print ' ' + fname + ": kind '" + kind + "' not ready !!" print ' Operations ready:', OpReady quit(-1) ncf.sync() ncf.close() return def getvars_tofile(values,ncfile,varname): """ Function to get variables from a file and pass them to another one values= [getfile] getfile: name of the file from which variables have to be taken ncfile= name of the file to receive the variables varname= ',' list of variables to get ('all', for all variables) NOTE: all coincident dimensions, variables will not be taken """ fname = 'getvars_tofile' if values == 'h': print fname + '_____________________________________________________________' print getvars_tofile.__doc__ quit() #arguments = '[getfile]' #gen.check_arguments(fname, values, arguments, ',') #getfile = values.split(',')[0] getfile = values if not os.path.isfile(getfile): print errormsg print ' ' + fname + ": file from which get variables '" + getfile + \ "' does not exist !!" quit(-1) ionc = NetCDFFile(getfile, 'r') ronc = NetCDFFile(ncfile, 'a') # Getting list of variables to get ivarns = ionc.variables.keys() if varname == 'all': varns = list(ivarns) else: varns = varname.split(',') print ' ' + fname + ": getting variables from file '" + getfile + "' to '" + \ ncfile + ' _______' rvarns = ronc.variables.keys() for vn in varns: rdimns = list(ronc.dimensions) if not gen.searchInlist(ivarns,vn): print errormsg print ' ' + fname + ": file '" + getfile + "' does not have variable '"+\ vn + "' !!" print ' available variables:', ivarns quit(-1) if not gen.searchInlist(rvarns,vn): print ' ' + fname + ": adding variable '" + vn + "' ..." iovar = ionc.variables[vn] vardims = list(iovar.dimensions) for vd in vardims: if not gen.searchInlist(rdimns,vd): ovd = ionc.dimensions[vd] if ovd.isunlimited(): dsize = None else: dsize = len(ovd) print ' ' + fname + ": adding dimension '" + vd + "' (", \ dsize, ') ...' ronc.createDimension(vd,dsize) varvals = iovar[:] vartype = iovar.dtype varattrs = iovar.ncattrs() if gen.searchInlist(varattrs, '_FillValue'): fillval = iovar.getncattr('_FillValue') newvar = ronc.createVariable(vn, vartype, vardims, fill_value=fillval) varattrs.remove('_FillValue') else: newvar = ronc.createVariable(vn, vartype, vardims) newvar[:] = varvals[:] for attrn in varattrs: attrv = iovar.getncattr(attrn) newattr = set_attribute(newvar,attrn,attrv) ronc.sync() ionc.close() # Global values newattr = set_attribute(ronc, 'PyNCplot', 'http://www.xn--llusfb-5va.cat/python/PyNCplot/') newattr = set_attribute(ronc, 'script', fname) newattr = set_attribute(ronc, 'get_file', getfile) newattr = set_attribute(ronc, 'get_variables', varname) ronc.sync() ronc.close() return def addDim(values, ncfile, dimn): """ Operation to add a new dimension in a file values= [size] [size]: size of the dimension ('unlimited' for unlimited) ncfile= name of the file dimn= name of the new dimension """ fname = 'addDim' if values == 'h': print fname + '_____________________________________________________________' print addDim.__doc__ quit() onc = NetCDFFile(ncfile, 'a') if onc.dimensions.has_key(dimn): print errormsg print ' ' + fname + ": file '" + ncfile + "' already has dimension '" + \ dimn + "' !!" onc.close() quit(-1) if values == 'unlimited': Ldim = None else: Ldim = int(values) newdim = onc.createDimension(dimn, Ldim) onc.sync() onc.close() print fname + ": succesful addition of dimension '" + dimn + "' size:", Ldim,'!!' return def addVar(values, ncfile, varn): """ Operation to add a new variable in a file using exisintg dimensions values= [dnames]|[varattributes]|[kind] [dnames]: [dimn1],...,[dimnN], ',' list of name of dimensions for the new variable (already existing) 'scalar', for scalar variables [attributes]: [std_name]@[long_name]@[units], standard name, long name and units of the variable ('!' for spaces) 'None' for no standard attributes [kind]: type of variable (standard netCDF4/C-like values, 'c', 'i', 'f', 'f8',...) ncfile= name of the file varn= name of the new variable """ fname = 'addVar' if values == 'h': print fname + '_____________________________________________________________' print addVar.__doc__ quit() expectargs = '[dimensions]|[varattributes]|[kind]' gen.check_arguments(fname,values,expectargs,'|') dnames = gen.str_list(values.split('|')[0],',') attributes = values.split('|')[1].replace('!',' ') kind = values.split('|')[2] onc = NetCDFFile(ncfile, 'a') dimns = onc.dimensions varns = onc.variables.keys() # Checking new varname if gen.searchInlist(varns, varn): print errormsg print ' ' +fname+ ": file '" + ncfile + "' already has variable '" + varn + \ "' !!" print ' existing ones:', varns quit(-1) if values.split('|')[0] == 'scalar': print ' ' + fname + ': creation of scalar variable !!' # Variable if kind == 'c': newvar = onc.createVariable(varn, 'c') # newvar[:] = np.zeros(tuple(dsize), dtype=np.float) elif kind == 'f' or kind == 'f4': newvar = onc.createVariable(varn, 'f4') newvar[:] = np.float(0.) elif kind == 'f8': newvar = onc.createVariable(varn, 'f8') newvar[:] = np.float64(0.) elif kind == 'i': newvar = onc.createVariable(varn, 'i') newvar[:] = int(0) else: print errormsg print ' ' + fname + ": variable kind '" + kind + "' not ready!!" quit(-1) elif type(dnames) == type([1,2]): # Checking dimensions dsize = [] for dimn in dnames: if not gen.searchInlist(dimns, dimn): print errormsg print ' ' +fname+ ": file '" + ncfile + "' does not have " + \ "dimension '" + dimn + "' !!" print ' existing ones:', dimns quit(-1) else: dsize.append(len(onc.dimensions[dimn])) # Variable if kind == 'c': newvar = onc.createVariable(varn, 'c', tuple(dnames)) # newvar[:] = np.zeros(tuple(dsize), dtype=np.float) elif kind == 'f' or kind == 'f4': newvar = onc.createVariable(varn, 'f4', tuple(dnames), \ fill_value=gen.fillValueF) newvar[:] = np.zeros(tuple(dsize), dtype=np.float) elif kind == 'f8': newvar = onc.createVariable(varn, 'f8', tuple(dnames), \ fill_value=gen.fillValueD) newvar[:] = np.zeros(tuple(dsize), dtype=np.float64) elif kind == 'i': newvar = onc.createVariable(varn, 'i', tuple(dnames), \ fill_value=gen.fillValueI) newvar[:] = np.zeros(tuple(dsize), dtype=int) else: print errormsg print ' ' + fname + ": variable kind '" + kind + "' not ready!!" quit(-1) else: print errormsg print ' ' + fname + ": wrong dimension values:", dnames, " !!" quit(-1) # Setting standard attributes if values.split('|')[1] != 'None': sname = attributes.split('@')[0] lname = attributes.split('@')[1] u = attributes.split('@')[2] newattr = basicvardef(newvar, sname, lname, u) onc.sync() onc.close() return def setvar_nc(values, ncfile, varn): """ Operation to set values of a variable from values of an other one values= [infile],[invar] [infile]: name of the file with the reference variable [invar]: name of the variable in [infile] [dims]: '|' list of [dimn];[dimval] dimension and value to define slice of values to use from [invar] * [integer]: which value of the dimension * -1: all along the dimension * -9: last value of the dimension * [beg]:[end]:[frq] slice from [beg] to [end] every [frq] * NOTE, no dim name all the dimension size ncfile= name of the file with the variable to fill varn= name of the variable to fill """ fname = 'setvar_nc' if values == 'h': print fname + '_____________________________________________________________' print setvar_nc.__doc__ quit() expectargs = '[infile],[invar],[dims]' gen.check_arguments(fname,values,expectargs,',') infile = values.split(',')[0] invar = values.split(',')[1] dims = values.split(',')[2].split('|') oinc = NetCDFFile(infile, 'r') onc = NetCDFFile(ncfile, 'a') if not onc.variables.has_key(varn): print errormsg print ' '+fname+": file '" + ncfile + "' does not have variable '" + varn + \ "' to be filled !!" print ' existing ones:', onc.variables.keys() quit(-1) if not oinc.variables.has_key(invar): print errormsg print ' '+fname+": file '" + infile + "' does not have variable '" + invar +\ "' to use for filling !!" print ' existing ones:', oinc.variables.keys() quit(-1) oinvar = oinc.variables[invar] ovar = onc.variables[varn] # Slicing input varibale slicedict = {} for dimns in dims: dimn = dimns.split(';')[0] dimv = dimns.split(';')[1] if dimv.find(':') != -1: slicedict[dimn] = np.array(dimv.split(':'), dtype=int) else: slicedict[dimn] = int(dimv) invarslice, invardims = SliceVarDict(oinvar, slicedict) invals = oinvar[tuple(invarslice)] # Checking same shapes gen.same_shape(invals, ovar[:]) ovar[:] = invals[:] oinc.close() onc.sync() onc.close() return def rm_FillValue(values, ncfile, varn): """ Operation to remove the _FillValue from a variable inside a netCDF file values= [newvalue] [newvalue]: value to use to replace '_FillValue' ncfile= name of the file with the variable to fill varn= name of the variable to fill """ import numpy.ma as ma fname = 'rm_FillValue' if values == 'h': print fname + '_____________________________________________________________' print rm_FillValue.__doc__ quit() # expectargs = '[newvalue]' # gen.check_arguments(fname,values,expectargs,',') newvalue = values onc = NetCDFFile(ncfile, 'a') if not onc.variables.has_key(varn): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have variable '" + \ varn + "' !!" print ' available ones:'. onc.variables.keys() quit(-1) ovar = onc.variables[varn] varattrs = ovar.ncattrs() # Checking for _FillValue if not gen.searchInlist(varattrs, '_FillValue'): print warnmsg print ' ' + fname + ": varibale '" + varn + "' does not have _FillValue " + \ "attribute !!" print ' attributes:', varattrs print " Use `nc_var.py -o valmod -f " + ncfile + " -S '[...]' -v " + \ varn + " instead !!" quit(-1) varattrs.remove('_FillValue') maskvarvals = ovar[:] newvals = maskvarvals.filled(gen.retype(newvalue,ovar.dtype)) attr = ovar.delncattr('_FillValue') ovar[:] = newvals[:] onc.sync() onc.close() return def interp_curve(ov, xdn, ydn, curvpos, wgts, Nwgt, ijcurv): """ Function to interpolate (as weighted mean with 4-closest ones) a variable following a curve using a 3x3 matrix of weights ov= netcdf object to interpolate xdn= x-dimension name of the curve ydn= y-dimension name of the curve curvpos= position of the curve within the grid wgts= matrix with the weights to use to interpolate Nwgt= number of useful weights (<=4) ijcurv= j,i indices of the distance-sorted grid points from 3x3 """ fname = 'interp_curve' dimvar = ov.dimensions Ncurv = curvpos.shape[0] dictslice = {} for dn in ov.dimensions: if dn == xdn or dn == ydn: ds = 0 else: ds = -1 dictslice[dn] = ds varslice, dimrefvals = SliceVarDict(ov, dictslice) varref = ov[varslice] dimref = list(varref.shape) curvevar = np.zeros(tuple(dimref+[Ncurv]), dtype=np.float) for icv in range(Ncurv): # Getting all `useful' weights ix = int(curvpos[icv,1]) iy = int(curvpos[icv,0]) sortedwgt = list(wgts[icv,:,:].flatten()) sortedwgt.sort(reverse=True) for isp in range(Nwgt[icv]): dictslice[ydn] = iy + int(ijcurv[icv,0,isp]) dictslice[xdn] = ix + int(ijcurv[icv,1,isp]) varslice, dimvarvals = SliceVarDict(ov, dictslice) iwgt = sortedwgt[isp] vaL = ov[tuple(varslice)]*iwgt # Is the variable masked? if type(ov[:]) == type(gen.mamat): curvevar[...,icv] = np.where(ov[:].mask[tuple(varslice)], \ gen.fillValueF, curvevar[...,icv] + vaL) else: curvevar[...,icv] = curvevar[...,icv] + ov[tuple(varslice)]*iwgt return curvevar def curve_section(values,ncfile,varn): """ Function to provide a section of a file following a given curve values= [kindcurve]: kind of curve to follow 'gridline',[xdimn],[ydimn],[iBL],[jBL],[iUR],[jUR],[Ncurve]: line from starting `bottom-left' grid point [iBL],[jBL] to `right-up' grid point [iUR],[jUR] and [Ncurve] number of line-points with [xdimn],[ydimn] name of the x and y dimensions 'lonlatline',[londimn]|[lonvarname],[latdimn]|[latvarname],[lonSW],[latSW],[lonNE],[latNE],[Ncurve]: line from starting `SW' lonlat point [lonSW],[latSW] to `NE' lonlat point [lonNE],[latNE] and [Ncurve] line-points with '[lon/latdimn]|[lon/latvarn]' name of the dimension and variable-dimension of longitude and latitude 'gridfilecurve',[xdimn],[ydimn],[curvefile]: curve following grid-values in the file [curvefile](x y; two columns space separated, '#' commment) with [xdimn],[ydimn] name of the x and y dimensions 'lonlatfilecurve',[londimn]|[lonvarname],[latdimn]|[latvarname],[curvefile]: curve following lonlat-values in the file [curvefile](lon lat; two columns space separated, '#' commment) with '[lon/latdimn]|[lon/latvarn]' name of the dimension and variable-dimension of longitude and latitude ncfile= name of the file with the variable to fill varn= name of the variable to get ('all', for all variables) """ fname = 'curve_section' if values == 'h': print fname + '_____________________________________________________________' print curve_section.__doc__ quit() availcurves = ['gridline', 'lonlatline', 'gridfilecurve', 'lonlatfilecurve'] if values.split(',')[0] == 'gridline': expectargs = 'gridline,[xdimn],[ydimn],[iBL],[jBL],[iUR],[jUR],[Ncurve]' elif values.split(',')[0] == 'lonlatline': expectargs = 'lonlatline,[londimn]|[lonvarname],[latdimn]|[latvarname],' + \ '[iSW],[jSW],[iNE],[jNE],[Ncurve]' elif values.split(',')[0] == 'gridfilecurve': expectargs = 'gridfilecurve,[xdimn],[ydimn],[curvefile]' elif values.split(',')[0] == 'lonlatfilecurve': expectargs = 'lonlatfilecurve,[londimn]|[lonvarname],[latdimn]|' + \ '[latvarname],[curvefile]' else: print errormsg print ' ' + fname + ": kind of curve: '" + values.split(',')[0] + \ "' not ready !!" print ' available ones:', availcurves quit(-1) gen.check_arguments(fname,values,expectargs,',') ofile = 'curve_section.nc' onc = NetCDFFile(ncfile, 'r') if varn == 'all': varns = onc.variables.keys() else: varns = [varn] # Getting curve if values.split(',')[0] == 'gridline': # 'gridline,[xdimn],[ydimn],[iBL],[jBL],[iUR],[jUR],[Ncurve]' xdimn = values.split(',')[1] ydimn = values.split(',')[2] iBL = np.float(values.split(',')[3]) jBL = np.float(values.split(',')[4]) iUR = np.float(values.split(',')[5]) jUR = np.float(values.split(',')[6]) Ncurve = int(values.split(',')[7]) Dxcurve = (iUR-iBL)/(Ncurve-1.) Dycurve = (jUR-jBL)/(Ncurve-1.) curve = np.zeros((Ncurve,2), dtype=np.float) for icv in range(Ncurve): curve[icv,0] = jBL + Dycurve*icv curve[icv,1] = iBL + Dxcurve*icv elif values.split(',')[0] == 'lonlatline': # 'lonlatline,[londimn]|[lonvarname],[latdimn]|[latvarname],[iSW],[jSW],[iNE],[jNE],[Ncurve]' Lcurve = 'lon/lat' xdimn = values.split(',')[1].split('|')[0] xvardimn = values.split(',')[1].split('|')[1] ydimn = values.split(',')[2].split('|')[0] yvardimn = values.split(',')[2].split('|')[1] lonSW = np.float(values.split(',')[3]) latSW = np.float(values.split(',')[4]) lonNE = np.float(values.split(',')[5]) latNE = np.float(values.split(',')[6]) Ncurve = int(values.split(',')[7]) Dxcurve = (lonNE-lonSW)/(Ncurve-1.) Dycurve = (latNE-latSW)/(Ncurve-1.) curve = np.zeros((Ncurve,2), dtype=np.float) for icv in range(Ncurve): curve[icv,0] = latSW + Dycurve*icv curve[icv,1] = lonSW + Dxcurve*icv elif values.split(',')[0] == 'gridfilecurve': # 'gridfilecurve,[xdimn],[ydimn],[curvefile]' xdimn = values.split(',')[1] ydimn = values.split(',')[2] curvefile = values.split(',')[3] Scurve = [] Ncurve = 0 ocurve = open(curvefile, 'r') for line in ocurve: if line[0:1] != '#': Scurve.append(line.split(' ')) Ncurve = Ncurve + 1 ocurve.close() curve = np.zeros((Ncurve,2), dtype=np.float) for icv in range(Ncurve): curv = Scurve[icv] curve[icv,0] = np.float(curv[1]) curve[icv,1] = np.float(curv[0]) elif values.split(',')[0] == 'lonlatfilecurve': # 'lonlatfilecurve,[londimn]|[lonvarname],[latdimn]|[latvarname],[curvefile]' xdimn = values.split(',')[1].split('|')[0] xvardimn = values.split(',')[1].split('|')[1] ydimn = values.split(',')[2].split('|')[0] yvardimn = values.split(',')[2].split('|')[1] curvefile = values.split(',')[3] Scurve = [] Ncurve = 0 ocurve = open(curvefile, 'r') for line in ocurve: if line[0:1] != '#': Scurve.append(line.split(' ')) Ncurve = Ncurve + 1 ocurve.close() curve = np.zeros((Ncurve,2), dtype=np.float) for icv in range(Ncurve): curv = Scurve[icv] curve[icv,0] = np.float(curv[1]) curve[icv,1] = np.float(curv[0]) # Getting dimension-variables onc = NetCDFFile(ncfile, 'r') if not gen.searchInlist(onc.dimensions, xdimn): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have dimension '" + \ xdimn + "' !!" print ' available dimensions:', onc.dimensions.keys() onc.close() quit(-1) if not gen.searchInlist(onc.dimensions, ydimn): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have dimension '" + \ ydimn + "' !!" print ' available dimensions:', onc.dimensions.keys() onc.close() quit(-1) dimx = len(onc.dimensions[xdimn]) dimy = len(onc.dimensions[ydimn]) if values.split(',')[0]== 'lonlatline' or values.split(',')[0]=='lonlatfilecurve': if not onc.variables.has_key(xvardimn): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have variable-" + \ "dimension '" + xvardimn + "' !!" print ' available variables:', onc.variables.keys() onc.close() quit(-1) if not onc.variables.has_key(yvardimn): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have variable-" + \ "dimension '" + yvardimn + "' !!" print ' available variables:', onc.variables.keys() onc.close() quit(-1) ovar = onc.variables[xvardimn] xvals0 = ovar[:] Lcurvex = xvardimn cxunit = ovar.units ovar = onc.variables[yvardimn] yvals0 = ovar[:] Lcurvey = yvardimn cyunit = ovar.units xvals, yvals = gen.lonlat2D(xvals0, yvals0) else: x1vals = np.arange((dimx), dtype=np.float) y1vals = np.arange((dimy), dtype=np.float) xvals, yvals = gen.lonlat2D(x1vals, y1vals) Lcurvex = 'grid' Lcurvey = 'grid' cxunit = '1' cyunit = '1' # Checking consistency with values if values.split(',')[0]== 'gridline': if iBL >= dimx or iUR >= dimx: print errormsg print ' ' + fname + ': wrong iBL or iUR !!' print ' x-curve range must lay between 0,',dimx-1 print ' provided values iBL:', iBL, 'iUR:', iUR print ' line must lay within 0, 0 x ',dimx-1,',',dimy-1 print ' values given: ', iBL,',',jBL,'x',iUR,',',jUR quit(-1) if jBL >= dimy or jUR >= dimy: print errormsg print ' ' + fname + ': wrong jBL or jUR !!' print ' x-curve range must lay between 0,',dimy-1 print ' provided values jBL:', jBL, 'jUR:', jUR print ' line must lay within 0, 0 x ',dimx-1,',',dimy-1 print ' values given: ', iBL,',',jBL,'x',iUR,',',jUR quit(-1) # Getting curve-values loccurve, curvewgts, Nwgts, curveloc3x3, exctloccurve = gen.curvelocalize_2D( \ curve,xvals,yvals) # Creation of file newnc = NetCDFFile(ofile, 'w') # Dimensions newdim = newnc.createDimension('curve', Ncurve) newdim = newnc.createDimension('box', 3) newdim = newnc.createDimension('pt', 2) newdim = newnc.createDimension('boxbox', 9) # Var-dimensions newvar = newnc.createVariable('xcurve', 'f8', ('curve')) newvar[:] = curve[:,1] basicvardef(newvar,'xcurve', 'x-' +Lcurvex+ ' localization of the curve', cxunit) newvar = newnc.createVariable('ycurve', 'f8', ('curve')) newvar[:] = curve[:,0] basicvardef(newvar,'ycurve', 'y-' +Lcurvey+ ' localization of the curve', cyunit) # variable curvetype curvetype = values.split(',')[0] newvar = newnc.createVariable(curvetype, 'c') if curvetype == 'gridline': desc = "line from starting `bottom-left` grid point [iBL],[jBL] to " + \ "`right-up` grid point [iUR],[jUR] and [Ncurve] number of line-points " + \ "with [xdimn],[ydimn] name of the x and y dimensions" set_attributek(newvar,'description',desc,'S') set_attributek(newvar,'xdimn',xdimn,'S') set_attributek(newvar,'ydimn',ydimn,'S') set_attributek(newvar,'iBL',iBL,'I') set_attributek(newvar,'jBL',jBL,'I') set_attributek(newvar,'iUR',iUR,'I') set_attributek(newvar,'jUR',jUR,'I') set_attributek(newvar,'Ncurve',Ncurve,'I') elif curvetype == 'lonlatline': desc = "line from starting `SW` lonlat point [lonSW],[latSW] to `NE` " + \ "lonlat point [lonNE],[latNE] and [Ncurve] line-points with " + \ "`[lon/latdimn]|[lon/latvarn]` name of the dimension and " + \ "variable-dimension of longitude and latitude" set_attributek(newvar,'description',desc,'S') set_attributek(newvar,'londimn',xdimn,'S') set_attributek(newvar,'lonvarname',xvardimn,'S') set_attributek(newvar,'latdimn',ydimn,'S') set_attributek(newvar,'latvarname',yvardimn,'S') set_attributek(newvar,'lonSW',lonSW,'R') set_attributek(newvar,'latSW',latSW,'R') set_attributek(newvar,'lonNE',lonNE,'R') set_attributek(newvar,'latNE',latNE,'R') set_attributek(newvar,'Ncurve',Ncurve,'I') elif curvetype == 'gridfilecurve': desc = "curve following grid-values in the file [curvefile](x y; two " + \ "columns space separated, `#` commment) with [xdimn],[ydimn] name of " + \ "the x and y dimensions" set_attributek(newvar,'description',desc,'S') set_attributek(newvar,'xdimn',xdimn,'S') set_attributek(newvar,'ydimn',ydimn,'S') set_attributek(newvar,'curvefile',curvefile,'S') elif curvetype == 'lonlatfilecurve': desc = "curve following lonlat-values in the file [curvefile](lon lat; " + \ "two columns space separated, `#` commment) with `[lon/latdimn]|" + \ "[lon/latvarn]` name of the dimension and variable-dimension of " + \ "longitude and latitude" set_attributek(newvar,'description',desc,'S') set_attributek(newvar,'londimn',xdimn,'S') set_attributek(newvar,'lonvarname',xvardimn,'S') set_attributek(newvar,'latdimn',ydimn,'S') set_attributek(newvar,'latvarname',yvardimn,'S') set_attributek(newvar,'curvefile',curvefile,'S') # Localization curve within file newvar = newnc.createVariable('icurve', 'f8', ('curve')) newvar[:] = exctloccurve[:,1] basicvardef(newvar,'icurve', 'x-axis localization of the curve', '1') newvar = newnc.createVariable('jcurve', 'f8', ('curve')) newvar[:] = exctloccurve[:,0] basicvardef(newvar,'jcurve', 'y-axis localization of the curve', '1') # variable weight newvar = newnc.createVariable('weights', 'f', ('box','box','curve')) for icv in range(Ncurve): newvar[:,:,icv] = curvewgts[icv,:,:] basicvardef(newvar,'weights', 'weights on the 3x3 around box following curve','1') # variable Nweigth newvar = newnc.createVariable('Nweights', 'f', ('curve')) newvar[:] = Nwgts[:] basicvardef(newvar,'Nweights', 'number of usable weights of the 3x3 around box '+\ 'following curve', '-') # variable coordweights newvar = newnc.createVariable('coordweights', 'f', ('pt','boxbox','curve')) for icv in range(Ncurve): newvar[:,:,icv] = curveloc3x3[icv,:,:] basicvardef(newvar,'coordweights', 'coordinates of the usable weights of the ' + \ '3x3 around box following curve', '-') # File variables to get the section for vn in varns: if not onc.variables.has_key(vn): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have variable '" +\ vn + "' !!" print ' available ones:', onc.variables.keys() onc.close() quit(-1) ovar = onc.variables[vn] if len(ovar.shape) == 0: print warnmsg print ' ' +fname+ ": scalar variable '" + vn + "' section impossible !!" newvar = newnc.createVariable(vn, ovar.dtype) newvar[:] = ovar[:] add_varattrs(onc,newnc,[vn],[vn]) else: print ' ' + fname + ": adding section of '" + vn + "' ..." dimvar = list(ovar.dimensions) # Look if variable has the dimension along the curve foundxdim = False foundydim = False if gen.searchInlist(dimvar, xdimn): foundxdim = True if gen.searchInlist(dimvar, ydimn): foundydim = True if foundxdim and foundydim: dimvar.remove(xdimn) dimvar.remove(ydimn) varcurve = interp_curve(ovar, xdimn, ydimn, loccurve, curvewgts, \ Nwgts, curveloc3x3) # Writing values of variable along the curve add_dims(onc,newnc,dimvar) newvar = newnc.createVariable(vn+'curve', 'f4', tuple(dimvar+['curve'])) newvar[:] = varcurve[:] add_varattrs(onc,newnc,[vn],[vn+'curve']) set_attribute(newvar,'along','xcurve,ycurve') set_attribute(newvar,'curve',curvetype) newnc.sync() else: print warnmsg print ' ' + fname + ": variable '" + vn + "' does not have curve "+ \ " dimensions '", xdimn, "' and '", ydimn, "' !!" print ' keeping as it is in output file' # Writing values of variable along the curve add_dims(onc,newnc,dimvar) newvar = newnc.createVariable(vn, nctype(ovar.dtype), tuple(dimvar)) print ' Lluis shapes newvar:', newvar.shape, 'ovar:', ovar.shape newvar[:] = ovar[:] newnc.sync() # Global attributes add_globattrs(onc,newnc,'all') newnc.setncattr('author', 'L. Fita') newattr = set_attributek(newnc, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') newnc.setncattr('university', 'Pierre Marie Curie - Jussieu') newnc.setncattr('center', 'Centre National de Recherches Scientifiques') newnc.setncattr('city', 'Paris') newnc.setncattr('country', 'France') newnc.setncattr('source', 'http://www.xn--llusfb-5va.cat/python/PyNCplot/') newnc.setncattr('script', 'nc_var_tools.py') newnc.setncattr('function', fname) print fname + ": successfull writing of '" + ofile + "' !!" return def slice_createArray(listslice, typev=np.float): """ Function to create a variable from a list which contains slicing-variable values listslice= list with ordered slice insrtuctions to be used to slice a variable typev= type of the array to create >>> slice_createArray([1, slice(0,3,None)], typev=np.float) [ 0. 0. 0.] >>> slice_createArray([slice(0,20,3), slice(0,3,None)], typev=type(True)) [[False False False] [False False False] [False False False] [False False False] [False False False] [False False False]] """ fname = 'slice_createArray' dimsize = [] for slc in listslice: if type(slc) == type(slice(0,1,None)): iv = slc.start ev = slc.stop sv = slc.step if sv is not None: Lslc = (ev - iv)/sv else: Lslc = ev - iv elif type(slc) == int or type(slc) == np.int64: Lslc = 0 if Lslc != 0: dimsize.append(Lslc) if typev == int: arrayvalues = np.zeros(tuple(dimsize), dtype=int) if typev == np.int32: arrayvalues = np.zeros(tuple(dimsize), dtype=np.int32) elif typev == np.float: arrayvalues = np.zeros(tuple(dimsize), dtype=np.float) elif typev == np.float32: arrayvalues = np.zeros(tuple(dimsize), dtype=np.float32) elif typev == np.float64: arrayvalues = np.zeros(tuple(dimsize), dtype=np.float64) elif typev == type(True): arrayvalues = np.zeros(tuple(dimsize), dtype=bool) else: print errormsg print ' ' + fname + ': type', typev, 'not ready !!' quit(-1) return arrayvalues def get_varunits(ovar): """ Getting the units of a variable object ovar: object variable from which get the units """ fname = 'get_varunits' varattrs = ovar.ncattrs() if gen.searchInlist(varattrs,'units'): varunits = ovar.units else: varunits = gen.variables_values(ovar.name)[5] return varunits def lonlat_polygon(values,ncfile,varn): """ Function to define a lon/lat region giving the coordinates of the vertexs of a given polygon lonlat_polygon(values,ncfile,varn) values= [asciipolygfile] [lon1] [lat1] (...) [lonN] [latN] ncfile= name of the file with the variable to fill varn= [lonvname],[latvname] name of the longitude and latitude variable in the file """ import matplotlib.path as mpltPath fname = 'lonlat_polygon' ofile = fname + '.nc' if values == 'h': print fname + '_____________________________________________________________' print lonlat_polygon.__doc__ quit() asciipolygfile = values lonvname = varn.split(',')[0] latvname = varn.split(',')[1] if not os.path.isfile(asciipolygfile): print errormsg print ' ' + fname + ": ASCII vertex polygon file '" + asciipolygfile + \ "' does not exist !!" quit(-1) # getting lon,lat onc = NetCDFFile(ncfile, 'r') if not onc.variables.has_key(lonvname): print errormsg print ' ' + fname + ": netCDF file '" + ncfile + "' does not have variable"+\ " longitude '" + lonvname + "' !!" quit(-1) if not onc.variables.has_key(latvname): print errormsg print ' ' + fname + ": netCDF file '" + ncfile + "' does not have variable"+\ " latitude '" + latvname + "' !!" quit(-1) lon, lat = gen.lonlat2D(onc.variables[lonvname], onc.variables[latvname]) onc.close() # getting vertices overtf = open(asciipolygfile, 'r') lLvertices = [] for line in overtf: if line[0:1] != '#' and len(line) > 3: linevals = line.replace('\n','').split(' ') lLvertices.append([np.float(linevals[0]), np.float(linevals[1])]) overtf.close() # Getting ij in lon,lat space of the vertices Nvertices = len(lLvertices) ijvertices = [] for iv in range(Nvertices): vertex = lLvertices[iv] ijvertices.append(gen.lonlatFind(lon,lat,vertex[0],vertex[1])[0]) print iv, vertex, ijvertices[iv] plane = np.zeros((lon.shape), dtype=int) # Getting the points which are corssed by the faces of the polygon for iiv in range(Nvertices): lL = lLvertices[iiv] l = lL[0] L = lL[1] ij = ijvertices[iiv] iv = ij[1] jv = ij[0] # Next point if iiv == Nvertices - 1: nextiiv = 0 else: nextiiv = iiv+1 lLdest = lLvertices[nextiiv] ldest = lLdest[0] Ldest = lLdest[1] ijdest = ijvertices[nextiiv] ivdest = ijdest[1] jvdest = ijdest[0] # connecting line y = a + by dlon = ldest - l dlat = Ldest - L if dlon != 0.: b = 1.*dlat/dlon sdx = dlon/np.abs(dlon) else: b = 0. sdx = 1 if dlat != 0.: sdy = dlat/np.abs(dlat) else: sdy = 1 # Get more grid points (just in case) Nlon = np.abs(int((ivdest - iv)*1.5)) Nlat = np.abs(int((jvdest - jv)*1.5)) if dlon != 0. and Nlon == 0: Nlon = Nlat if Nlat > Nlon: Nlon = Nlat #print iiv, ',', lL, lLdest, ':', dlon, dlat, b, 'Npts:', Nlon, Nlat # Moving across the line folloing lon,lat if dlon != 0.: for iil in range(Nlon + 1): ilon = dlon*iil/Nlon ilat = L + ilon*b [iy,ix], dist = gen.lonlatFind(lon,lat,l+ilon,ilat) #print ' ',iil,l+ilon,ilat,ix,iy,dist plane[iy,ix] = 1 else: ilon = l for iiL in range(Nlat + 1): ilat = L + dlat*iiL/Nlat [iy,ix], dist = gen.lonlatFind(lon,lat,ilon,ilat) #print ' ',iiL,ilon,ilat,ix,iy,dist plane[iy,ix] = 1 #print ' last point of the face:', l+ilon, ilat, 'destiny:', ldest, Ldest # Filling the polygon # FROM: http://stackoverflow.com/questions/36399381/whats-the-fastest-way-of-checking-if-a-point-is-inside-a-polygon-in-python lonvals = lon.flatten() latvals = lat.flatten() lonlatvals = zip(lonvals, latvals) path = mpltPath.Path(lLvertices) inside = path.contains_points(lonlatvals) insidemat = inside.reshape(lon.shape) insideones = np.where(insidemat, 1, 0) # output file newnc = NetCDFFile(ofile, 'w') # dimensions newdim = newnc.createDimension('lon',lon.shape[1]) newdim = newnc.createDimension('lat',lon.shape[0]) newdim = newnc.createDimension('vertex',Nvertices) # variable-dimensions newvar = newnc.createVariable('lon', 'f8', ('lat', 'lon')) newvar[:] = lon basicvardef(newvar,'longitude','Longitude','degrees_East') newvar.setncattr('axis', 'X') newvar.setncattr('_CoordinateAxisType', 'Lon') newvar = newnc.createVariable('lat', 'f8', ('lat', 'lon')) newvar[:] = lat basicvardef(newvar,'latitude','Latitude','degrees_North') newvar.setncattr('axis', 'Y') newvar.setncattr('_CoordinateAxisType', 'Lat') lnewvar = newnc.createVariable('vertexlon', 'f8', ('vertex')) basicvardef(lnewvar,'vertexlon','longitude coordinate of the vertex of polygon','degrees_East') Lnewvar = newnc.createVariable('vertexlat', 'f8', ('vertex')) basicvardef(Lnewvar,'vertexlat','latitude coordinate of the vertex of polygon','degrees_North') inewvar = newnc.createVariable('ivertex', 'i', ('vertex')) basicvardef(inewvar,'ivertex','coordinate on axis-x of the vertex of polygon','1') jnewvar = newnc.createVariable('jvertex', 'i', ('vertex')) basicvardef(jnewvar,'jvertex','coordinate on axis-y of the vertex of polygon','1') for i in range(Nvertices): lL = lLvertices[i] ij = ijvertices[i] lnewvar[i] = lL[0] Lnewvar[i] = lL[1] inewvar[i] = ij[0] jnewvar[i] = ij[1] newattr = set_attribute(lnewvar, 'src_polygon_file', asciipolygfile) newattr = set_attribute(Lnewvar, 'src_polygon_file', asciipolygfile) newattr = set_attribute(inewvar, 'src_projection_file', ncfile) newattr = set_attribute(jnewvar, 'src_projection_file', ncfile) # variable newvar = newnc.createVariable('polygon_countour', 'i', ('lat', 'lon')) newvar[:] = plane basicvardef(newvar,'polygon_countour','countour of the Polygon','1') newvar.setncattr('coordinates', 'lon lat') newvar = newnc.createVariable('polygon_sfc', 'i', ('lat', 'lon')) newvar[:] = insideones basicvardef(newvar,'polygon_surface','surface of the Polygon','1') newvar.setncattr('coordinates', 'lon lat') newnc.sync() # Global attributes add_global_PyNCplot(newnc, main, fname, '1.0') newnc.close() print fname + ": succesfull written of file with polygon surface '" + ofile + "' !!" return #lonlat_polygon('/home/lluis/PY/square.dat','/home/lluis/PY/wrfout_d01_1995-01-01_00:00:00', 'XLONG,XLAT') #lonlat_polygon('/home/lluis/PY/star.dat','/home/lluis/PY/wrfout_d01_1995-01-01_00:00:00', 'XLONG,XLAT') def merge_files(values, ncfiles, variable): """ Function to merge variables from two files values= [dimvar]:[mergedfilen] [dimvarn]: ',' list of couples [dimvarn]|[dimn] of the name of dimension and the variable with its values related to the variables to merge [mergedfilen]: name of the merged file ncfile= ',' list of file names to merge variable= ',' list of names of variables to merge ('all' for all variables) """ fname = 'merge_files' if values == 'h': print fname + '_____________________________________________________________' print merge_files.__doc__ quit() expectargs = '[dimvar]:[mergedfilen]' gen.check_arguments(fname, values, expectargs, ':') dimvarns = gen.str_list(values.split(':')[0], ',') mergedfilen = values.split(':')[1] filen1 = ncfiles.split(',')[0] filen2 = ncfiles.split(',')[1] if not os.path.isfile(filen1): print errormsg print ' ' + fname + ": file named '" + filen1 + "' does not exist !!" quit(-1) if not os.path.isfile(filen2): print errormsg print ' ' + fname + ": file named '" + filen2 + "' does not exist !!" quit(-1) onc1 = NetCDFFile(filen1, 'r') onc2 = NetCDFFile(filen2, 'r') dimn1 = onc1.dimensions dimn2 = onc2.dimensions varn1 = onc1.variables.keys() varn2 = onc2.variables.keys() # Getting dimension variables dimvar = {} for dimvarn in dimvarns: dimn = dimvarn.split('|')[0] dvn = dimvarn.split('|')[1] if not gen.searchInlist(dimn1, dimn): print errormsg print ' ' +fname+ ": file '" + filen1 + "' does not have dimension: '"+ \ dimn print ' available ones:', dimn1 quit(-1) if not gen.searchInlist(dimn2, dimn): print errormsg print ' ' +fname+ ": file '" + filen2 + "' does not have dimension: '"+ \ dimn print ' available ones:', dimn2 quit(-1) if not gen.searchInlist(varn1, dvn): print errormsg print ' ' +fname+ ": file '" + filen1 + "' does not have variable: '" + \ dimn print ' available ones:', varn1 quit(-1) if not gen.searchInlist(varn2, dvn): print errormsg print ' ' +fname+ ": file '" + filen2 + "' does not have variable: '" + \ dimn print ' available ones:', varn2 quit(-1) dimvar[dimn] = dvn if variable == 'all': # concident variables setvarn1 = set(varn1) setvarn2 = set(varn2) coincidentvarn = list(setvarn1.intersection(setvarn2)) else: coincidentvarn = gen.str_list(variable, ',') # Creation of merged file onewnc = NetCDFFile(mergedfilen, 'w') # Looping on coincident variables for varn in coincidentvarn: print ' ' + fname + ": merging on variable '" + varn + "' ...." ovar1 = onc1.variables[varn] ovar2 = onc2.variables[varn] dimvn1 = ovar1.dimensions dimvn2 = ovar2.dimensions # Assuming same name of dimensions. Too busy to generalize it... ;) NOcoindimns = list(set(dimvn1).symmetric_difference(set(dimvn2))) if len(NOcoindimns) != 0: print errormsg print ' ' + fname + ": variable to merge '" + varn + "' have " + \ "different dimensions in files!!" print " file '" + filen1 + "' dimensions: ", dimvn1 print " file '" + filen2 + "' dimensions: ", dimvn2 quit(-1) # Getting values of the dimensions dimvals = {} for dimn in dimvn1: odvar1 = onc1.variables[dimvar[dimn]] odvar2 = onc2.variables[dimvar[dimn]] vdd1 = odvar1[:] vdd2 = odvar2[:] if len(vdd1.shape) != 1: print errormsg print ' ' + fname + ": dimension-variable '" + dimvar[dimn] + \ "' is not 1D!!" print " shape", vdd1.shape quit(-1) lvdd1 = list(vdd1) lvdd2 = list(vdd2) # Looking for non coincident values notvdd = list(set(lvdd2).difference(set(lvdd1))) Lnotvdd = len(notvdd) dimv = np.zeros((len(vdd1)+Lnotvdd), dtype=vdd1.dtype) if Lnotvdd == 0: newdimv = np.array(lvdd1) else: # Getting the right order and sense notvdd.sort() if lvdd1[1] < lvdd1[0]: notvdd.reverse() # Getting dimension values starting with filen1 as reference print infmsg print ' ' + fname + ": dimension '" + dimvar[dimn] + "' with ", \ Lnotvdd, "different values !!" print ' size from file1:', len(lvdd1), 'file2:', len(lvdd2) # Filling dimension newdimv = np.array(gen.InsertVal_list_sign(lvdd1, notvdd)) # Creation of dimension Lnewdim = len(newdimv) if not gen.searchInlist(onewnc.dimensions, dimn): print infmsg print ' ' + fname + ": creation of dimension '" + dimn+ "' (" + \ str(Lnewdim) + ") !!" if onc1.dimensions[dimn].isunlimited(): newdim = onewnc.createDimension(dimn, None) else: newdim = onewnc.createDimension(dimn, Lnewdim) # Creation of variable-dimension if not onewnc.variables.has_key(dimvar[dimn]): print infmsg print ' ' + fname + ": creation of variable '" + dimvar[dimn] + \ "'", newdimv.shape, "!!" if gen.searchInlist(odvar1.ncattrs(), '_FillValue'): newvar = onewnc.createVariable(dimvar[dimn], nctype(vdd1.dtype), \ (dimn), fill_value=ovar1.getncattr('_FillValue')) else: newvar = onewnc.createVariable(dimvar[dimn], nctype(vdd1.dtype), \ (dimn)) newvar[:] = newdimv[:] for attrn in odvar1.ncattrs(): attrv = odvar1.getncattr(attrn) if attrn != '_FillValue': newattr = set_attribute(newvar, attrn, \ attrv) onewnc.sync() # Looking for the values of the variables and filling netCDF file vvar1 = ovar1[:] vvar2 = ovar2[:] Lv1 = len(vvar1) Lv2 = len(vvar2) v1dim = ovar1.dimensions[0] odimv = onewnc.variables[dimvar[v1dim]] dimv = odimv[:] Lnewdim = len(dimv) newvarvals = np.zeros((Lnewdim), vdd1.dtype) iv1 = 0 iv2 = 0 for iv in range(Lnewdim): if dimv[iv] == vdd1[iv1]: newvarvals[iv] = vvar1[iv1] iv1 = np.min([iv1 + 1, Lv1-1]) else: newvarvals[iv] = vvar2[iv2] iv2 = np.min([iv2 + 1, Lv2-1]) # Creation of variable if not onewnc.variables.has_key(varn): print infmsg print ' '+fname+": creation of variable '" +varn+ "' (" +dimn+ ") !!" if gen.searchInlist(ovar1.ncattrs(), '_FillValue'): newvar = onewnc.createVariable(dimvar[dimn], nctype(vdd1.dtype), \ (dimn), fill_value=ovar1.getncattr('_FillValue')) else: newvar = onewnc.createVariable(dimvar[dimn], nctype(vdd1.dtype), \ (dimn)) newvar[:] = newdimv for attrn in ovar1.ncattrs(): attrv = ovar1.getncattr(attrn) if attrn != '_FillValue': newattr = set_attribute(newvar, attrn, attrv) onewnc.sync() # Global attributes gattrs = onc1.ncattrs() for attrn in gattrs: attrv = onc1.getncattr(attrn) newattr = set_attribute(onewnc, attrn, attrv) onewnc.sync() # Adding PyNCplot add_global_PyNCplot(onewnc, main, fname, '0.1') onewnc.sync() onewnc.close() print fname + ": Variables succesfuly merged in file '" + mergedfilen + "' !!" return def compute_WRFtime(timewrfv, refdate='19491201000000', tunitsval='minutes'): """ Function to copmute CFtimes from WRFtime variable refdate= [YYYYMMDDMIHHSS] format of reference date tunitsval= CF time units timewrfv= matrix string values of WRF 'Times' variable """ fname = 'compute_WRFtime' yrref=refdate[0:4] monref=refdate[4:6] dayref=refdate[6:8] horref=refdate[8:10] minref=refdate[10:12] secref=refdate[12:14] refdateS = yrref + '-' + monref + '-' + dayref + ' ' + horref + ':' + minref + \ ':' + secref dt = timewrfv.shape[0] WRFtime = np.zeros((dt), dtype=np.float) for it in range(dt): wrfdates = gen.datetimeStr_conversion(timewrfv[it,:],'WRFdatetime', 'matYmdHMS') WRFtime[it] = gen.realdatetime1_CFcompilant(wrfdates, refdate, tunitsval) tunits = tunitsval + ' since ' + refdateS return WRFtime, tunits def reconstruct_matrix_from_vector(values, ncfile, variable): """ Function to reconstruct a 2D matrix from a pair of syncronized vectors with the positions on x and y coordinates values=[varXpos]:[varYpos]:[Xmin]:[Xmax]:[Ymin]:[Ymax]:[matProj]:[maxdiff] [varXpos]: variable with the vector of X positions [varYpos]: variable with the vector of Y positions [Xmin]: minimum X-value of the range of the matrix to be reconstructed ('all' for all range) [Xmax]: maximum X-value of the range of the matrix to be reconstructed ('all' for all range) [Ymin]: minimum Y-value of the range of the matrix to be reconstructed ('all' for all range) [Ymax]: maximum Y-value of the range of the matrix to be reconstructed ('all' for all range) [matProj]: assumed projection of the final matrix 'latlon': regular lon/lat projection [maxdiff]: maximum allowed distance between matrix grid point and vector location ncfile= netCDF file to use variable= ',' list of variables to re-matrix ('all' for all variables) """ import module_ForSci as Sci fname = 'reconstruct_matrix_from_vector' availProj = ['latlon'] if values == 'h': print fname + '_____________________________________________________________' print reconstruct_matrix_from_vector.__doc__ quit() expectargs = '[varXpos]:[varYpos]:[Xmin]:[Xmax]:[Ymin]:[Ymax]:[matProj]:' + \ '[maxdiff]' gen.check_arguments(fname, values, expectargs, ':') varXpos = values.split(':')[0] varYpos = values.split(':')[1] if values.split(':')[2] != 'all': Xmin = np.float(values.split(':')[2]) Xmax = np.float(values.split(':')[3]) Ymin = np.float(values.split(':')[4]) Ymax = np.float(values.split(':')[5]) else: Xmin = values.split(':')[2] matProj = values.split(':')[6] maxdiff = np.float(values.split(':')[7]) if not gen.searchInlist(availProj, matProj): print errormsg print ' ' + fname + ": projection '" + matProj + "' not available !!" print ' available ones:', availProj uit(-1) onc = NetCDFFile(ncfile, 'r') ncvars = onc.variables.keys() ncdims = onc.dimensions # checking if not gen.searchInlist(ncvars, varXpos): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have variable '" + \ varXpos + "' !!" print ' available ones:', ncvars quit(-1) if not gen.searchInlist(ncvars, varYpos): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have variable '" + \ varYpos + "' !!" print ' available ones:', ncvars quit(-1) if variable == 'all': varns = ncvars else: varns = variable.split(',') for vn in varns: if not gen.searchInlist(ncvars, vn): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have " + \ " variable '" + vn + "' !!" print ' available ones:', ncvars quit(-1) # Getting values oxpos = onc.variables[varXpos] oypos = onc.variables[varYpos] xposv = oxpos[:] yposv = oypos[:] gen.same_shape(xposv, yposv) vecdim = oxpos.dimensions[0] if gen.searchInlist(oxpos.ncattrs(), 'units'): xunits = oxpos.getncattr('units') else: xunits = '-' if gen.searchInlist(oypos.ncattrs(), 'units'): yunits = oypos.getncattr('units') else: yunits = '-' if Xmin == 'all': Xmin = np.min(xposv) Xmax = np.max(xposv) Ymin = np.min(yposv) Ymax = np.max(yposv) # Matrix values if matProj == 'latlon': ddx = xposv[1]- xposv[0] ddy = yposv[1]- yposv[0] dimx = (Xmax - Xmin+ddx)/ddx dimy = (Ymax - Ymin+ddy)/ddy print dir(Sci) matindt, matXt, matYt = Sci.reconstruct_matrix(vectorxpos=xposv, vectorypos=yposv, \ dvec=len(xposv), xmin=Xmin, xmax=Xmax, ymin=Ymin, ymax=Ymax, dmatx=dimx, \ dmaty=dimy, matproj=matProj, maxdiff=maxdiff) matind = matindt.transpose() # Creation of file onewnc = NetCDFFile('reconstruct_matrix.nc', 'w') # Dimensions newdim = onewnc.createDimension('x', dimx) newdim = onewnc.createDimension('y', dimy) # Variable-dimension newvar = onewnc.createVariable(varXpos, 'f8', ('y', 'x')) newvar[:] = matXt.transpose() basicvardef(newvar, varXpos, varXpos, xunits) newvar = onewnc.createVariable(varYpos, 'f8', ('y', 'x')) newvar[:] = matYt.transpose() basicvardef(newvar, varYpos, varYpos, yunits) onewnc.sync() # Getting variables for vn in varns: if not onewnc.variables.has_key(vn): ovar = onc.variables[vn] indn = ovar.dimensions vardims = [] shapevar = [] for dn in indn: if not gen.searchInlist(onewnc.dimensions, dn) and dn != Xdimn and \ dn != Ydimn: if onc.dimensions[dn].isunlimited(): newdim = nc.createDimension(dn, None) else: newdim = nc.createDimension(dn, len(onc.dimensions[dn])) if dn == vecdim: vardims.append('y') vardims.append('x') shapevar.append(dimy) shapevar.append(dimx) else: vardims.append(dn) shapevar.append(len(onc.dimensions[dn])) if ovar.dtype == type(int(2)): newvar= onewnc.createVariable(vn, nctype(ovar.dtype), tuple(vardims),\ fill_value=gen.fillValueI) varvals = np.ones(tuple(shapevar), dtype=ovar.dtype)*gen.fillValueI else: newvar= onewnc.createVariable(vn, nctype(ovar.dtype), tuple(vardims),\ fill_value=gen.fillValueF) varvals = np.ones(tuple(shapevar), dtype=ovar.dtype)*gen.fillValueF slices = gen.provide_slices(vardims, shapevar, ['x','y']) # Filling variable. It would be faster if we can avoid this loop... I'm feeling lazy! iix = vardims.index('x') iiy = vardims.index('y') for slc in slices: ival = slc[iix] jval = slc[iiy] oldvarslice = [] for dn in indn: iid = vardims.index(dn) if dn == vecdimn: oldvarslice.append(matind(jval,ival)) else: oldvarslice.append(slc[iid]) newvar[tuple(slc)] = onc[tuple(oldvarslice)] # Attributes for atn in ovar.ncattrs(): if atn != 'fill_Value': atv = ovar.getncattr(atn) set_attribute(newvar, atn, atv) onewnc.sync() # Global attributes for atn in onc.ncattrs(): atv = ovar.getncattr(atn) set_attribute(onewnc, atn, atv) onewnc.sync() add_global_PyNCplot(onewnc, pyscript, fname, '0.1') return #quit()