import numpy as np from netCDF4 import Dataset as NetCDFFile import os import re import numpy.ma as ma main = 'nc_var_tools.py' errormsg='ERROR -- error -- ERROR -- error' warnmsg='WARNING -- warning -- WARNING -- warning' fillValue = 1.e20 fillValueF = 1.e20 fillValueC = '-' fillValueI = -99999 fillValueI16 = -99999 fillValueF64 = 1.e20 fillValueI32 = -99999 def values_line(line, splitv, chars): """ Function to retrieve values from a line of ASCII text removing a series of characters line= line of ASCII text to work with splitv= character to use to split with the line chars= list of characters/strings to remove >>> values_line('0.0 0.0 1 [ 0 0 ]', ' ', ['[', ']']) ['0.0', '0.0', '1', '0', '0'] """ fname = 'values_line' vals = line.replace('\n', '') for ic in chars: vals = vals.replace(ic, '') vals0 = vals.split(splitv) vals = [] for val in vals0: if not len(val) == 0: vals.append(val) return vals def filexist(filen, emsg, fmsg): """ Check existence of a file filen= file name emsg= general error message fmsg= message before generic text of inexistence of file """ if not os.path.isfile(filen): print emsg print ' ' + fmsg + ' file "' + filen + '" does not exist !!' print emsg quit(-1) def varinfile(ncf, filen, emsg, vmsg, varn): """ Check existence of a variable inside a netCDF file ncf= netCDF object filen= file name emsg= general error message vmsg= message before generic text of inexistence of variable varn= variable name """ if not ncf.variables.has_key(varn): print emsg print ' file "' + filen + '" does not have ' + vmsg + ' variable "' + varn + '" !!' print emsg ncf.close() quit(-1) class testvarinfile(object): """ Check existence of a variable inside a netCDF file ncf= netCDF object filen= file name emsg= general error message vmsg= message before generic text of inexistence of variable varn= variable name self.exist= boolean value of existence self.message= error message """ def __init__(self, ncf, filen, emsg, vmsg, varn): if ncf is None: self.exist= None self.message = None else: if not ncf.variables.has_key(varn): self.exist = False self.message = emsg + '\n' + ' file "' + filen + '" does not have "' + vmsg + '" variable "' + varn + '" !!\n' + \ emsg else: self.exist = True def attrinvar(varobj, emsg, vmsg, attrn): """ Check existence of an attribute in a netCDF variable object varobj= netCDF variable object emsg= general error message vmsg= message before generic text of inexistence of variable attrrn= attribute name """ varattrs = varobj.ncattrs() if not isInlist(varattrs, attrn): print emsg print ' variable "' + varobj.name + '" does not have attribute ' + attrn + '" !!' print emsg ncf.close() quit(-1) def reduce_spaces(string): """ Function to give words of a line of text removing any extra space """ values = string.replace('\n','').split(' ') vals = [] for val in values: if len(val) > 0: vals.append(val) return vals def reduce_tabulars(string): """ Function to give words of a line of text removing any extra space and tabular separation """ values = string.replace('\n','').split('\t') vals = [] for val in values: if len(val) > 0: vals.append(val) return vals def check_arguments(funcname,args,expectargs,char): """ Function to check the number of arguments if they are coincident check_arguments(funcname,Nargs,args,char) funcname= name of the function/program to check args= passed arguments expectargs= expected arguments char= character used to split the arguments """ fname = 'check_arguments' Nvals = len(args.split(char)) Nargs = len(expectargs.split(char)) if Nvals != Nargs: print errormsg print ' ' + fname + ': wrong number of arguments:',Nvals," passed to '", \ funcname, "' which requires:",Nargs,'!!' print ' # given expected _______' Nmax = np.max([Nvals, Nargs]) for ia in range(Nmax): if ia > Nvals-1: aval = ' ' else: aval = args.split(char)[ia] if ia > Nargs-1: aexp = ' ' else: aexp = expectargs.split(char)[ia] print ' ', ia, aval, aexp quit(-1) return def Str_Bool(val): """ Function to transform from a String value to a boolean one >>> Str_Bool('True') True >>> Str_Bool('0') False >>> Str_Bool('no') False """ fname = 'Str_Bool' if val == 'True' or val == 'true' or val == '1' or val == 'yes' or val == 'T' or val == 't': boolv = True boolv = True elif val == 'False' or val == 'false' or val == '0' or val== 'no' or val == 'F' or val == 'f': boolv = False else: print errormsg print ' ' + fname + ": value '" + val + "' not ready!!" quit(-1) return boolv def Nchar(N,string): """ Function to provide 'N' times concatenated 'string' N= number of copies strin= string to reproduce >>> Nchar(4,'#') #### """ fname = 'Nchar' newstring='' for it in range(N): newstring=newstring + string return newstring def string_dicvals_char(dictionary, string, char): """ Function to provide a string taking values from a given [string] where each single character coincide with the key of the dictionary [dictionary], separated by a given set of characters [char] dictionary= dictionary of the values dim([c]) = {[value]} string = [c1][...[cN]] String with the combination of [c]s >>> dictvals = {'i': 'dimx', 'j': 'dimy', 'k': 'dimz', 't': 'dimt'} >>> string_dicvals_char(dictvals, 'ijkt', ', ') dimx, dimy, dimz, dimt """ fname = 'string_dicvals_char' Lstring = len(string) newstring = '' for ic in range(Lstring): cc = string[ic:ic+1] if not dictionary.has_key(cc): print errormsg print ' ' + fname + ": dictinoary of values does not have character '" +\ cc + "' !!" print ' it has:',dictionary if ic == 0: newstring = dictionary[cc] else: newstring = newstring + char + dictionary[cc] return newstring def substring(stringv,pos,char): """ Function to substitute a character of a given string stringv= string to change pos= position to change char= characters to use as substitution >>> substring('0123456',2,'ii') 01ii3456 """ fname = 'substring' # From: http://stackoverflow.com/questions/1228299/change-one-character-in-a-string-in-python newstring = stringv[:pos] + char + stringv[pos+1:] return newstring def period_information(idate, edate, totunits): """ Function to provide the information of a given period idate, edate (dates in [YYYY][MM][DD][HH][MI][SS] format) [idate]= initial date of the period (in [YYYY][MM][DD][HH][MI][SS] format) [edate]= end date of the period (in [YYYY][MM][DD][HH][MI][SS] format) [totunits]= latest number of entire temporal units to compute: 'year', 'month', 'day', 'hour', 'minute', 'second' resultant information given as ',' list of: [dT]: sign of the temporal increment ('pos', 'neg') [idate]: initial date as [YYYY]:[MM]:[DD]:[HH]:[MI]:[SS] [edate]: end date as [YYYY]:[MM]:[DD]:[HH]:[MI]:[SS] [Nsecs]: total seconds between dates [Nyears]: Number of years taken as entire units ([iYYYY]0101000000 to [eYYYY+1]0101000000) [ExactYears]: Exact year-dates of the period as a '@' separated list of [YYYY]0101000000 [Nmonths]: Number of months taken as entire units ([iYYYY][iMM]01000000 to [eYYYY][eMM+1]01000000) [ExactMonths]: Exact mon-dates of the period as a '@' separated list of [YYYY][MM]01000000 [Ndays]: Number of days taken as entire units ([iYYYY][iMM][iDD]000000 to [eYYYY][eMM][eDD+1]000000) [ExactDays]: Exact day-dates of the period as a '@' separated list of [YYYY][MM][DD]000000 [Nhours]: Number of hours taken as entire units ([iYYYY][iMM][iDD][iHH]0000 to [eYYYY][eMM][eDD][eHH+1]0000) [ExactHours]: Exact hour-dates of the period as a '@' separated list of [YYYY][MM][DD][HH]0000 [Nminutes]: Number of minutes taken as entire units ([iYYYY][iMM][iDD][iHH][iMI]00 to [eYYYY][eMM][eDD][eHH][eMI+1]00) [ExactMinutes]: Exact minutes-dates of the period as a '@' separated list of [YYYY][MM][DD][HH][MI]00 [Nsecs]: Number of minutes taken as entire units ([iYYYY][iMM][iDD][iHH][iMI][iS] to [eYYYY][eMM][eDD][eHH][eMI][eSE+1]) [ExactSeconds]: Exact seconds-dates of the period as a '@' separated list of [YYYY][MM][DD][HH][MI][SS] >>> period_information( '19760217083110', '19770828000000', 'month') pos,1976:02:17:08:31:10,1977:08:28:00:00:00,48180530.0,19,19760201000000@19760301000000@19760401000000@ 19760501000000@19760601000000@19760701000000@19760801000000@19760901000000@19761001000000@19761101000000@ 19761201000000@19770101000000@19770201000000@19770301000000@19770401000000@19770501000000@19770601000000@ 19770701000000@19770801000000@19770901000000 """ import datetime as dt fname = 'period_information' if idate == 'h': print fname + '_____________________________________________________________' print variables_values.__doc__ quit() expectargs = '[idate],[edate],[totunits]' check_arguments(fname,str(idate)+','+str(edate)+','+str(totunits),expectargs,',') # Checking length Lidate = len(idate) Ledate = len(edate) if Lidate != Ledate: print errormsg print ' ' + fname + ': string dates of the period have different length !!' print " idate: '" + idate + "' (L=", Lidate, ") edate: '" + edate + \ "' (L=", Ledate, ')' quit(-1) else: if Lidate != 14: print errormsg print ' ' + fname + ": wrong initial date: '" + idate + "' must " + \ 'have 14 characters!!' print ' and it has:', Lidate quit(-1) if Ledate != 14: print errormsg print ' ' + fname + ": wrong end date: '" + edate + "' must " + \ 'have 14 characters!!' print ' and it has:', Ledate quit(-1) # Checking for right order of dates if int(idate) > int(edate): print warnmsg print ' ' + fname + ": initial date '" + idate + "' after end date '" + \ edate + "' !!" print " re-sorting them!" i1 = edate e1 = idate idate = i1 edate = e1 oinf = 'neg' else: oinf = 'pos' # Year, month, day, hour, minute, second initial date iyrS = idate[0:4] imoS = idate[4:6] idaS = idate[6:8] ihoS = idate[8:10] imiS = idate[10:12] iseS = idate[12:14] oinf = oinf + ',' + iyrS+':'+imoS+':'+idaS+':'+ihoS+':'+imiS+':'+iseS # Year, month, day, hour, minute, second end date eyrS = edate[0:4] emoS = edate[4:6] edaS = edate[6:8] ehoS = edate[8:10] emiS = edate[10:12] eseS = edate[12:14] oinf = oinf + ',' + eyrS+':'+emoS+':'+edaS+':'+ehoS+':'+emiS+':'+eseS idateT = dt.datetime(int(iyrS),int(imoS),int(idaS),int(ihoS),int(imiS),int(iseS)) edateT = dt.datetime(int(eyrS),int(emoS),int(edaS),int(ehoS),int(emiS),int(eseS)) # Seconds between dates DT = edateT - idateT Nsecs = DT.total_seconds() oinf = oinf + ',' + str(Nsecs) # Looking for number of years tacking exact beginning of the units [iYYYY]0101000000, [eYYYY+1]0101000000 ## if totunits == 'year': itime = int(iyrS) if edate[4:15] != '0101000000': etime = int(eyrS)+1 else: etime = int(eyrS) itimeT = dt.datetime(itime,1,1,0,0,0) Nyears = 1 ExactYears = itimeT.strftime("%Y%m%d%H%M%S") it = int(iyrS) while it + 1 <= etime: it = it + 1 Nyears = Nyears + 1 itT = dt.datetime(it,1,1,0,0,0) ExactYears = ExactYears + '@' + itT.strftime("%Y%m%d%H%M%S") oinf = oinf + ',' + str(Nyears) + ',' + ExactYears # Looking for number of months tacking exact beginning of the units [iYYYY][iMM]01000000, [eYYYY][eMM+1]01000000 ## if totunits == 'month': itime = int(iyrS)*100 + int(imoS) if edate[6:15] != '01000000': emo1 = int(emoS)+1 if emo1 > 13: eyr1 = str(int(eyrS) + 1) emo1 = 01 else: eyr1 = eyrS else: eyr1 = eyrS emo1 = emoS etime = int(eyr1)*100 + int(emo1) itimeT = dt.datetime(int(iyrS),int(imoS),1,0,0,0) Nmonths = 1 ExactMonths = itimeT.strftime("%Y%m%d%H%M%S") it = itime while it + 1 <= etime: it = it + 1 Nmonths = Nmonths + 1 ityr = it/100 itmo = it - ityr*100 if itmo > 12: ityr = ityr + 1 itmo = 1 it = ityr * 100 + itmo itT = dt.datetime(ityr,itmo,1,0,0,0) ExactMonths = ExactMonths + '@' + itT.strftime("%Y%m%d%H%M%S") oinf = oinf + ',' + str(Nmonths) + ',' + ExactMonths # Looking for number of days tacking exact beginning of the units [iYYYY][iMM][iDD]000000, [eYYYY][eMM][eDD+1]000000 ## if totunits == 'day': itime = dt.datetime(int(iyrS),int(imoS),int(idaS),0,0,0) if edate[8:15] != '000000': etime = dt.datetime(int(eyrS), int(emoS), int(edaS)+1,0,0,0) else: etime = edateT Ndays = 1 ExactDays = itime.strftime("%Y%m%d%H%M%S") it = itime while it + dt.timedelta(days=1) <= etime: it = it + dt.timedelta(days=1) Ndays = Ndays + 1 ExactDays = ExactDays + '@' + it.strftime("%Y%m%d%H%M%S") oinf = oinf + ',' + str(Ndays) + ',' + ExactDays # Looking for number of hours tacking exact beginning of the units [iYYYY][iMM][iDD][iHH]0000, [eYYYY][eMM][eDD][iHH+1]0000 ## if totunits == 'hour': itime = dt.datetime(int(iyrS),int(imoS),int(idaS),int(ihoS),0,0) if edate[10:15] != '0000': etime = dt.datetime(int(eyrS), int(emoS), int(edaS), int(ehoS)+1,0,0) else: etime = edateT Nhours = 1 ExactHours = itime.strftime("%Y%m%d%H%M%S") it = itime while it + dt.timedelta(hours=1) <= etime: it = it + dt.timedelta(hours=1) Nhours = Nhours + 1 ExactHours = ExactHours + '@' + it.strftime("%Y%m%d%H%M%S") oinf = oinf + ',' + str(Nhours) + ',' + ExactHours # Looking for number of minutes tacking exact beginning of the units [iYYYY][iMM][iDD][iHH][iMI]00, [eYYYY][eMM][eDD][iHH][iMI+1]00 ## if totunits == 'minute': itime = dt.datetime(int(iyrS),int(imoS),int(idaS),int(ihoS),int(imiS),0) if edate[12:15] != '00': etime = dt.datetime(int(eyrS), int(emoS), int(edaS), int(ehoS), int(emiS)+1,0) else: etime = edateT Nminutes = 1 ExactMinutes = itime.strftime("%Y%m%d%H%M%S") it = itime while it + dt.timedelta(minutes=1) <= etime: it = it + dt.timedelta(minutes=1) Nminutes = Nminutes + 1 ExactMinutes = ExactMinutes + '@' + it.strftime("%Y%m%d%H%M%S") oinf = oinf + ',' + str(Nminutes) + ',' + ExactMinutes # Looking for number of seconds tacking exact beginning of the units [iYYYY][iMM][iDD][iHH][iMI][iSE], # [eYYYY][eMM][eDD][iHH][iMI][iSE+1] ## if totunits == 'second': itime = dt.datetime(int(iyrS),int(imoS),int(idaS),int(ihoS),int(imiS),int(iseS)) print 'Lluis ', edate[12:15], '00' if edate[12:15] != '00': etime = dt.datetime(int(eyrS), int(emoS), int(edaS), int(ehoS), int(emiS), int(eseS)+1) else: etime = edateT Nseconds = 1 ExactSeconds = itime.strftime("%Y%m%d%H%M%S") it = itime while it + dt.timedelta(seconds=1) < etime: it = it + dt.timedelta(seconds=1) Nseconds = Nseconds + 1 ExactSeconds = ExactSeconds + '@' + it.strftime("%Y%m%d%H%M%S") oinf = oinf + ',' + str(Nseconds) + ',' + ExactSeconds return oinf # LluisWORKING ####### ###### ##### #### ### ## # def variables_values(varName): """ Function to provide values to plot the different variables values from ASCII file 'variables_values.dat' variables_values(varName) [varName]= name of the variable return: [var name], [std name], [minimum], [maximum], [long name]('|' for spaces), [units], [color palette] (following: http://matplotlib.org/1.3.1/examples/color/colormaps_reference.html) [varn]: original name of the variable NOTE: It might be better doing it with an external ASII file. But then we got an extra dependency... >>> variables_values('WRFght') ['z', 'geopotential_height', 0.0, 80000.0, 'geopotential|height', 'm2s-2', 'rainbow'] """ import subprocess as sub fname='variables_values' if varName == 'h': print fname + '_____________________________________________________________' print variables_values.__doc__ quit() # This does not work.... # folderins = sub.Popen(["pwd"], stdout=sub.PIPE) # folder = list(folderins.communicate())[0].replace('\n','') # From http://stackoverflow.com/questions/4934806/how-can-i-find-scripts-directory-with-python folder = os.path.dirname(os.path.realpath(__file__)) infile = folder + '/variables_values.dat' if not os.path.isfile(infile): print errormsg print ' ' + fname + ": File '" + infile + "' does not exist !!" quit(-1) # Variable name might come with a statistical surname... stats=['min','max','mean','stdv', 'sum'] # Variables with a statistical section on their name... NOstatsvars = ['zmaxth', 'zmax_th', 'lmax_th', 'lmaxth'] ifst = False if not searchInlist(NOstatsvars, varName.lower()): for st in stats: if varName.find(st) > -1: print ' '+ fname + ": varibale '" + varName + "' with a " + \ "statistical surname: '",st,"' !!" Lst = len(st) LvarName = len(varName) varn = varName[0:LvarName - Lst] ifst = True break if not ifst: varn = varName ncf = open(infile, 'r') for line in ncf: if line[0:1] != '#': values = line.replace('\n','').split(',') if len(values) != 8: print errormsg print "problem in varibale:'", values[0], \ 'it should have 8 values and it has',len(values) quit(-1) if varn[0:6] == 'varDIM': # Variable from a dimension (all with 'varDIM' prefix) Lvarn = len(varn) varvals = [varn[6:Lvarn+1], varn[6:Lvarn+1], 0., 1., \ "variable|from|size|of|dimension|'" + varn[6:Lvarn+1] + "'", '1', \ 'rainbow'] else: varvals = [values[1].replace(' ',''), values[2].replace(' ',''), \ np.float(values[3]), np.float(values[4]),values[5].replace(' ',''),\ values[6].replace(' ',''), values[7].replace(' ','')] if values[0] == varn: ncf.close() return varvals break print errormsg print ' ' + fname + ": variable '" + varn + "' not defined !!!" ncf.close() quit(-1) return def variables_values_old(varName): """ Function to provide values to plot the different variables variables_values(varName) [varName]= name of the variable return: [var name], [std name], [minimum], [maximum], [long name]('|' for spaces), [units], [color palette] (following: http://matplotlib.org/1.3.1/examples/color/colormaps_reference.html) [varn]: original name of the variable NOTE: It might be better doing it with an external ASII file. But then we got an extra dependency... >>> variables_values('WRFght') ['z', 'geopotential_height', 0.0, 80000.0, 'geopotential|height', 'm2s-2', 'rainbow'] """ fname='variables_values' if varName == 'h': print fname + '_____________________________________________________________' print variables_values.__doc__ quit() # Variable name might come with a statistical surname... stats=['min','max','mean','stdv', 'sum'] ifst = False for st in stats: if varName.find(st) > -1: print ' '+ fname + ": varibale '" + varName + "' with a statistical "+\ " surname: '",st,"' !!" Lst = len(st) LvarName = len(varName) varn = varName[0:LvarName - Lst] ifst = True break if not ifst: varn = varName if varn[0:6] == 'varDIM': # Variable from a dimension (all with 'varDIM' prefix) Lvarn = len(varn) varvals = [varn[6:Lvarn+1], varn[6:Lvarn+1], 0., 1., \ "variable|from|size|of|dimension|'" + varn[6:Lvarn+1] + "'", '1', 'rainbox'] elif varn == 'a_tht' or varn == 'LA_THT': varvals = ['ath', 'total_thermal_plume_cover', 0., 1., \ 'total|column|thermal|plume|cover', '1', 'YlGnBu'] elif varn == 'acprc' or varn == 'RAINC': varvals = ['acprc', 'accumulated_cmulus_precipitation', 0., 3.e4, \ 'accumulated|cmulus|precipitation', 'mm', 'Blues'] elif varn == 'acprnc' or varn == 'RAINNC': varvals = ['acprnc', 'accumulated_non-cmulus_precipitation', 0., 3.e4, \ 'accumulated|non-cmulus|precipitation', 'mm', 'Blues'] elif varn == 'bils' or varn == 'LBILS': varvals = ['bils', 'surface_total_heat_flux', -100., 100., \ 'surface|total|heat|flux', 'Wm-2', 'seismic'] elif varn == 'landcat' or varn == 'category': varvals = ['landcat', 'land_categories', 0., 22., 'land|categories', '1', \ 'rainbow'] elif varn == 'c' or varn == 'QCLOUD' or varn == 'oliq' or varn == 'OLIQ': varvals = ['c', 'condensed_water_mixing_ratio', 0., 3.e-4, \ 'condensed|water|mixing|ratio', 'kgkg-1', 'BuPu'] elif varn == 'ci' or varn == 'iwcon' or varn == 'LIWCON': varvals = ['ci', 'cloud_iced_water_mixing_ratio', 0., 0.0003, \ 'cloud|iced|water|mixing|ratio', 'kgkg-1', 'Purples'] elif varn == 'cl' or varn == 'lwcon' or varn == 'LLWCON': varvals = ['cl', 'cloud_liquidwater_mixing_ratio', 0., 0.0003, \ 'cloud|liquid|water|mixing|ratio', 'kgkg-1', 'Blues'] elif varn == 'cld' or varn == 'CLDFRA' or varn == 'rneb' or varn == 'lrneb' or \ varn == 'LRNEB': varvals = ['cld', 'cloud_area_fraction', 0., 1., 'cloud|fraction', '1', \ 'gist_gray'] elif varn == 'cldc' or varn == 'rnebcon' or varn == 'lrnebcon' or \ varn == 'LRNEBCON': varvals = ['cldc', 'convective_cloud_area_fraction', 0., 1., \ 'convective|cloud|fraction', '1', 'gist_gray'] elif varn == 'cldl' or varn == 'rnebls' or varn == 'lrnebls' or varn == 'LRNEBLS': varvals = ['cldl', 'large_scale_cloud_area_fraction', 0., 1., \ 'large|scale|cloud|fraction', '1', 'gist_gray'] elif varn == 'clt' or varn == 'CLT' or varn == 'cldt' or \ varn == 'Total cloudiness': varvals = ['clt', 'cloud_area_fraction', 0., 1., 'total|cloud|cover', '1', \ 'gist_gray'] elif varn == 'cll' or varn == 'cldl' or varn == 'LCLDL' or \ varn == 'Low-level cloudiness': varvals = ['cll', 'low_level_cloud_area_fraction', 0., 1., \ 'low|level|(p|>|680|hPa)|cloud|fraction', '1', 'gist_gray'] elif varn == 'clm' or varn == 'cldm' or varn == 'LCLDM' or \ varn == 'Mid-level cloudiness': varvals = ['clm', 'mid_level_cloud_area_fraction', 0., 1., \ 'medium|level|(440|<|p|<|680|hPa)|cloud|fraction', '1', 'gist_gray'] elif varn == 'clh' or varn == 'cldh' or varn == 'LCLDH' or \ varn == 'High-level cloudiness': varvals = ['clh', 'high_level_cloud_area_fraction', 0., 1., \ 'high|level|(p|<|440|hPa)|cloud|fraction', '1', 'gist_gray'] elif varn == 'clmf' or varn == 'fbase' or varn == 'LFBASE': varvals = ['clmf', 'cloud_base_max_flux', -0.3, 0.3, 'cloud|base|max|flux', \ 'kgm-2s-1', 'seismic'] elif varn == 'clp' or varn == 'pbase' or varn == 'LPBASE': varvals = ['clp', 'cloud_base_pressure', -0.3, 0.3, 'cloud|base|pressure', \ 'Pa', 'Reds'] elif varn == 'cpt' or varn == 'ptconv' or varn == 'LPTCONV': varvals = ['cpt', 'convective_point', 0., 1., 'convective|point', '1', \ 'seismic'] elif varn == 'dqajs' or varn == 'LDQAJS': varvals = ['dqajs', 'dry_adjustment_water_vapor_tendency', -0.0003, 0.0003, \ 'dry|adjustment|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqcon' or varn == 'LDQCON': varvals = ['dqcon', 'convective_water_vapor_tendency', -3e-8, 3.e-8, \ 'convective|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqdyn' or varn == 'LDQDYN': varvals = ['dqdyn', 'dynamics_water_vapor_tendency', -3.e-7, 3.e-7, \ 'dynamics|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqeva' or varn == 'LDQEVA': varvals = ['dqeva', 'evaporation_water_vapor_tendency', -3.e-6, 3.e-6, \ 'evaporation|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqlscst' or varn == 'LDQLSCST': varvals = ['dqlscst', 'stratocumulus_water_vapor_tendency', -3.e-7, 3.e-7, \ 'stratocumulus|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqlscth' or varn == 'LDQLSCTH': varvals = ['dqlscth', 'thermals_water_vapor_tendency', -3.e-7, 3.e-7, \ 'thermal|plumes|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqlsc' or varn == 'LDQLSC': varvals = ['dqlsc', 'condensation_water_vapor_tendency', -3.e-6, 3.e-6, \ 'condensation|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqphy' or varn == 'LDQPHY': varvals = ['dqphy', 'physics_water_vapor_tendency', -3.e-7, 3.e-7, \ 'physics|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqthe' or varn == 'LDQTHE': varvals = ['dqthe', 'thermals_water_vapor_tendency', -3.e-7, 3.e-7, \ 'thermal|plumes|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqvdf' or varn == 'LDQVDF': varvals = ['dqvdf', 'vertical_difussion_water_vapor_tendency', -3.e-8, 3.e-8,\ 'vertical|difussion|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dqwak' or varn == 'LDQWAK': varvals = ['dqwak', 'wake_water_vapor_tendency', -3.e-7, 3.e-7, \ 'wake|water|vapor|tendency', 'kg/kg/s', 'seismic'] elif varn == 'dta' or varn == 'tnt' or varn == 'LTNT': varvals = ['dta', 'tendency_air_temperature', -3.e-3, 3.e-3, \ 'tendency|of|air|temperature', 'K/s', 'seismic'] elif varn == 'dtac' or varn == 'tntc' or varn == 'LTNTC': varvals = ['dtac', 'moist_convection_tendency_air_temperature', -3.e-3, \ 3.e-3, 'moist|convection|tendency|of|air|temperature', 'K/s', 'seismic'] elif varn == 'dtar' or varn == 'tntr' or varn == 'LTNTR': varvals = ['dtar', 'radiative_heating_tendency_air_temperature', -3.e-3, \ 3.e-3, 'radiative|heating|tendency|of|air|temperature', 'K/s', 'seismic'] elif varn == 'dtascpbl' or varn == 'tntscpbl' or varn == 'LTNTSCPBL': varvals = ['dtascpbl', \ 'stratiform_cloud_precipitation_BL_mixing_tendency_air_temperature', \ -3.e-6, 3.e-6, \ 'stratiform|cloud|precipitation|Boundary|Layer|mixing|tendency|air|' + 'temperature', 'K/s', 'seismic'] elif varn == 'dtajs' or varn == 'LDTAJS': varvals = ['dtajs', 'dry_adjustment_thermal_tendency', -3.e-5, 3.e-5, \ 'dry|adjustment|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtcon' or varn == 'LDTCON': varvals = ['dtcon', 'convective_thermal_tendency', -3.e-5, 3.e-5, \ 'convective|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtdyn' or varn == 'LDTDYN': varvals = ['dtdyn', 'dynamics_thermal_tendency', -3.e-4, 3.e-4, \ 'dynamics|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dteva' or varn == 'LDTEVA': varvals = ['dteva', 'evaporation_thermal_tendency', -3.e-3, 3.e-3, \ 'evaporation|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtlscst' or varn == 'LDTLSCST': varvals = ['dtlscst', 'stratocumulus_thermal_tendency', -3.e-4, 3.e-4, \ 'stratocumulus|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtlscth' or varn == 'LDTLSCTH': varvals = ['dtlscth', 'thermals_thermal_tendency', -3.e-4, 3.e-4, \ 'thermal|plumes|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtlsc' or varn == 'LDTLSC': varvals = ['dtlsc', 'condensation_thermal_tendency', -3.e-3, 3.e-3, \ 'condensation|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtlwr' or varn == 'LDTLWR': varvals = ['dtlwr', 'long_wave_thermal_tendency', -3.e-3, 3.e-3, \ 'long|wave|radiation|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtphy' or varn == 'LDTPHY': varvals = ['dtphy', 'physics_thermal_tendency', -3.e-4, 3.e-4, \ 'physics|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtsw0' or varn == 'LDTSW0': varvals = ['dtsw0', 'cloudy_sky_short_wave_thermal_tendency', -3.e-4, 3.e-4, \ 'cloudy|sky|short|wave|radiation|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtthe' or varn == 'LDTTHE': varvals = ['dtthe', 'thermals_thermal_tendency', -3.e-4, 3.e-4, \ 'thermal|plumes|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtvdf' or varn == 'LDTVDF': varvals = ['dtvdf', 'vertical_difussion_thermal_tendency', -3.e-5, 3.e-5, \ 'vertical|difussion|thermal|tendency', 'K/s', 'seismic'] elif varn == 'dtwak' or varn == 'LDTWAK': varvals = ['dtwak', 'wake_thermal_tendency', -3.e-4, 3.e-4, \ 'wake|thermal|tendency', 'K/s', 'seismic'] elif varn == 'ducon' or varn == 'LDUCON': varvals = ['ducon', 'convective_eastward_wind_tendency', -3.e-3, 3.e-3, \ 'convective|eastward|wind|tendency', 'ms-2', 'seismic'] elif varn == 'dudyn' or varn == 'LDUDYN': varvals = ['dudyn', 'dynamics_eastward_wind_tendency', -3.e-3, 3.e-3, \ 'dynamics|eastward|wind|tendency', 'ms-2', 'seismic'] elif varn == 'duvdf' or varn == 'LDUVDF': varvals = ['duvdf', 'vertical_difussion_eastward_wind_tendency', -3.e-3, \ 3.e-3, 'vertical|difussion|eastward|wind|tendency', 'ms-2', 'seismic'] elif varn == 'dvcon' or varn == 'LDVCON': varvals = ['dvcon', 'convective_difussion_northward_wind_tendency', -3.e-3, \ 3.e-3, 'convective|northward|wind|tendency', 'ms-2', 'seismic'] elif varn == 'dvdyn' or varn == 'LDVDYN': varvals = ['dvdyn', 'dynamics_northward_wind_tendency', -3.e-3, \ 3.e-3, 'dynamics|difussion|northward|wind|tendency', 'ms-2', 'seismic'] elif varn == 'dvvdf' or varn == 'LDVVDF': varvals = ['dvvdf', 'vertical_difussion_northward_wind_tendency', -3.e-3, \ 3.e-3, 'vertical|difussion|northward|wind|tendency', 'ms-2', 'seismic'] elif varn == 'etau' or varn == 'ZNU': varvals = ['etau', 'etau', 0., 1, 'eta values on half (mass) levels', '-', \ 'reds'] elif varn == 'evspsbl' or varn == 'LEVAP' or varn == 'evap' or varn == 'SFCEVPde': varvals = ['evspsbl', 'water_evaporation_flux', 0., 1.5e-4, \ 'water|evaporation|flux', 'kgm-2s-1', 'Blues'] elif varn == 'evspsbl' or varn == 'SFCEVPde': varvals = ['evspsblac', 'water_evaporation_flux_ac', 0., 1.5e-4, \ 'accumulated|water|evaporation|flux', 'kgm-2', 'Blues'] elif varn == 'g' or varn == 'QGRAUPEL': varvals = ['g', 'grauepl_mixing_ratio', 0., 0.0003, 'graupel|mixing|ratio', \ 'kgkg-1', 'Purples'] elif varn == 'h2o' or varn == 'LH2O': varvals = ['h2o', 'water_mass_fraction', 0., 3.e-2, \ 'mass|fraction|of|water', '1', 'Blues'] elif varn == 'h' or varn == 'QHAIL': varvals = ['h', 'hail_mixing_ratio', 0., 0.0003, 'hail|mixing|ratio', \ 'kgkg-1', 'Purples'] elif varn == 'hfls' or varn == 'LH' or varn == 'LFLAT' or varn == 'flat': varvals = ['hfls', 'surface_upward_latent_heat_flux', -400., 400., \ 'upward|latnt|heat|flux|at|the|surface', 'Wm-2', 'seismic'] elif varn == 'hfss' or varn == 'LSENS' or varn == 'sens': varvals = ['hfss', 'surface_upward_sensible_heat_flux', -150., 150., \ 'upward|sensible|heat|flux|at|the|surface', 'Wm-2', 'seismic'] elif varn == 'hus' or varn == 'WRFrh' or varn == 'LMDZrh' or varn == 'rhum' or \ varn == 'LRHUM': varvals = ['hus', 'specific_humidity', 0., 1., 'specific|humidty', '1', \ 'BuPu'] elif varn == 'huss' or varn == 'WRFrhs' or varn == 'LMDZrhs' or varn == 'rh2m' or\ varn == 'LRH2M': varvals = ['huss', 'specific_humidity', 0., 1., 'specific|humidty|at|2m', \ '1', 'BuPu'] elif varn == 'i' or varn == 'QICE': varvals = ['i', 'iced_water_mixing_ratio', 0., 0.0003, \ 'iced|water|mixing|ratio', 'kgkg-1', 'Purples'] elif varn == 'lat' or varn == 'XLAT' or varn == 'XLAT_M' or varn == 'latitude': varvals = ['lat', 'latitude', -90., 90., 'latitude', 'degrees North', \ 'seismic'] elif varn == 'lcl' or varn == 's_lcl' or varn == 'ls_lcl' or varn == 'LS_LCL': varvals = ['lcl', 'condensation_level', 0., 2500., 'level|of|condensation', \ 'm', 'Greens'] elif varn == 'lambdath' or varn == 'lambda_th' or varn == 'LLAMBDA_TH': varvals = ['lambdath', 'thermal_plume_vertical_velocity', -30., 30., \ 'thermal|plume|vertical|velocity', 'm/s', 'seismic'] elif varn == 'lmaxth' or varn == 'LLMAXTH': varvals = ['lmaxth', 'upper_level_thermals', 0., 100., 'upper|level|thermals'\ , '1', 'Greens'] elif varn == 'lon' or varn == 'XLONG' or varn == 'XLONG_M': varvals = ['lon', 'longitude', -180., 180., 'longitude', 'degrees East', \ 'seismic'] elif varn == 'longitude': varvals = ['lon', 'longitude', 0., 360., 'longitude', 'degrees East', \ 'seismic'] elif varn == 'orog' or varn == 'HGT' or varn == 'HGT_M': varvals = ['orog', 'orography', 0., 3000., 'surface|altitude', 'm','terrain'] elif varn == 'pfc' or varn == 'plfc' or varn == 'LPLFC': varvals = ['pfc', 'pressure_free_convection', 100., 1100., \ 'pressure|free|convection', 'hPa', 'BuPu'] elif varn == 'plcl' or varn == 'LPLCL': varvals = ['plcl', 'pressure_lifting_condensation_level', 700., 1100., \ 'pressure|lifting|condensation|level', 'hPa', 'BuPu'] elif varn == 'pr' or varn == 'RAINTOT' or varn == 'precip' or \ varn == 'LPRECIP' or varn == 'Precip Totale liq+sol': varvals = ['pr', 'precipitation_flux', 0., 1.e-4, 'precipitation|flux', \ 'kgm-2s-1', 'BuPu'] elif varn == 'prprof' or varn == 'vprecip' or varn == 'LVPRECIP': varvals = ['prprof', 'precipitation_profile', 0., 1.e-3, \ 'precipitation|profile', 'kg/m2/s', 'BuPu'] elif varn == 'prprofci' or varn == 'pr_con_i' or varn == 'LPR_CON_I': varvals = ['prprofci', 'precipitation_profile_convective_i', 0., 1.e-3, \ 'precipitation|profile|convective|i', 'kg/m2/s', 'BuPu'] elif varn == 'prprofcl' or varn == 'pr_con_l' or varn == 'LPR_CON_L': varvals = ['prprofcl', 'precipitation_profile_convective_l', 0., 1.e-3, \ 'precipitation|profile|convective|l', 'kg/m2/s', 'BuPu'] elif varn == 'prprofli' or varn == 'pr_lsc_i' or varn == 'LPR_LSC_I': varvals = ['prprofli', 'precipitation_profile_large_scale_i', 0., 1.e-3, \ 'precipitation|profile|large|scale|i', 'kg/m2/s', 'BuPu'] elif varn == 'prprofll' or varn == 'pr_lsc_l' or varn == 'LPR_LSC_L': varvals = ['prprofll', 'precipitation_profile_large_scale_l', 0., 1.e-3, \ 'precipitation|profile|large|scale|l', 'kg/m2/s', 'BuPu'] elif varn == 'pracc' or varn == 'ACRAINTOT': varvals = ['pracc', 'precipitation_amount', 0., 100., \ 'accumulated|precipitation', 'kgm-2', 'BuPu'] elif varn == 'prc' or varn == 'LPLUC' or varn == 'pluc' or varn == 'WRFprc' or \ varn == 'RAINCde': varvals = ['prc', 'convective_precipitation_flux', 0., 2.e-4, \ 'convective|precipitation|flux', 'kgm-2s-1', 'Blues'] elif varn == 'prci' or varn == 'pr_con_i' or varn == 'LPR_CON_I': varvals = ['prci', 'convective_ice_precipitation_flux', 0., 0.003, \ 'convective|ice|precipitation|flux', 'kgm-2s-1', 'Purples'] elif varn == 'prcl' or varn == 'pr_con_l' or varn == 'LPR_CON_L': varvals = ['prcl', 'convective_liquid_precipitation_flux', 0., 0.003, \ 'convective|liquid|precipitation|flux', 'kgm-2s-1', 'Blues'] elif varn == 'pres' or varn == 'presnivs' or varn == 'pressure' or \ varn == 'lpres' or varn == 'LPRES': varvals = ['pres', 'air_pressure', 0., 103000., 'air|pressure', 'Pa', \ 'Blues'] elif varn == 'prls' or varn == 'WRFprls' or varn == 'LPLUL' or varn == 'plul' or \ varn == 'RAINNCde': varvals = ['prls', 'large_scale_precipitation_flux', 0., 2.e-4, \ 'large|scale|precipitation|flux', 'kgm-2s-1', 'Blues'] elif varn == 'prsn' or varn == 'SNOW' or varn == 'snow' or varn == 'LSNOW': varvals = ['prsn', 'snowfall', 0., 1.e-4, 'snowfall|flux', 'kgm-2s-1', 'BuPu'] elif varn == 'prw' or varn == 'WRFprh': varvals = ['prw', 'atmosphere_water_vapor_content', 0., 10., \ 'water|vapor"path', 'kgm-2', 'Blues'] elif varn == 'ps' or varn == 'psfc' or varn =='PSFC' or varn == 'psol' or \ varn == 'Surface Pressure': varvals=['ps', 'surface_air_pressure', 85000., 105400., 'surface|pressure', \ 'hPa', 'cool'] elif varn == 'psl' or varn == 'mslp' or varn =='WRFmslp': varvals=['psl', 'air_pressure_at_sea_level', 85000., 104000., \ 'mean|sea|level|pressure', 'Pa', 'Greens'] elif varn == 'qth' or varn == 'q_th' or varn == 'LQ_TH': varvals = ['qth', 'thermal_plume_total_water_content', 0., 25., \ 'total|water|cotent|in|thermal|plume', 'mm', 'YlOrRd'] elif varn == 'r' or varn == 'QVAPOR' or varn == 'ovap' or varn == 'LOVAP': varvals = ['r', 'water_mixing_ratio', 0., 0.03, 'water|mixing|ratio', \ 'kgkg-1', 'BuPu'] elif varn == 'r2' or varn == 'Q2': varvals = ['r2', 'water_mixing_ratio_at_2m', 0., 0.03, 'water|mixing|' + \ 'ratio|at|2|m','kgkg-1', 'BuPu'] elif varn == 'rsds' or varn == 'SWdnSFC' or varn == 'SWdn at surface' or \ varn == 'SWDOWN': varvals=['rsds', 'surface_downwelling_shortwave_flux_in_air', 0., 1200., \ 'downward|SW|surface|radiation', 'Wm-2' ,'Reds'] elif varn == 'rsdsacc': varvals=['rsdsacc', 'accumulated_surface_downwelling_shortwave_flux_in_air', \ 0., 1200., 'accumulated|downward|SW|surface|radiation', 'Wm-2' ,'Reds'] elif varn == 'rvor' or varn == 'WRFrvor': varvals = ['rvor', 'air_relative_vorticity', -2.5E-3, 2.5E-3, \ 'air|relative|vorticity', 's-1', 'seismic'] elif varn == 'rvors' or varn == 'WRFrvors': varvals = ['rvors', 'surface_air_relative_vorticity', -2.5E-3, 2.5E-3, \ 'surface|air|relative|vorticity', 's-1', 'seismic'] elif varn == 's' or varn == 'QSNOW': varvals = ['s', 'snow_mixing_ratio', 0., 0.0003, 'snow|mixing|ratio', \ 'kgkg-1', 'Purples'] elif varn == 'stherm' or varn == 'LS_THERM': varvals = ['stherm', 'thermals_excess', 0., 0.8, 'thermals|excess', 'K', \ 'Reds'] elif varn == 'ta' or varn == 'WRFt' or varn == 'temp' or varn == 'LTEMP' or \ varn == 'Air temperature': varvals = ['ta', 'air_temperature', 195., 320., 'air|temperature', 'K', \ 'YlOrRd'] elif varn == 'tah' or varn == 'theta' or varn == 'LTHETA': varvals = ['tah', 'potential_air_temperature', 195., 320., \ 'potential|air|temperature', 'K', 'YlOrRd'] elif varn == 'tas' or varn == 'T2' or varn == 't2m' or varn == 'T2M' or \ varn == 'Temperature 2m': varvals = ['tas', 'air_temperature', 240., 310., 'air|temperature|at|2m', ' \ K', 'YlOrRd'] elif varn == 'tds' or varn == 'TH2': varvals = ['tds', 'air_dew_point_temperature', 240., 310., \ 'air|dew|point|temperature|at|2m', 'K', 'YlGnBu'] elif varn == 'tke' or varn == 'TKE' or varn == 'tke' or varn == 'LTKE': varvals = ['tke', 'turbulent_kinetic_energy', 0., 0.003, \ 'turbulent|kinetic|energy', 'm2/s2', 'Reds'] elif varn == 'time'or varn == 'time_counter': varvals = ['time', 'time', 0., 1000., 'time', \ 'hours|since|1949/12/01|00:00:00', 'Reds'] elif varn == 'tmla' or varn == 's_pblt' or varn == 'LS_PBLT': varvals = ['tmla', 'atmosphere_top_boundary_layer_temperature', 250., 330., \ 'atmosphere|top|boundary|layer|temperature', 'K', 'Reds'] elif varn == 'ua' or varn == 'vitu' or varn == 'U' or varn == 'Zonal wind' or \ varn == 'LVITU': varvals = ['ua', 'eastward_wind', -30., 30., 'eastward|wind', 'ms-1', \ 'seismic'] elif varn == 'uas' or varn == 'u10m' or varn == 'U10' or varn =='Vent zonal 10m': varvals = ['uas', 'eastward_wind', -30., 30., 'eastward|2m|wind', \ 'ms-1', 'seismic'] elif varn == 'va' or varn == 'vitv' or varn == 'V' or varn == 'Meridional wind' \ or varn == 'LVITV': varvals = ['va', 'northward_wind', -30., 30., 'northward|wind', 'ms-1', \ 'seismic'] elif varn == 'vas' or varn == 'v10m' or varn == 'V10' or \ varn =='Vent meridien 10m': varvals = ['vas', 'northward_wind', -30., 30., 'northward|2m|wind', 'ms-1', \ 'seismic'] elif varn == 'wakedeltaq' or varn == 'wake_deltaq' or varn == 'lwake_deltaq' or \ varn == 'LWAKE_DELTAQ': varvals = ['wakedeltaq', 'wake_delta_vapor', -0.003, 0.003, \ 'wake|delta|mixing|ratio', '-', 'seismic'] elif varn == 'wakedeltat' or varn == 'wake_deltat' or varn == 'lwake_deltat' or \ varn == 'LWAKE_DELTAT': varvals = ['wakedeltat', 'wake_delta_temp', -0.003, 0.003, \ 'wake|delta|temperature', '-', 'seismic'] elif varn == 'wakeh' or varn == 'wake_h' or varn == 'LWAKE_H': varvals = ['wakeh', 'wake_height', 0., 1000., 'height|of|the|wakes', 'm', \ 'YlOrRd'] elif varn == 'wakeomg' or varn == 'wake_omg' or varn == 'lwake_omg' or \ varn == 'LWAKE_OMG': varvals = ['wakeomg', 'wake_omega', 0., 3., 'wake|omega', \ '-', 'BuGn'] elif varn == 'wakes' or varn == 'wake_s' or varn == 'LWAKE_S': varvals = ['wakes', 'wake_area_fraction', 0., 0.5, 'wake|spatial|fraction', \ '1', 'BuGn'] elif varn == 'wa' or varn == 'W' or varn == 'Vertical wind': varvals = ['wa', 'upward_wind', -10., 10., 'upward|wind', 'ms-1', \ 'seismic'] elif varn == 'wap' or varn == 'vitw' or varn == 'LVITW': varvals = ['wap', 'upward_wind', -3.e-10, 3.e-10, 'upward|wind', 'mPa-1', \ 'seismic'] elif varn == 'wss' or varn == 'SPDUV': varvals = ['wss', 'air_velocity', 0., 30., 'surface|horizontal|wind|speed', \ 'ms-1', 'Reds'] # Water budget # Water budget de-accumulated elif varn == 'ccond' or varn == 'CCOND' or varn == 'ACCCONDde': varvals = ['ccond', 'cw_cond', 0., 30., \ 'cloud|water|condensation', 'mm', 'Reds'] elif varn == 'wbr' or varn == 'ACQVAPORde': varvals = ['wbr', 'wbr', 0., 30., 'Water|Budget|water|wapor', 'mm', 'Blues'] elif varn == 'diabh' or varn == 'DIABH' or varn == 'ACDIABHde': varvals = ['diabh', 'diabh', 0., 30., 'diabatic|heating', 'K', 'Reds'] elif varn == 'wbpw' or varn == 'WBPW' or varn == 'WBACPWde': varvals = ['wbpw', 'water_budget_pw', 0., 30., 'Water|Budget|water|content',\ 'mms-1', 'Reds'] elif varn == 'wbf' or varn == 'WBACF' or varn == 'WBACFde': varvals = ['wbf', 'water_budget_hfcqv', 0., 30., \ 'Water|Budget|horizontal|convergence|of|water|vapour|(+,|' + \ 'conv.;|-,|div.)', 'mms-1', 'Reds'] elif varn == 'wbfc' or varn == 'WBFC' or varn == 'WBACFCde': varvals = ['wbfc', 'water_budget_fc', 0., 30., \ 'Water|Budget|horizontal|convergence|of|cloud|(+,|conv.;|-,|' +\ 'div.)', 'mms-1', 'Reds'] elif varn == 'wbfp' or varn == 'WBFP' or varn == 'WBACFPde': varvals = ['wbfp', 'water_budget_cfp', 0., 30., \ 'Water|Budget|horizontal|convergence|of|precipitation|(+,|' + \ 'conv.;|-,|div.)', 'mms-1', 'Reds'] elif varn == 'wbz' or varn == 'WBZ' or varn == 'WBACZde': varvals = ['wbz', 'water_budget_z', 0., 30., \ 'Water|Budget|vertical|convergence|of|water|vapour|(+,|conv.' +\ ';|-,|div.)', 'mms-1', 'Reds'] elif varn == 'wbc' or varn == 'WBC' or varn == 'WBACCde': varvals = ['wbc', 'water_budget_c', 0., 30., \ 'Water|Budget|Cloud|water|species','mms-1', 'Reds'] elif varn == 'wbqvd' or varn == 'WBQVD' or varn == 'WBACQVDde': varvals = ['wbqvd', 'water_budget_qvd', 0., 30., \ 'Water|Budget|water|vapour|divergence', 'mms-1', 'Reds'] elif varn == 'wbqvblten' or varn == 'WBQVBLTEN' or varn == 'WBACQVBLTENde': varvals = ['wbqvblten', 'water_budget_qv_blten', 0., 30., \ 'Water|Budget|QV|tendency|due|to|pbl|parameterization', \ 'kg kg-1 s-1', 'Reds'] elif varn == 'wbqvcuten' or varn == 'WBQVCUTEN' or varn == 'WBACQVCUTENde': varvals = ['wbqvcuten', 'water_budget_qv_cuten', 0., 30., \ 'Water|Budget|QV|tendency|due|to|cu|parameterization', \ 'kg kg-1 s-1', 'Reds'] elif varn == 'wbqvshten' or varn == 'WBQVSHTEN' or varn == 'WBACQVSHTENde': varvals = ['wbqvshten', 'water_budget_qv_shten', 0., 30., \ 'Water|Budget|QV|tendency|due|to|shallow|cu|parameterization', \ 'kg kg-1 s-1', 'Reds'] elif varn == 'wbpr' or varn == 'WBP' or varn == 'WBACPde': varvals = ['wbpr', 'water_budget_pr', 0., 30., \ 'Water|Budget|recipitation', 'mms-1', 'Reds'] elif varn == 'wbpw' or varn == 'WBPW' or varn == 'WBACPWde': varvals = ['wbpw', 'water_budget_pw', 0., 30., \ 'Water|Budget|water|content', 'mms-1', 'Reds'] elif varn == 'wbcondt' or varn == 'WBCONDT' or varn == 'WBACCONDTde': varvals = ['wbcondt', 'water_budget_condt', 0., 30., \ 'Water|Budget|condensation|and|deposition', 'mms-1', 'Reds'] elif varn == 'wbqcm' or varn == 'WBQCM' or varn == 'WBACQCMde': varvals = ['wbqcm', 'water_budget_qcm', 0., 30., \ 'Water|Budget|hydrometeor|change|and|convergence', 'mms-1', 'Reds'] elif varn == 'wbsi' or varn == 'WBSI' or varn == 'WBACSIde': varvals = ['wbsi', 'water_budget_si', 0., 30., \ 'Water|Budget|hydrometeor|sink', 'mms-1', 'Reds'] elif varn == 'wbso' or varn == 'WBSO' or varn == 'WBACSOde': varvals = ['wbso', 'water_budget_so', 0., 30., \ 'Water|Budget|hydrometeor|source', 'mms-1', 'Reds'] # Water Budget accumulated elif varn == 'ccondac' or varn == 'ACCCOND': varvals = ['ccondac', 'cw_cond_ac', 0., 30., \ 'accumulated|cloud|water|condensation', 'mm', 'Reds'] elif varn == 'rac' or varn == 'ACQVAPOR': varvals = ['rac', 'ac_r', 0., 30., 'accumualted|water|wapor', 'mm', 'Blues'] elif varn == 'diabhac' or varn == 'ACDIABH': varvals = ['diabhac', 'diabh_ac', 0., 30., 'accumualted|diabatic|heating', \ 'K', 'Reds'] elif varn == 'wbpwac' or varn == 'WBACPW': varvals = ['wbpwac', 'water_budget_pw_ac', 0., 30., \ 'Water|Budget|accumulated|water|content', 'mm', 'Reds'] elif varn == 'wbfac' or varn == 'WBACF': varvals = ['wbfac', 'water_budget_hfcqv_ac', 0., 30., \ 'Water|Budget|accumulated|horizontal|convergence|of|water|vapour|(+,|' + \ 'conv.;|-,|div.)', 'mm', 'Reds'] elif varn == 'wbfcac' or varn == 'WBACFC': varvals = ['wbfcac', 'water_budget_fc_ac', 0., 30., \ 'Water|Budget|accumulated|horizontal|convergence|of|cloud|(+,|conv.;|-,|' +\ 'div.)', 'mm', 'Reds'] elif varn == 'wbfpac' or varn == 'WBACFP': varvals = ['wbfpac', 'water_budget_cfp_ac', 0., 30., \ 'Water|Budget|accumulated|horizontal|convergence|of|precipitation|(+,|' + \ 'conv.;|-,|div.)', 'mm', 'Reds'] elif varn == 'wbzac' or varn == 'WBACZ': varvals = ['wbzac', 'water_budget_z_ac', 0., 30., \ 'Water|Budget|accumulated|vertical|convergence|of|water|vapour|(+,|conv.' +\ ';|-,|div.)', 'mm', 'Reds'] elif varn == 'wbcac' or varn == 'WBACC': varvals = ['wbcac', 'water_budget_c_ac', 0., 30., \ 'Water|Budget|accumulated|Cloud|water|species','mm', 'Reds'] elif varn == 'wbqvdac' or varn == 'WBACQVD': varvals = ['wbqvdac', 'water_budget_qvd_ac', 0., 30., \ 'Water|Budget|accumulated|water|vapour|divergence', 'mm', 'Reds'] elif varn == 'wbqvbltenac' or varn == 'WBACQVBLTEN': varvals = ['wbqvbltenac', 'water_budget_qv_blten_ac', 0., 30., \ 'Water|Budget|accumulated|QV|tendency|due|to|pbl|parameterization', \ 'kg kg-1 s-1', 'Reds'] elif varn == 'wbqvcutenac' or varn == 'WBACQVCUTEN': varvals = ['wbqvcutenac', 'water_budget_qv_cuten_ac', 0., 30., \ 'Water|Budget|accumulated|QV|tendency|due|to|cu|parameterization', \ 'kg kg-1 s-1', 'Reds'] elif varn == 'wbqvshtenac' or varn == 'WBACQVSHTEN': varvals = ['wbqvshtenac', 'water_budget_qv_shten_ac', 0., 30., \ 'Water|Budget|accumulated|QV|tendency|due|to|shallow|cu|parameterization', \ 'kg kg-1 s-1', 'Reds'] elif varn == 'wbprac' or varn == 'WBACP': varvals = ['wbprac', 'water_budget_pr_ac', 0., 30., \ 'Water|Budget|accumulated|precipitation', 'mm', 'Reds'] elif varn == 'wbpwac' or varn == 'WBACPW': varvals = ['wbpwac', 'water_budget_pw_ac', 0., 30., \ 'Water|Budget|accumulated|water|content', 'mm', 'Reds'] elif varn == 'wbcondtac' or varn == 'WBACCONDT': varvals = ['wbcondtac', 'water_budget_condt_ac', 0., 30., \ 'Water|Budget|accumulated|condensation|and|deposition', 'mm', 'Reds'] elif varn == 'wbqcmac' or varn == 'WBACQCM': varvals = ['wbqcmac', 'water_budget_qcm_ac', 0., 30., \ 'Water|Budget|accumulated|hydrometeor|change|and|convergence', 'mm', 'Reds'] elif varn == 'wbsiac' or varn == 'WBACSI': varvals = ['wbsiac', 'water_budget_si_ac', 0., 30., \ 'Water|Budget|accumulated|hydrometeor|sink', 'mm', 'Reds'] elif varn == 'wbsoac' or varn == 'WBACSO': varvals = ['wbsoac', 'water_budget_so_ac', 0., 30., \ 'Water|Budget|accumulated|hydrometeor|source', 'mm', 'Reds'] elif varn == 'xtime' or varn == 'XTIME': varvals = ['xtime', 'time', 0., 1.e5, 'time', \ 'minutes|since|simulation|start', 'Reds'] elif varn == 'x' or varn == 'X': varvals = ['x', 'x', 0., 100., 'x', '-', 'Reds'] elif varn == 'y' or varn == 'Y': varvals = ['y', 'y', 0., 100., 'y', '-', 'Blues'] elif varn == 'z' or varn == 'Z': varvals = ['z', 'z', 0., 100., 'z', '-', 'Greens'] elif varn == 'zg' or varn == 'WRFght' or varn == 'Geopotential height' or \ varn == 'geop' or varn == 'LGEOP': varvals = ['zg', 'geopotential_height', 0., 80000., 'geopotential|height', \ 'm2s-2', 'rainbow'] elif varn == 'zmaxth' or varn == 'zmax_th' or varn == 'LZMAX_TH': varvals = ['zmaxth', 'thermal_plume_height', 0., 4000., \ 'maximum|thermals|plume|height', 'm', 'YlOrRd'] elif varn == 'zmla' or varn == 's_pblh' or varn == 'LS_PBLH': varvals = ['zmla', 'atmosphere_boundary_layer_thickness', 0., 2500., \ 'atmosphere|boundary|layer|thickness', 'm', 'Blues'] else: print errormsg print ' ' + fname + ": variable '" + varn + "' not defined !!!" quit(-1) return varvals def numVector_String(vec,char): """ Function to transform a vector of numbers to a single string [char] separated numVector_String(vec,char) vec= vector with the numerical values char= single character to split the values >>> print numVector_String(np.arange(10),' ') 0 1 2 3 4 5 6 7 8 9 """ fname = 'numVector_String' # if vec == 'h': # print fname + '_____________________________________________________________' # print numVector_String.__doc__ # quit() Nvals = len(vec) string='' for i in range(Nvals): if i == 0: string = str(vec[i]) else: string = string + char + str(vec[i]) return string def index_mat(mat,val): """ Function to provide the coordinates of a given value inside a matrix index_mat(mat,val) mat= matrix with values val= value to search >>> index_mat(np.arange(27).reshape(3,3,3),22) [2 1 1] """ fname = 'index_mat' matshape = mat.shape matlist = list(mat.flatten()) ifound = matlist.index(val) Ndims = len(matshape) valpos = np.zeros((Ndims), dtype=int) baseprevdims = np.zeros((Ndims), dtype=int) for dimid in range(Ndims): baseprevdims[dimid] = np.product(matshape[dimid+1:Ndims]) if dimid == 0: alreadyplaced = 0 else: alreadyplaced = np.sum(baseprevdims[0:dimid]*valpos[0:dimid]) valpos[dimid] = int((ifound - alreadyplaced )/ baseprevdims[dimid]) return valpos def multi_index_mat(mat,val): """ Function to provide the multiple coordinates of a given value inside a matrix index_mat(mat,val) mat= matrix with values val= value to search >>> vals = np.ones((24), dtype=np.float).reshape(2,3,4) vals[:,:,2] = 0. vals[1,:,:] = np.pi vals[:,2,:] = -1. multi_index_mat(vals,1.) [array([0, 0, 0]), array([0, 0, 1]), array([0, 0, 3]), array([0, 1, 0]), array([0, 1, 1]), array([0, 1, 3])] """ fname = 'multi_index_mat' matshape = mat.shape ivalpos = [] matlist = list(mat.flatten()) Lmatlist = len(matlist) val0 = val - val if val != val0: valdiff = val0 else: valdiff = np.ones((1), dtype = type(val)) ifound = 0 while ifound < Lmatlist: if matlist.count(val) == 0: ifound = Lmatlist + 1 else: ifound = matlist.index(val) Ndims = len(matshape) valpos = np.zeros((Ndims), dtype=int) baseprevdims = np.zeros((Ndims), dtype=int) for dimid in range(Ndims): baseprevdims[dimid] = np.product(matshape[dimid+1:Ndims]) if dimid == 0: alreadyplaced = 0 else: alreadyplaced = np.sum(baseprevdims[0:dimid]*valpos[0:dimid]) valpos[dimid] = int((ifound - alreadyplaced )/ baseprevdims[dimid]) matlist[ifound] = valdiff ivalpos.append(valpos) return ivalpos def addfileInfile(origfile,destfile,addfile,addsign): """ Function to add the content of a file [addfile] to another one [origfile] at a given position [addsign] creating a new file called [destfile] origfile= original file destfile= destination file addfile= content of the file to add addsign= sign where to add the file """ fname = 'addfileInfile' if not os.path.isfile(origfile): print errormsg print ' ' + fname + ": original file '" + origfile + "' does not exist !!" quit() if not os.path.isfile(addfile): print errormsg print ' ' + fname + ": adding file '" + addfile + "' does not exist !!" quit() ofile = open(origfile, 'r') # Inspecting flag ## Nfound = 0 for line in ofile: if line == addsign + '\n': Nfound = Nfound + 1 if Nfound == 0: print errormsg print ' ' + fname + ": adding sign '" + addsign + "' not found !!" quit() print ' '+ fname + ': found', Nfound," times sign '" + addsign + "' " ofile.seek(0) dfile = open(destfile, 'w') for line in ofile: if line != addsign + '\n': dfile.write(line) else: afile = open(addfile,'r') for aline in afile: print aline dfile.write(aline) afile.close() ofile.close() dfile.close() print main + " successful writting of file '" + destfile + "' !!" return ####### ###### ##### #### ### ## # def valmodoper(varVal, valuesS): """ Function to run the modification of a variable varVAl= matrix with the values valuesS= [modins],[[modval1],...,[modvalN]] modification instruction, value with which modify [modins]: Modifiers: sumc: add [modval1] subc: remove [modval1] mulc: multiply by [modval1] divc: divide by [modval1] lowthres: if [val] < [modval1]; val = [modval2] upthres: if [val] > [modval1]; val = [modval2] lowthres@oper: if [val] < [modval1]; val = [oper] (operation as [modval2],[modval3]) upthres@oper: if [val] > [modval1]; val = [oper] (operation as [modval2],[modval3]) potc: [val] ** [modval1] """ fname='valmodoper' if valuesS == 'h': print fname + '_____________________________________________________________' print valmodoper.__doc__ quit() valsS = valuesS.split(',') modins = valsS[0] modval = float(valsS[1]) if modins == 'sumc': varVal[:] = varVal[:] + modval elif modins == 'subc': varVal[:] = varVal[:] - modval elif modins == 'mulc': varVal[:] = varVal[:] * modval elif modins == 'divc': varVal[:] = varVal[:] / modval elif modins == 'lowthres': varVal2 = np.where(varVal[:] < float(valsS[1]), float(valsS[2]), varVal[:]) varVal[:] = varVal2 elif modins == 'upthres': varVal2 = np.where(varVal[:] > float(valsS[1]), float(valsS[2]), varVal[:]) varVal[:] = varVal2 elif modins == 'lowthres@oper': if valsS[2] == 'sumc': varVal2 = np.where(varVal[:] < float(valsS[1]), \ varVal[:] + float(valsS[3]), varVal[:]) varVal[:] = varVal2 elif valsS[2] == 'subc': varVal2 = np.where(varVal[:] < float(valsS[1]), \ varVal[:] - float(valsS[3]), varVal[:]) varVal[:] = varVal2 elif valsS[2] == 'mulc': varVal2 = np.where(varVal[:] < float(valsS[1]), \ varVal[:] * float(valsS[3]), varVal[:]) varVal[:] = varVal2 elif valsS[2] == 'divc': varVal2 = np.where(varVal[:] < float(valsS[1]), \ varVal[:] / float(valsS[3]), varVal[:]) varVal[:] = varVal2 elif valsS[2] == 'potc': varVal2 = np.where(varVal[:] < float(valsS[1]), \ varVal[:] ** float(valsS[3]), varVal[:]) varVal[:] = varVal2 else: print errormsg print ' ' + fname + ": Operation to modify values '" + modins + \ "' is not defined !!" quit(-1) elif modins == 'upthres@oper': if valsS[2] == 'sumc': varVal2 = np.where(varVal[:] > float(valsS[1]), \ varVal[:] + float(valsS[3]), varVal[:]) varVal[:] = varVal2 elif valsS[2] == 'subc': varVal2 = np.where(varVal[:] > float(valsS[1]), \ varVal[:] - float(valsS[3]), varVal[:]) varVal[:] = varVal2 elif valsS[2] == 'mulc': varVal2 = np.where(varVal[:] > float(valsS[1]), \ varVal[:] * float(valsS[3]), varVal[:]) varVal[:] = varVal2 elif valsS[2] == 'divc': varVal2 = np.where(varVal[:] > float(valsS[1]), \ varVal[:] / float(valsS[3]), varVal[:]) varVal[:] = varVal2 elif valsS[2] == 'potc': varVal2 = np.where(varVal[:] > float(valsS[1]), \ varVal[:] ** float(valsS[3]), varVal[:]) varVal[:] = varVal2 else: print errormsg print ' ' + fname + ": Operation to modify values '" + modins + \ "' is not defined !!" quit(-1) elif modins == 'potc': varVal[:] = varVal[:] ** modval else: print errormsg print ' ' + fname + ": Operation to modify values '" + modins + \ "' is not defined !!" quit(-1) return varVal def valmod(values, ncfile, varn): """ Function to modify the value of a variable values = modins,[modval1],[...,[modvalN]] modins = instruction: 'sumc', add [modval1] 'subc', substraction [modval1] 'mulc', multiply by [modval1] 'divc', divide by [modval1] 'lowthres': modify all values below [modval1] to [modval2] 'upthres': modify all values above [modval1] to [modval2] 'potc': [val] ** [modval1] ncfile = netCDF file name varn = name of the variable """ fname='valmod' if values == 'h': print fname + '_____________________________________________________________' print valmod.__doc__ quit() vals = values.split(',') modins = vals[0] modval = float(vals[1]) if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ": File '" + ncfile + "' does not have variable '" + \ varn + "' !!" print errormsg ncf.close() quit(-1) var = ncf.variables[varn] varshape = var.shape vartype = var.dtype Ndims = len(varshape) varshapevals = list(varshape) Nvars = len(varshape) if not Nvars == 0: if Nvars == 1: varvals = np.zeros(varshape[0], dtype=vartype) elif Nvars == 2: varvals = np.zeros((varshape[0], varshape[1]), dtype=vartype) elif Nvars == 3: varvals = np.zeros((varshape[1], varshape[2]), dtype=vartype) elif Nvars == 4: varvals = np.zeros((varshape[2], varshape[3]), dtype=vartype) elif Nvars == 5: varvals = np.zeros((varshape[3], varshape[4]), dtype=vartype) elif Nvars == 6: varvals = np.zeros((varshape[4], varshape[5]), dtype=vartype) else: print errormsg print ' ' + fname + ': variable size ',Nvars,' is not ready!!!!' # Allocating all the necessary memory (just in case) if Nvars <= 2: varvals = var[:] var[:] = valmodoper(varvals, values) elif Nvars == 3: for i in range(varshape[0]): varvals[:] = var[i,:,:] var[i,:,:] = valmodoper(varvals, values) elif Nvars == 4: for i in range(varshape[0]): for j in range(varshape[1]): varvals[:] = var[i,j,:,:] var[i,j,:,:] = valmodoper(varvals, values) elif Nvars == 5: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): varvals[:] = var[i,j,k,:,:] var[i,j,k,:,:] = valmodoper(varvals, values) elif Nvars == 6: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): for l in range(varshape[3]): varvals[:] = var[i,j,k,l,:,:] var[i,j,k,l,:,:] = valmodoper(varvals, values) ncf.sync() ncf.close() def rangedim(end, shape): """Gives the instruction to retrieve values from a dimension of a variable >>> print rangedim(-1, 15) 15 """ if end == -1: return shape else: return end def varaddref(values, ncfile, varn): """ Function to add a variable in an existing file copying characteristics from an existing one values = [variable ref]:[attr name]@[value][:[attr2]@[value2], ...]:[value/file with values] add a new variable [varn] with dimension and attributes from an already existing [variable ref] with attributes [[attr name]@[value][:[attr2]@[value2], ...]] in the file [netcdf] and value [value/file with values] netcdf = netCDF file name varn = new variable name """ fname = 'varaddref' if values == 'h': print fname + '_____________________________________________________________' print varaddref.__doc__ quit() varvalues = values.split(':') Nvarvalues = len(varvalues) varprev = varvalues[0] newattrs = {} for iattr in range(Nvarvalues - 2): attrv = varvalues[iattr+1] newattrs[attrv.split('@')[0]] = attrv.split('@')[1] ncf = NetCDFFile(ncfile,'a') if ncf.variables.has_key(varn): print errormsg print ' varaddref: File already has the varible ' + varn + ' !!!' print errormsg ncf.close() quit(-1) if not ncf.variables.has_key(varprev): print errormsg print ' varaddref: File does not have variable ' + varprev + ' !!!!' print errormsg ncf.close() quit(-1) varp = ncf.variables[varprev] varpVal = varp[:] varpshape = varp.shape Nvarpshape = len(varpshape) varpattrs = varp.ncattrs() varptype = varp.dtype varpdims = varp.dimensions varpndims = varp.ndim ## print ' shape of the variable used as reference: ',varpshape # # variable characteristics if len(varpshape) == 4: dimpt = varpshape[0] dimpz = varpshape[1] dimpy = varpshape[2] dimpx = varpshape[3] elif len(varpshape) == 3: dimpt = varpshape[0] dimpy = varpshape[1] dimpx = varpshape[2] elif len(varpshape) == 3: dimpy = varpshape[0] dimpx = varpshape[1] newvar = ncf.createVariable(varn, varptype, dimensions=varpdims) newvar = ncf.variables[varn] # # Values varplen=1 for idim in range(varpndims): varplen=varplen*varpshape[idim] if varptype == 'float' or varptype == 'float32' or varptype == 'float64' or vartype == np.float(1.) or vartype == np.float32(1.): newvals = np.reshape(np.arange(varplen), varpshape)*1. else: newvals = np.reshape(np.arange(varplen), varpshape) if not os.path.isfile(varvalues[Nvarvalues-1]): ## print ' Using constant value' newvals[:] = float(varvalues[Nvarvalues-1]) else: ## print ' Using 2-D values from ' + varvalues[Nvarvalues-1] asciif = open(varvalues[Nvarvalues-1], 'r') fdimx = len(asciif.readlines()) asciif.seek(0) if len(varpshape) == 4: if not fdimx == dimpx: print errormsg print ' varaddref: Provided file has dimx=', fdimx, ' and variable has dimx=', dimpx, ' !!!' print errormsg ncf.close() quit(-1) iline = 0 idx = 0 for fline in asciif: line = fline.replace('\n','') yvals = [] for iyval in line.split(' '): yvals.append(float(iyval)) if iline == 0: fdimy = len(yvals) if not fdimy == dimpy: print errormsg print ' varaddref: Provided file has dimy=', fdimy, ' and variable has dimy= ', dimpy, ' !!!' print errormsg ncf.close() quit(-1) for it in range(dimpt): for iz in range(dimpz): newvals[it,iz,:,idx] = yvals idx = idx+1 iline = iline+1 elif len(varpshape) == 3: if not fdimx == dimpx: print errormsg print ' varaddref: Provided file has dimx=', fdimx, ' and variable has dimx=', dimpx, ' !!!' print errormsg ncf.close() quit(-1) iline = 0 idx = 0 for fline in asciif: line = fline.replace('\n','') yvals = [] for iyval in line.split(' '): yvals.append(float(iyval)) if iline == 0: fdimy = len(yvals) if not fdimy == dimpy: print errormsg print ' varaddref: Provided file has dimy=', fdimy, ' and variable has dimy= ',dimpy, ' !!!' print errormsg ncf.close() quit(-1) for it in range(dimpt): newvals[it,:,idx] = yvals idx = idx+1 iline = iline+1 elif len(varpshape) == 2: if not fdimx == dimpx: print errormsg print ' varaddref: Provided file has dimx=', fdimx, ' and variable has dimx=', dimpx, ' !!!' print errormsg ncf.close() quit(-1) iline = 0 idx = 0 for fline in asciif: line = fline.replace('\n','') yvals = [] for iyval in line.split(' '): yvals.append(float(iyval)) if iline == 0: fdimy = len(yvals) if not fdimy == dimpy: print errormsg print ' varaddref: Provided file has dimy=', fdimy, ' and variable has dimy= ',dimpy, ' !!!' print errormsg ncf.close() quit(-1) newvals[:,idx] = yvals idx = idx+1 iline = iline+1 asciif.close() newvar[:] = newvals # # Attributes for iattr in range(len(varpattrs)): attrn = varpattrs[iattr] attrval = varp.getncattr(attrn) if attrn in newattrs: newattr = newvar.setncattr(attrn, newattrs[attrn]) else: newattr = newvar.setncattr(attrn, attrval) ncf.sync() ncf.close() def varoutold(values, ncfile, varn): """ Function when we want to output variable values values = [optsIrange]:[optsErange] [optsIrange]: val1,val2,...,valN inital value for the 'N' dimensions of the variable [optsErange]: val1,val2,...,valN ending value for the 'N' dimensions of the variable ncfile = netCDF file name varn = variable name """ import numpy as np from netCDF4 import Dataset as NetCDFFile optsIrange = values.split(':')[0] optsErange = values.split(':')[1] inirange=optsIrange.split(',') endrange=optsErange.split(',') irange = [int(val) for val in inirange] erange = [int(val) for val in endrange] if not len(irange) == len(erange): print errormsg print ' varout: Different number of values in each range!' print ' varout: initial range: ' + optsIrange print ' varout: ending range: ' + optsErange print errormsg quit(-1) else: ndims=len(irange) if not os.path.isfile(ncfile): print errormsg print ' varout: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'r') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' varout: File "' + ncfile + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) var = ncf.variables[varn] varshape = var.shape Nvarshape = len(varshape) if not Nvarshape == ndims: print errormsg print ' varout: Provided number of values of the range ' + ndims + ' is different of the shape of the variable ' + Nvarshape + ' !!!' print errormsg ncf.close() quit(-1) if Nvarshape == 1: varValrange = var[irange[0]:rangedim(erange[0], varshape[0])] elif Nvarshape == 2: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1]), varshape[1]] elif Nvarshape == 3: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1], varshape[1]), irange[2]:rangedim(erange[2], varshape[2])] elif Nvarshape == 4: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1], varshape[1]), irange[2]:rangedim(erange[2], varshape[2]), \ irange[3]:rangedim(erange[3], varshape[3])] elif Nvarshape == 5: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1], varshape[1]), irange[2]:rangedim(erange[2], varshape[2]), \ irange[3]:rangedim(erange[3], varshape[3]), irange[4]:rangedim(erange[4], varshape[4])] elif Nvarshape == 6: varValrange = var[irange[0]:rangedim(erange[0], varshape[0]), \ irange[1]:rangedim(erange[1], varshape[1]), irange[2]:rangedim(erange[2], varshape[2]), \ irange[3]:rangedim(erange[3], varshape[3]), irange[4]:rangedim(erange[4], varshape[4]), \ irange[5]:rangedim(erange[5], varshape[5])] ncf.close() Nshaperange = len(varValrange.shape) Noneshape = 0 for ir in range(Nshaperange): if varValrange.shape[ir] == 1: Noneshape = Noneshape + 1 if Noneshape == Nshaperange - 1: for i in range(len(varValrange)): print '%2s %f' % ( 'NC', varValrange[i] ) def varout(values, ncfile, varn): """ Function when we want to output variable values values = [dimname1]:[valdim1]|[dimname2]:[valdim2]|[...,[dimnameN]:[valdimN]] [valdim]: * [integer]: which value of the dimension * -1: all along the dimension * -9: last value of the dimension * [beg]:[end] slice from [beg] to [end] * NOTE, no dim name all the dimension size ncfile = netCDF file name varn = variable name """ fname = 'varout' if values == 'h': print fname + '_____________________________________________________________' print varout.__doc__ quit() ncobj = NetCDFFile(ncfile, 'r') if not ncobj.variables.has_key(varn): print errormsg print ' ' + fname + ': file "' + ncfile + '" does not have variable "' + \ varn + '" !!' quit(-1) vobj = ncobj.variables[varn] newvar, newdims = slice_variable(vobj, values) Nnewdims = len(newdims) if Nnewdims == 0: print '%2s %f' % ( 'NC',newvar ) elif Nnewdims == 1: for i in range(newvar.shape[0]): print '%2s %f' % ( 'NC',newvar[i] ) elif Nnewdims == 2: for i0 in range(newvar.shape[0]): row = numVector_String(newvar[i0,:],' ') print 'NC ' + row else: for i0 in range(newvar.shape[Nnewdims-2]): varslice = [] dimsinf = '' for idim in range(Nnewdims - 2): for d in range(newvar.shape[idim]): if idim == 0: dimsinf = newdims[idim] + ': ' + str(d) else: dimsinf = dimsinf + ' ' + newdims[idim] + ': ' + str(d) varslice.append(d) varslice.append(i0) varslice.append(0,newvar.shape[Nnewdims-1]) row = numVector_String(newvar[tuple(varslice)],' ') print 'NCinf ' + dimsinf + '_________________' print 'NC ' + row ncobj.close() return def chdimname(values, ncfile, varn): """ Changing the name of the dimension values = [olddimname]:[newdimname] [olddimname]: old name of the dimension [newdimname]: new name of the dimension ncfile = netCDF file name varn = variable name """ fname = 'chdimname' if values == 'h': print fname + '_____________________________________________________________' print chdimname.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' chdimname: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') olddimname=values.split(':')[0] newdimname=values.split(':')[1] if not ncf.dimensions.has_key(olddimname): print warnmsg print ' chdimname: File "' + ncfile + '" does not have dimension "' + olddimname + '" !!!!' ncf.close() quit() if not olddimname == newdimname and not ncf.dimensions.has_key(newdimname): newname = ncf.renameDimension(olddimname, newdimname) ncf.sync() ncf.close() else: print warnmsg print ' chdimname: File "' + ncfile + '" already has dimension name "' + newdimname + '" ' print ' chdimname: modifying all the variables which use the old dimension' filevars = ncf.variables for fvarn in filevars: if ncf.variables.has_key(fvarn): fvar = ncf.variables[fvarn] fvardims = fvar.dimensions if searchInlist(fvardims, olddimname): print ' variable "' + fvarn + '" uses dimension "' + olddimname + '" ' varinf = variable_inf(fvar) newdims = tuple(fvardims) change = {olddimname: newdimname} # From http://stackoverflow.com/questions/9067043/python-replace-list-values-using-a-tuple newdims = tuple([ change.get(x,x) for x in fvardims ]) newvar = ncf.createVariable(fvarn + 'tmpdname', varinf.dtype, newdims, fill_value=varinf.FillValue) varv = fvar[:] newvar[:] = varv for attrn in varinf.attributes: attrv = fvar.getncattr(attrn) newar = newvar.setncattr(attrn, attrv) fvar = ncf.renameVariable(fvarn, fvarn + 'rmdname') ncf.sync() newvar = ncf.renameVariable(fvarn + 'tmpdname', fvarn) ncf.sync() ncf.close() varrm(ncfile, fvarn + 'rmdname') ncf = NetCDFFile(ncfile,'a') def chvarname(values, ncfile, varn): """Changing the name of the variable values = new variable name ncfile = netCDF file varn = name of the variable """ fname = 'chvarname' if values == 'h': print fname + '_____________________________________________________________' print chvarname.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print warnmsg print ' ' + fname + ': File does not have variable "' + varn + '" !!!!' ncf.close() quit(-1) if not varn == values and not ncf.variables.has_key(values): newname = ncf.renameVariable(varn, values) ncf.sync() ncf.close() def searchInlist(listname, nameFind): """ Function to search a value within a list listname = list nameFind = value to find >>> searInlist(['1', '2', '3', '5'], '5') True """ for x in listname: if x == nameFind: return True return False def set_attribute(ncv, attrname, attrvalue): """ Sets a value of an attribute of a netCDF variable. Removes previous attribute value if exists ncv = object netcdf variable attrname = name of the attribute attrvalue = value of the attribute """ attvar = ncv.ncattrs() if searchInlist(attvar, attrname): attr = ncv.delncattr(attrname) attr = ncv.setncattr(attrname, attrvalue) return attr def set_attributek(ncv, attrname, attrval, attrkind): """ Sets a value of an attribute of a netCDF variable with a kind. Removes previous attribute value if exists ncvar = object netcdf variable attrname = name of the attribute attrvalue = value of the attribute atrtrkind = kind of attribute: 'S', string ('!' as spaces); 'U', unicode ('!' as spaces); 'I', integer; 'Inp32', numpy integer 32; 'R', real; ' npfloat', np.float; 'D', np.float64 """ fname = 'set_attributek' validk = {'S': 'string', 'U': 'unicode', 'I': 'integer', \ 'Inp32': 'integer long (np.int32)', 'R': 'float', 'npfloat': 'np.float', \ 'D': 'float long (np.float64)'} if type(attrkind) == type('s'): if attrkind == 'S': attrvalue = str(attrval.replace('!', ' ')) elif attrkind == 'U': attrvalue = unicode(attrval.replace('!',' ')) elif attrkind == 'I': attrvalue = np.int(attrval) elif attrkind == 'R': attrvalue = float(attrval) elif attrkind == 'npfloat': attrvalue = np.float(attrval) elif attrkind == 'D': attrvalue = np.float64(attrval) else: print errormsg print ' ' + fname + ": '" + attrkind + "' kind of attribute is not ready!" print ' valid values: _______' for key in validk.keys(): print ' ', key,':', validk[key] quit(-1) else: if attrkind == type(str('a')): attrvalue = str(attrval.replace('!', ' ')) elif attrkind == type(unicode('a')): attrvalue = unicode(attrval.replace('!',' ')) elif attrkind == type(np.int(1)): attrvalue = np.int(attrval) elif attrkind == np.dtype('i'): attrvalue = np.int32(attrval) elif attrkind == type(float(1.)): attrvalue = float(attrval) elif attrkind == type(np.float(1.)): attrvalue = np.float(attrval) elif attrkind == np.dtype('float32'): attrvalue = np.array(attrval, dtype='f') elif attrkind == type(np.float32(1.)): attrvalue = np.float32(attrval) elif attrkind == type(np.float64(1.)): attrvalue = np.float64(attrval) else: print errormsg print ' ' + fname + ": '" + attrkind + "' kind of attribute is not ready!" print ' valid values: _______' for key in validk.keys(): print ' ', key,':', validk[key] quit(-1) attvar = ncv.ncattrs() if searchInlist(attvar, attrname): attr = ncv.delncattr(attrname) attr = ncv.setncattr(attrname, attrvalue) return attr def basicvardef(varobj, vstname, vlname, vunits): """ Function to give the basic attributes to a variable varobj= netCDF variable object vstname= standard name of the variable vlname= long name of the variable vunits= units of the variable """ attr = varobj.setncattr('standard_name', vstname) attr = varobj.setncattr('long_name', vlname) attr = varobj.setncattr('units', vunits) return def gaddattr(values, ncfile): """ Add a global attribute to a netCDF. Removes previous attribute if it exist values = [attrname]|[attrvalue ('!' as spaces)] ncfile = netCDF file """ if not os.path.isfile(ncfile): print errormsg print ' gaddattr: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') attrvals=values.split('|') attrn=attrvals[0] attrv=attrvals[1].replace('!', ' ') ncf = set_attribute(ncf, attrn, attrv) ncf.sync() ncf.close() def gaddattrk(values, ncfile): """ Add a global attribute to a netCDF caring about the type. Removes previous attribute if it exist values = [attrname]|[attrvalue]|[attrk] attrname = name of the attribute attrvalue = value of the attribute attrk = 'S', string ('!' as spaces); 'I', integer; 'Inp32', numpy integer 32; 'R', real; 'D', double ncfile = netCDF file """ if not os.path.isfile(ncfile): print errormsg print ' gaddattrk: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') attrvals=values.split('|') attrn=attrvals[0] attrv0=attrvals[1] attrk=attrvals[2] if attrk == 'S': attrv = str(attrv0.replace('!', ' ')) elif attrk == 'I': attrv = int(attrv0) elif attrk == 'Inp32': attrv = np.int32(attrv0) elif attrk == 'R': attrv = float(attrv0) elif attrk == 'D': attrv = np.float64(attrv0) else: print errormsg print ' gaddattrk: "' + attrk + '" kind of attribute is not ready!' ncf.close() quit(-1) ncf = set_attribute(ncf, attrn, attrv) ncf.sync() ncf.close() def varaddattr(values, ncfile, varn): """ Add an attribute to a variable. Removes previous attribute if it exists values = [attrname]|[attrvalue('!' as spaces)] ncfile = netCDF file name varn = name of the variable """ fname = 'varaddattr' if values == 'h': print fname + '_____________________________________________________________' print varaddattr.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname +': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname +': File "' + ncfile + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) attrvals=values.split('|') attrn=attrvals[0] attrv=attrvals[1].replace('!', ' ') var = ncf.variables[varn] var = set_attribute(var, attrn, attrv) ncf.sync() ncf.close() def varaddattrk(values, ncfile, varn): """ Add an attribute to a variable caring about the type values = [attrname]|[attrvalue]|[attrk] attrname = name of the attribute attrvalue = value of the attribute attrk = 'S', string ('!' as spaces); 'I', integer; 'R', real; 'D', double; 'npR', numpy Real, 'npR32', numpy Real-32 ncfile = netCDF file varn = variable name """ fname = 'varaddattrk' if values == 'h': print fname + '_____________________________________________________________' print varaddattrk.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ': File "' + ncfile + '"does not have variable ' + varn + ' !!!!' print errormsg ncf.close() quit(-1) attrvals=values.split('|') attrn=attrvals[0] attrv0=attrvals[1] attrk=attrvals[2] var = ncf.variables[varn] if attrk == 'S': attrv = str(attrv0.replace('!', ' ')) elif attrk == 'I': attrv = int(attrv0) elif attrk == 'R': attrv = float8(attrv0) elif attrk == 'D': attrv = np.float64(attrv0) elif attrk == 'npR': attrv = np.float(attrv0) elif attrk == 'npR32': attrv = np.float32(attrv0) else: print errormsg print ' ' + fname + ': "' + attrk + '" kind of attribute is not ready!' ncf.close() quit(-1) var = set_attribute(var, attrn, attrv) ncf.sync() ncf.close() def varrmattr(values, ncfile, varn): """ Removing an attribute from a variable values = attribute name ncfile = netCDF file name varn = variable name """ fname = 'varrmattr' if values == 'h': print fname + '_____________________________________________________________' print varrmattr.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname +': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname +': File "' + ncfile + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) var = ncf.variables[varn] attvar = var.ncattrs() if searchInlist(attvar, values): attr = var.delncattr(values) else: print warnmsg print ' ' + fname +': "' + varn + '" does not have attribute: ' + values ncf.sync() ncf.close() def grmattr(values, ncfile): """ Removing a global attribute values = attribute name ncfile = netCDF file """ if not os.path.isfile(ncfile): print errormsg print ' grmattr: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') attvar = ncf.ncattrs() if searchInlist(attvar, values): attr = ncf.delncattr(values) else: print warnmsg print ' grmattr: "' + ncfile + '" does not have attribute: ' + values ncf.sync() ncf.close() def fvaradd(values, ncfile): """ Adding variable (and all its attributes and dimensions) from a reference file to a file values = [netCDFref],[varnref] netCDFref = netCDF file name as reference for the variable to add varnref = name of the variable from [netCDFref] to be added ncfile = netCDF file name """ fname = 'fvaradd' if values == 'h': print fname + '_____________________________________________________________' print fvaradd.__doc__ quit() refnc = values.split(',')[0] refvar = values.split(',')[1] if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) if not os.path.isfile(refnc): print errormsg print ' ' + fname + ': Reference file "' + refnc + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') refvars = ncref.variables if searchInlist(refvars, refvar): refvarv = ncref.variables[refvar] else: print errormsg print ' ' + fname + ': File "' + refnc + '" does not have variable: ' + refvar ncf.close() ncref.close() quit(-1) vardims = refvarv.dimensions vartype = refvarv.dtype varattr = refvarv.ncattrs() # Checking dimensions ## newdims = ncf.dimensions for rdim in vardims: if not searchInlist(newdims, rdim): print ' ' + fname + ': Adding dimension ' + rdim ncf.close() ncref.close() fdimadd(refnc + ',' + rdim, ncfile) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') # Checking fill value ## if searchInlist(varattr, '_FillValue'): varfil = refvarv._FillValue else: varfil = False if not ncf.variables.has_key(refvar): print ' ' + fname + ': Adding refvar:', refvar, 'shape: ', refvarv.shape var = ncf.createVariable(refvar, vartype, vardims, fill_value=varfil) varshape = refvarv.shape varvals = np.zeros(tuple(varshape), dtype=vartype) Nvars = len(varshape) # Allocating all the necessary memory (just in case) var[:] = varvals if Nvars <= 2: varvals = refvarv[:] var[:] = varvals elif Nvars == 3: for i in range(varshape[0]): varvals[i,:,:] = refvarv[i,:,:] var[i,:,:] = varvals[i,:,:] elif Nvars == 4: for i in range(varshape[0]): for j in range(varshape[1]): varvals[i,j,:,:] = refvarv[i,j,:,:] var[i,j,:,:] = varvals[i,j,:,:] elif Nvars == 5: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): varvals[i,j,k,:,:] = refvarv[i,j,k,:,:] var[i,j,k,:,:] = varvals[i,j,k,:,:] elif Nvars == 6: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): for l in range(varshape[3]): varvals[i,j,k,l,:,:] = refvarv[i,j,k,l,:,:] var[i,j,k,l,:,:] = varvals[i,j,k,l,:,:] newvar = ncf.variables[refvar] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = refvarv.getncattr(attr) if not searchInlist(newvarattrs, attr): newvar.setncattr(attr, attrv) ncf.sync() ncf.close() ncref.close() return def fdimadd(values, ncfile): """ Adding dimension from another reference file values = [refnc],[refdim] refnc = netCDF file name as reference refdim = name of the dimension to be added ncfile = netCDF file name """ fname='fdimadd' refnc = values.split(',')[0] refdim = values.split(',')[1] if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' print errormsg quit(-1) if not os.path.isfile(refnc): print errormsg print ' ' + fname + ': Reference file "' + refnc + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') refdims = ncref.dimensions if searchInlist(refdims, refdim): refdimv = ncref.dimensions[refdim] else: print errormsg print ' ' + fname + ': File "' + refnc + '" does not have dimension: "' + refdim + '"' ncf.close() ncref.close() quit(-1) if refdimv.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(refdimv) print ' ' + fname + ': Adding refdim:', refdim, 'size:', dimsize dim = ncf.createDimension(refdim, dimsize) ncf.sync() ncf.close() ncref.close() def fattradd(var, values, ncfile): """ Adding attributes from a reference file var = variable to which has to be added the attribute values = [refnc]:[refvar] refnc = netCDF file name as reference refvar = variable from the reference file ncfile = netCDF file name """ refnc = values.split(':')[0] refvar = values.split(':')[1] if not os.path.isfile(ncfile): print errormsg print ' fattradd: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) if not os.path.isfile(refnc): print errormsg print ' fattradd: Reference file "' + refnc + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') vars = ncf.variables if searchInlist(vars, var): varv = ncf.variables[var] else: print ' fattradd: File "' + ncfile + '" does not have variable: "' + var + '"' ncf.close() ncref.close() quit(-1) refvars = ncref.variables if searchInlist(refvars, refvar): refvarv = ncref.variables[refvar] else: print ' fattradd: File "' + refnc + '" does not have variable: "' + refvar + '"' ncf.close() ncref.close() quit(-1) refvarattrs = refvarv.ncattrs() Nattrs = len(refvarattrs) print ' fattradd: Adding ', Nattrs,' atributes from:', refvar for attr in refvarattrs: attrv = refvarv.getncattr(attr) atvar = set_attribute(varv, attr, attrv) ncf.sync() ncf.close() ncref.close() def fgaddattr(values, ncfile): """ Adding global attributes from a reference file values = netCDF file name as reference ncfile = netCDF file name """ refnc = values if not os.path.isfile(ncfile): print errormsg print ' fgaddattr: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) if not os.path.isfile(refnc): print errormsg print ' fgaddattr: Reference file "' + refnc + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') ncref = NetCDFFile(refnc,'r') refgattrs = ncref.ncattrs() Nattrs = len(refgattrs) print ' fgaddattr: Adding ', Nattrs,' global atributes' for attr in refgattrs: attrv = ncref.getncattr(attr) atvar = set_attribute(ncf, attr, attrv) ncf.sync() ncf.close() ncref.close() return def varrm(ncfile, var): """ Removing a variable from a file ncfile = netCDF object file var = variable name to remove """ import shutil as shu fname = 'varrm' if var == 'h': print fname + '_____________________________________________________________' print varrm.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' varrm: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') ncvars = ncf.variables ncf.close() if not searchInlist(ncvars, var): print ' varrm: File "' + ncfile + '" does not have variable: "' + var + '"' ncf.close() quit(-1) tmpncf = NetCDFFile('tmp_py.nc' , 'w') gtmpattr = set_attribute(tmpncf, 'copy', 'temporal') tmpncf.sync() tmpncf.close() for varn in ncvars: if not varn == var: fvaradd(ncfile + ',' + varn, 'tmp_py.nc') fgaddattr(ncfile, 'tmp_py.nc') shu.copyfile('tmp_py.nc', ncfile) os.remove('tmp_py.nc') def ivars(ncfile): """Give all the variable names of a file ncfile= netCDFfile name from which all variables will be given """ fname = 'ivars' if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not exist !!' quit(-1) ncf = NetCDFFile(ncfile,'r') ncvars = ncf.variables allvars='' for var in ncvars: print var allvars=allvars + ':' + var print ' # allvars= ' + allvars ncf.close() def igattrs(ncfile): """Give all the global attributes of a file """ if not os.path.isfile(ncfile): print errormsg print ' igattrs: File "' + ncfile + '" does not exist !!' print errormsg quit() ncf = NetCDFFile(ncfile,'r') ncattrs = ncf.ncattrs() allattrs='' allattrsdic = {} for attr in ncattrs: attrv=ncf.getncattr(attr) if type(attrv) == type(str('1')): attrk = 'S' elif type(attrv) == type(unicode('1')): attrk = 'S' elif type(attrv) == type(int(1)): attrk = 'I' elif type(attrv) == type(np.int(1)): attrk = 'Inp' elif type(attrv) == type(np.int32(1)): attrk = 'Inp32' elif type(attrv) == type(float(1.)): attrk = 'R' elif type(attrv) == type(np.float32(1.)): attrk = 'Rnp32' elif type(attrv) == type(np.float64(1.)): attrk = 'Rnp64' else: print errormsg print ' igattr: Reading attribute "', type(attrv), '" not ready! value:', attrv ncf.close() quit(-1) print attr, '|', attrv, '|', attrk allattrsdic[attr] = [attrv, attrk] allattrs=allattrs + ':' + attr + '|' + str(attrv) + '|' + attrk print '####### ###### ##### #### ### ## #' print '# allgattrs= ' + allattrs ncf.close() return allattrsdic def isgattrs(values, ncfile): """Give a single global attribute of a file and its type values = attribute name ncfile = netCDF file name output: attribute name, '|', attribute value, '|', attribute kind ('S', string '!' as spaces; 'I', integer; 'R', real; 'D', double ) """ if not os.path.isfile(ncfile): print errormsg print ' isgattrs: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'r') ncattrs = ncf.ncattrs() if not searchInlist(ncattrs, values): print ' isgattrs: File "' + ncfile + '" does not have global attribute: "' + values + '" ' ncf.close() quit(-1) attrv=ncf.getncattr(values) if type(attrv) == type(str('1')): attrk = 'S' attrv = attrv.replace(' ','!') elif type(attrv) == type(unicode('1')): attrk = 'S' attrv = attrv.replace(' ','!') elif type(attrv) == type(int(1)): attrk = 'I' elif type(attrv) == type(np.int(1)): attrk = 'I' elif type(attrv) == type(np.int32(1)): attrk = 'I' elif type(attrv) == type(float(1.)): attrk = 'R' elif type(attrv) == type(np.float32(1.)): attrk = 'R' elif type(attrv) == type(np.float64(1.)): attrk = 'D' else: print errormsg print ' isgattr: Reading attribute "', type(attrv), '" not ready! value:', attrv ncf.close() quit(-1) print values, '|', attrv, '|', attrk ncf.close() return [values, attrv, attrk] def ivattrs(ncfile, varn): """Give all the attributes of a variable and its type ncfile = netCDF file name var = variable name output: attribute name, '|', attribute value, '|', attribute kind ('S', string '!' as spaces; 'I', integer; 'R', real; 'D', double ) """ if not os.path.isfile(ncfile): print errormsg print ' ivattrs: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'r') ncvars = ncf.variables if not searchInlist(ncvars, varn): print ' ivattrs: File "' + ncfile + '" does not have variable: "' + varn + '" ' ncf.close() quit(-1) varval = ncf.variables[varn] ncattrs = varval.ncattrs() allattrs='' allattrsdic = {} for attr in ncattrs: attrv=varval.getncattr(attr) if type(attrv) == type(str('1')): attrk = 'S' attrv = attrv.replace(' ','!') elif type(attrv) == type(unicode('1')): attrk = 'S' attrv = attrv.replace(' ','!') elif type(attrv) == type(int(1)): attrk = 'I' elif type(attrv) == type(np.int(1)): attrk = 'I' elif type(attrv) == type(np.int32(1)): attrk = 'I' elif type(attrv) == type(float(1.)): attrk = 'R' elif type(attrv) == type(np.float32(1.)): attrk = 'R' elif type(attrv) == type(np.float64(1.)): attrk = 'D' else: print errormsg print ' ivattrs: Reading attribute "', type(attrv), '" not ready! value:', attrv ncf.close() quit(-1) print attr, '|', attrv, '|', attrk allattrsdic[attr] = [attrv, attrk] allattrs=allattrs + ':' + attr + '|' + str(attrv) + '|' + attrk print '####### ###### ##### #### ### ## #' print '# allvattrs= ' + allattrs ncf.close() return allattrsdic def isvattrs(values, ncfile, varn): """Give a single attribute of a variable values = attribute name ncfile = netCDF file name varn = variable name """ if not os.path.isfile(ncfile): print errormsg print ' isvattrs:File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'r') ncvars = ncf.variables if not searchInlist(ncvars, varn): print errormsg print ' isvattrs:"' + ncfile + '" does not have variable: "' + varn + '" ' ncf.close() quit(-1) varval = ncf.variables[varn] ncattrs = varval.ncattrs() if not searchInlist(ncattrs, values): print errormsg print ' isvattrs:' + ncfile + ' does not have global attribute: "' + values + '" ' ncf.close() quit(-1) attrv=varval.getncattr(values) if type(attrv) == type(str('1')): attrk = 'S' elif type(attrv) == type(unicode('1')): attrk = 'S' elif type(attrv) == type(int(1)): attrk = 'I' elif type(attrv) == type(np.int(1)): attrk = 'I' elif type(attrv) == type(np.int32(1)): attrk = 'I' elif type(attrv) == type(float(1.)): attrk = 'R' elif type(attrv) == type(np.float(1.)): attrk = 'R' elif type(attrv) == type(np.float32(1.)): attrk = 'R' elif type(attrv) == type(np.float64(1.)): attrk = 'D' else: print errormsg print ' isvattr: Reading attribute "', type(attrv), '" not ready! value:', attrv ncf.close() quit(-1) print values, '|', attrv, '|', attrk ncf.close() return [values, attrv, attrk] def grattr(values, ncfile): """ Function to read a global atribute values = attribute name ncfile = netCDF file name """ ncf = NetCDFFile(ncfile,'r') glob_attrs = ncf.ncattrs() attrPos = searchInlist(glob_attrs, values) if not attrPos: print errormsg print ' grattr: File "' + ncfile + '" does not have attribute "' + values + '" !!!!' print errormsg ncf.close() quit(-1) print ncf.getncattr(values) ncf.close() def vrattr(values, ncfile, varn): """ Function to remove an atribute from a variable values = attribute name ncfile = netCDF file name varn = variable name """ fname = 'vrattr' ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ": File '" + ncfile + "' does not have variable '" + \ varn + "' !!!!" ncf.close() quit(-1) var = ncf.variables[varn] var_attrs = var.ncattrs() if not searchInlist(var_attrs, values): print errormsg print ' ' + fname + ": Variable '" + varn + "' does not have attribute '" + \ values + "' !!!!" ncf.close() quit(-1) else: attr = var.delncattr(values) ncf.close() return def datetimeStr_datetime(StringDT): """ Function to transform a string date ([YYYY]-[MM]-[DD]_[HH]:[MI]:[SS] format) to a date object >>> datetimeStr_datetime('1976-02-17_00:00:00') 1976-02-17 00:00:00 """ import datetime as dt fname = 'datetimeStr_datetime' dateD = np.zeros((3), dtype=int) timeT = np.zeros((3), dtype=int) dateD[0] = int(StringDT[0:4]) dateD[1] = int(StringDT[5:7]) dateD[2] = int(StringDT[8:10]) trefT = StringDT.find(':') if not trefT == -1: # print ' ' + fname + ': refdate with time!' timeT[0] = int(StringDT[11:13]) timeT[1] = int(StringDT[14:16]) timeT[2] = int(StringDT[17:19]) if int(dateD[0]) == 0: print warnmsg print ' ' + fname + ': 0 reference year!! changing to 1' dateD[0] = 1 newdatetime = dt.datetime(dateD[0], dateD[1], dateD[2], timeT[0], timeT[1], timeT[2]) return newdatetime def dateStr_date(StringDate): """ Function to transform a string date ([YYYY]-[MM]-[DD] format) to a date object >>> dateStr_date('1976-02-17') 1976-02-17 """ import datetime as dt dateD = StringDate.split('-') if int(dateD[0]) == 0: print warnmsg print ' dateStr_date: 0 reference year!! changing to 1' dateD[0] = 1 newdate = dt.date(int(dateD[0]), int(dateD[1]), int(dateD[2])) return newdate def timeStr_time(StringDate): """ Function to transform a string date ([HH]:[MI]:[SS] format) to a time object >>> datetimeStr_datetime('04:32:54') 04:32:54 """ import datetime as dt timeT = StringDate.split(':') if len(timeT) == 3: newtime = dt.time(int(timeT[0]), int(timeT[1]), int(timeT[2])) else: newtime = dt.time(int(timeT[0]), int(timeT[1]), 0) return newtime def timeref_datetime(refd, timeval, tu): """ Function to transform from a [timeval] in [tu] units from the time referece [tref] to datetime object refd: time of reference (as datetime object) timeval: time value (as [tu] from [tref]) tu: time units >>> timeref = date(1949,12,1,0,0,0) >>> timeref_datetime(timeref, 229784.36, hours) 1976-02-17 08:21:36 """ import datetime as dt import numpy as np ## Not in timedelta # if tu == 'years': # realdate = refdate + dt.timedelta(years=float(timeval)) # elif tu == 'months': # realdate = refdate + dt.timedelta(months=float(timeval)) if tu == 'weeks': realdate = refd + dt.timedelta(weeks=float(timeval)) elif tu == 'days': realdate = refd + dt.timedelta(days=float(timeval)) elif tu == 'hours': realdate = refd + dt.timedelta(hours=float(timeval)) elif tu == 'minutes': realdate = refd + dt.timedelta(minutes=float(timeval)) elif tu == 'seconds': realdate = refd + dt.timedelta(seconds=float(timeval)) elif tu == 'milliseconds': realdate = refd + dt.timedelta(milliseconds=float(timeval)) else: print errormsg print ' timeref_datetime: time units "' + tu + '" not ready!!!!' quit(-1) return realdate def timeref_datetime_mat(refd, timeval, tu): """ Function to transform from a [timeval] in [tu] units from the time referece [tref] to matrix with: year, day, month, hour, minute, second refd: time of reference (as datetime object) timeval: time value (as [tu] from [tref]) tu: time units >>> timeref = date(1949,12,1,0,0,0) >>> timeref_datetime(timeref, 229784.36, hours) [1976 2 17 8 36 21] """ import datetime as dt import numpy as np realdates = np.zeros(6, dtype=int) ## Not in timedelta # if tu == 'years': # realdate = refdate + dt.timedelta(years=float(timeval)) # elif tu == 'months': # realdate = refdate + dt.timedelta(months=float(timeval)) if tu == 'weeks': realdate = refd + dt.timedelta(weeks=float(timeval)) elif tu == 'days': realdate = refd + dt.timedelta(days=float(timeval)) elif tu == 'hours': realdate = refd + dt.timedelta(hours=float(timeval)) elif tu == 'minutes': realdate = refd + dt.timedelta(minutes=float(timeval)) elif tunits == 'seconds': realdate = refd + dt.timedelta(seconds=float(timeval)) elif tunits == 'milliseconds': realdate = refd + dt.timedelta(milliseconds=float(timeval)) else: print errormsg print ' timeref_datetime: time units "' + tu + '" not ready!!!!' quit(-1) realdates[0] = int(realdate.year) realdates[1] = int(realdate.month) realdates[2] = int(realdate.day) realdates[3] = int(realdate.hour) realdates[4] = int(realdate.second) realdates[5] = int(realdate.minute) return realdates def realdatetime_CFcompilant(times, Srefdate, tunits): """ Function to transform a matrix with real time values ([year, month, day, hour, minute, second]) to a netCDF one times= matrix with times Srefdate= reference date ([YYYY][MM][DD][HH][MI][SS] format) tunits= units of time respect to Srefdate >>> realdatetime_CFcompilant(np.array([ [1976, 2, 17, 8, 20, 0], [1976, 2, 18, 8, 20, 0]], dtype=int), '19491201000000', 'hours') [ 229784.33333333 229808.33333333] """ import datetime as dt yrref=int(Srefdate[0:4]) monref=int(Srefdate[4:6]) dayref=int(Srefdate[6:8]) horref=int(Srefdate[8:10]) minref=int(Srefdate[10:12]) secref=int(Srefdate[12:14]) refdate=dt.datetime(yrref, monref, dayref, horref, minref, secref) dimt=times.shape[0] cfdates = np.zeros((dimt), dtype=np.float64) if tunits == 'weeks': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = (cfdate.days + cfdate.seconds/(3600.*24.))/7. elif tunits == 'days': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days + cfdate.seconds/(3600.*24.) elif tunits == 'hours': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*24. + cfdate.seconds/3600. elif tunits == 'minutes': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*24.*60. + cfdate.seconds/60. elif tunits == 'seconds': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*24.*3600. + cfdate.seconds elif tunits == 'milliseconds': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*1000.*24.*3600. + cfdate.seconds*1000. elif tunits == 'microseconds': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*1000000.*24.*3600. + cfdate.seconds*1000000. else: print errormsg print ' ' + fname + ': time units "' + tunits + '" is not ready!!!' quit(-1) return cfdates def netCDFdatetime_realdatetime(units, tcalendar, times): """ Function to transfrom from netCDF CF-compilant times to real time """ import datetime as dt txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Calendar type ## is360 = False if tcalendar is not None: print ' netCDFdatetime_realdatetime: There is a calendar attribute' if tcalendar == '365_day' or tcalendar == 'noleap': print ' netCDFdatetime_realdatetime: No leap years!' isleapcal = False elif tcalendar == 'proleptic_gregorian' or tcalendar == 'standard' or tcalendar == 'gregorian': isleapcal = True elif tcalendar == '360_day': is360 = True isleapcal = False else: print errormsg print ' netCDFdatetime_realdatetime: Calendar "' + tcalendar + '" not prepared!' quit(-1) # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: print ' netCDFdatetime_realdatetime: refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) dimt = len(times) # datetype = type(dt.datetime(1972,02,01)) # realdates = np.array(dimt, datetype) # print realdates ## Not in timedelta # if tunits == 'years': # for it in range(dimt): # realdate = refdate + dt.timedelta(years=float(times[it])) # realdates[it] = int(realdate.year) # elif tunits == 'months': # for it in range(dimt): # realdate = refdate + dt.timedelta(months=float(times[it])) # realdates[it] = int(realdate.year) # realdates = [] realdates = np.zeros((dimt, 6), dtype=int) if tunits == 'weeks': for it in range(dimt): realdate = refdate + dt.timedelta(weeks=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'days': for it in range(dimt): realdate = refdate + dt.timedelta(days=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'hours': for it in range(dimt): realdate = refdate + dt.timedelta(hours=float(times[it])) # if not isleapcal: # Nleapdays = cal.leapdays(int(refdate.year), int(realdate.year)) # realdate = realdate - dt.timedelta(days=Nleapdays) # if is360: # Nyears360 = int(realdate.year) - int(refdate.year) + 1 # realdate = realdate -dt.timedelta(days=Nyears360*5) # realdates[it] = realdate # realdates = refdate + dt.timedelta(hours=float(times)) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'minutes': for it in range(dimt): realdate = refdate + dt.timedelta(minutes=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'seconds': for it in range(dimt): realdate = refdate + dt.timedelta(seconds=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'milliseconds': for it in range(dimt): realdate = refdate + dt.timedelta(milliseconds=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'microseconds': for it in range(dimt): realdate = refdate + dt.timedelta(microseconds=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] else: print errormsg print ' netCDFdatetime_realdatetime: time units "' + tunits + '" is not ready!!!' quit(-1) return realdates class cls_time_information(object): """ Classs to provide information about variable time ncfu = netCDF unit name tname = name of the variable time in [ncfu] self.calendar: calendar of the variable time self.unitsval: units value as it appears in attributes self.units: units of variable time self.Urefdate: reference date as it appears in the original 'units' section self.Srefdate: reference date as string [YYYY][MM][DD][HH][MI][SS] self.refdate: reference date self.attributes: attributes of time variable self.dimt: dimension of the time variable self.dt: distance between first two time-steps in self.tunits self.firstTu: first time value in self.units self.firstTt: first time value as datetime object self.firstTS: first time value as string [YYYY][MM][DD][HH][MI][SS] self.firstTm: first time value as matrix (from datetime; [Y], [M], [D], [H], [M], [S]) self.lastTu: last time value in self.units self.firstTt: last time value as datetime object self.lastTS: last time value as string [YYYY][MM][DD][HH][MI][SS] self.lastTm: last time value as matrix (from datetime; [Y], [M], [D], [H], [M], [S]) """ def __init__(self, ncfu, tname): import datetime as dt fname = 'cls_time_information' if ncfu is None: self.unitsval = None self.units = None self.calendar = None self.Urefdate = None self.Srefdate = None self.refdate = None self.attributes = None self.dimt = None self.dt = None self.firstTu = None self.firstTt = None self.firstTS = None self.firstTm = None self.lastTu = None self.lastTt = None self.lastTS = None self.lastTm = None else: times = ncfu.variables[tname] attvar = times.ncattrs() self.attributes = attvar if not searchInlist(attvar, 'units'): print warnmsg print ' ' + fname + ': time variable "' + tname + \ '" does not have attribute: "units"' units = None self.unitsval = None self.units = None self.calendar = None self.Urefdate = None self.Srefdate = None self.refdate = None if len(times.shape) == 1: self.dimt = len(times[:]) self.dt = times[1] - times[0] self.firstTu = times[0] self.firstTt = None self.firstTS = None self.firstTm = None self.lastTu = times[self.dimt] self.lastTt = None self.lastTS = None self.lastTm = None elif len(times.shape) == 2: if times.dimensions[1] == 'DateStrLen' and times.shape[1] == 19: print warnmsg print ' ' + fname + ": WRF 'Times' variable !!" self.unitsval = 'seconds since 1949-12-01 00:00:00' self.units = 'seconds' self.calendar = 'standard' self.Urefdate = '1949-12-01 00:00:00' self.Srefdate = '19491201000000' self.refdate = datetimeStr_datetime('1949-12-01_00:00:00') self.dimt = times.shape[0] self.firstTS = datetimeStr_conversion(times[0,:], \ 'WRFdatetime', 'YmdHMS') self.firstTm = datetimeStr_conversion(times[0,:], \ 'WRFdatetime', 'YmdHMS') self.lastTS = datetimeStr_conversion(times[self.dimt-1,:], \ 'WRFdatetime', 'YmdHMS') self.lastTm = datetimeStr_conversion(times[self.dimt-1,:], \ 'WRFdatetime', 'matYmdHMS') secondTS = datetimeStr_conversion(times[1,:], 'WRFdatetime', \ 'YmdHMS') secondTt = dt.datetime.strptime(secondTS, '%Y%m%d%H%M%S') self.firstTt = dt.datetime.strptime(self.firstTS, \ '%Y%m%d%H%M%S') self.lastTu = None self.lastTt = dt.datetime.strptime(self.lastTS, \ '%Y%m%d%H%M%S') difft = self.firstTt - secondTt self.dt = difft.days*3600.*24. + difft.seconds + \ difft.microseconds/10.e6 difft = self.firstTt - self.refdate self.firstTu = difft.days*3600.*24. + difft.seconds + \ difft.microseconds/10.e6 difft = self.lastTt - self.refdate self.lastTu = difft.days*3600.*24. + difft.seconds + \ difft.microseconds/10.e6 else: self.dimt = None self.dt = None self.firstTu = None self.firstTt = None self.firstTS = None self.firstTm = None self.lastTu = None self.lastTt = None self.lastTS = None self.lastTm = None else: units = times.getncattr('units') self.unitsval = units if not searchInlist(attvar, 'calendar'): print warnmsg print ' cls_time_information: time variable "', tname, \ '" does not have attribute: "calendar"' self.calendar = '-' else: self.calendar = times.getncattr('calendar') txtunits = units.split(' ') self.units = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: # print ' refdate with time!' self.refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2]+\ '_' + Srefdate) self.Urefdate = txtunits[len(txtunits) - 2] + ' ' + Srefdate else: self.refdate = datetimeStr_datetime(Srefdate + '_00:00:00') self.Urefdate = Srefdate self.Srefdate = self.refdate.strftime("%Y%m%d%H%M%S") timev = times[:] self.dimt = times.shape[0] self.dt = timev[1]-timev[0] self.firstTu = timev[0] self.firstTt = timeref_datetime(self.refdate, self.firstTu,self.units) self.firstTS = self.firstTt.strftime("%Y%m%d%H%M%S") self.firstTm = timeref_datetime_mat(self.refdate, self.firstTu, \ self.units) self.lastTu = timev[self.dimt-1] self.lastTt = timeref_datetime(self.refdate, self.lastTu, self.units) self.lastTS = self.lastTt.strftime("%Y%m%d%H%M%S") self.lastTm = timeref_datetime_mat(self.refdate, self.lastTu, \ self.units) return def time_information(ncfu, tname): """ Function to provide information about variable time ncfu = netCDF unit name tname = name of the variable time in [ncfu] """ times = ncfu.variables[tname] timeinf = [] attvar = times.ncattrs() if not searchInlist(attvar, 'units'): print errormsg print ' "time" does not have attribute: "units"' quit(-1) else: units = times.getncattr('units') txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: # print ' refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) timeinf.append(tunits) timeinf.append(Srefdate) timeinf.append(refdate) return timeinf def date_juliandate(refyr, TOTsecs): """ Function to transform from a total quantity of seconds since the beginning of a year to 360 days / year calendar TOTsecs= total number of seconds """ fname = 'date_juliandate' secsYear = 3600*24*30*12. secsMonth = 3600*24*30. secsDay = 3600*24. secsHour = 3600. secsMinute = 60. rldate = np.zeros((6), dtype=int) rldate[0] = refyr if TOTsecs > secsYear: rldate[0] = refyr + int(TOTsecs/secsYear) + 1 TOTsecs = TOTsecs - int(TOTsecs/secsYear)*secsYear if np.mod(TOTsecs,secsMonth) == 0: rldate[1] = int(TOTsecs/secsMonth) else: rldate[1] = int(TOTsecs/secsMonth) + 1 TOTsecs = TOTsecs - int(TOTsecs/secsMonth)*secsMonth if np.mod(TOTsecs,secsDay) == 0: rldate[2] = int(TOTsecs/secsDay) else: rldate[2] = int(TOTsecs/secsDay) + 1 TOTsecs = TOTsecs - int(TOTsecs/secsDay)*secsDay if np.mod(TOTsecs,secsHour) == 0: rldate[3] = int(TOTsecs/secsHour) else: rldate[3] = int(TOTsecs/secsHour) + 1 TOTsecs = TOTsecs - int(TOTsecs/secsHour)*secsHour if np.mod(TOTsecs,secsMinute) == 0: rldate[4] = int(TOTsecs/secsMinute) else: rldate[4] = int(TOTsecs/secsMinute) + 1 TOTsecs = TOTsecs - int(TOTsecs/secsMinute)*secsMinute rldate[5] = TOTsecs # print refyr,TOTsecs,':',rldate # quit() return rldate def CFtimes_datetime(ncfile, tname): """ Provide date/time array from a file with a series of netCDF CF-compilant time variable ncfile = netCDF file name tname = name of the variable time in [ncfile] output: array(dimt, 0) = year array(dimt, 1) = month array(dimt, 2) = day array(dimt, 3) = hour array(dimt, 4) = minute array(dimt, 5) = second """ import datetime as dt fname = 'CFtimes_datetime' times = ncfile.variables[tname] timevals = times[:] attvar = times.ncattrs() if not searchInlist(attvar, 'units'): print errormsg print ' ' + fname + ": '" + tname + "' does not have attribute: 'units'" quit(-1) else: units = times.getncattr('units') txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: # print ' refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) dimt = len(timevals) realdates = np.zeros((dimt, 6), dtype=int) secsDay=3600*24. # Checking calendar! ## y360 = False daycal360 = ['earth_360d', '360d', '360days', '360_day'] if searchInlist(attvar, 'calendar'): calendar = times.getncattr('calendar') if searchInlist(daycal360,calendar): print warnmsg print ' ' + fname + ': calendar of 12 months of 30 days !!' y360 = True ## Not in timedelta # if tunits == 'years': # for it in range(dimt): # realdate = refdate + dt.timedelta(years=float(times[it])) # realdates[it] = int(realdate.year) # elif tunits == 'months': # for it in range(dimt): # realdate = refdate + dt.timedelta(months=float(times[it])) # realdates[it] = int(realdate.year) if y360: if tunits == 'weeks': for it in range(dimt): deltat = dt.timedelta(weeks=float(times[it])) Tsecs = deltat.days*secsDay + deltat.seconds + deltat.microseconds/1000. realdates[it,:] = date_juliandate(refdate.year,Tsecs) elif tunits == 'days': for it in range(dimt): deltat = dt.timedelta(days=float(times[it])) Tsecs = deltat.days*secsDay + deltat.seconds + deltat.microseconds/1000. realdates[it,:] = date_juliandate(refdate.year,Tsecs) elif tunits == 'hours': for it in range(dimt): realdate = dt.timedelta(hours=float(times[it])) Tsecs = deltat.days*secsDay + deltat.seconds + deltat.microseconds/1000. realdates[it,:] = date_juliandate(refdate.year,Tsecs) elif tunits == 'minutes': for it in range(dimt): realdate = dt.timedelta(minutes=float(times[it])) Tsecs = deltat.days*secsDay + deltat.seconds + deltat.microseconds/1000. realdates[it,:] = date_juliandate(refdate.year,Tsecs) elif tunits == 'seconds': for it in range(dimt): realdate = dt.timedelta(seconds=float(times[it])) Tsecs = deltat.days*secsDay + deltat.seconds + deltat.microseconds/1000. realdates[it,:] = date_juliandate(refdate.year,Tsecs) elif tunits == 'miliseconds': for it in range(dimt): realdate = dt.timedelta(miliseconds=float(times[it])) Tsecs = deltat.days*secsDay + deltat.seconds + deltat.microseconds/1000. realdates[it,:] = date_juliandate(refdate.year,Tsecs) else: print errormsg print ' CFtimes_datetime: time units "' + tunits + '" not ready!!!!' quit(-1) else: if tunits == 'weeks': for it in range(dimt): realdate = refdate + dt.timedelta(weeks=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'days': for it in range(dimt): realdate = refdate + dt.timedelta(days=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'hours': for it in range(dimt): realdate = refdate + dt.timedelta(hours=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'minutes': for it in range(dimt): realdate = refdate + dt.timedelta(minutes=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'seconds': for it in range(dimt): realdate = refdate + dt.timedelta(seconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'milliseconds': for it in range(dimt): realdate = refdate + dt.timedelta(milliseconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) else: print errormsg print ' CFtimes_datetime: time units "' + tunits + '" not ready!!!!' quit(-1) return realdates class variable_inf(object): """ Class to provide the information from a given variable var = object netCDF variable self.name: name of the variable self.dtype: type of the variable self.attributes: list with the name of attributes self.FillValue: value of the missing value self.dimns: name of the dimensions of the variable self.dims: dimensions of the variable self.Ndims: number of dimensions self.dimx: length of dimension along x-axis self.dimy: length of dimension along y-axis self.sname: standard name self.lname: long name self.corr: attribute 'coordinates' self.units: units of the variable """ def __init__(self, var): if var is None: self.name = None self.dimns = None self.dims = None self.Ndims = None self.dimx = None self.dimy = None self.sname = None self.lname = None self.corr = None self.units = None self.FillValue = None self.dtype = None self.attributes = None else: self.name = var._name self.dimns = var.dimensions self.dtype = var.dtype self.attributes = var.ncattrs() self.dims = var.shape if searchInlist(self.attributes, 'standard_name'): self.sname = var.getncattr('standard_name') else: # print ' variable_inf.classpy: variable "' + self.name + '" does not have attribute "standard_name"' self.sname = None if searchInlist(self.attributes, 'long_name'): self.lname = var.getncattr('long_name') else: # print ' variable_inf.classpy: variable "' + self.name + '" does not have attribute "long_name"' self.lname = None if searchInlist(self.attributes, 'coordinates'): self.coor = var.getncattr('coordinates') else: # print ' variable_inf.classpy: variable "' + self.name + '" does not have attribute "coordinates"' self.coor = None if searchInlist(self.attributes, 'units'): self.units = var.getncattr('units') else: # print ' variable_inf.classpy: variable "' + self.name + '" does not have attribute "units"' self.units = None if searchInlist(self.attributes, '_FillValue'): self.FillValue = var.getncattr('_FillValue') else: # print ' variable_inf.classpy: variable "' + self.name + '" does not have attribute "_FillValue"' self.FillValue = None self.Ndims = len(self.dims) if self.Ndims == 1: self.dimx=self.dims[0] if self.Ndims == 2: self.dimy=self.dims[0] self.dimx=self.dims[1] if self.Ndims == 3: self.dimy=self.dims[1] self.dimx=self.dims[2] if self.Ndims == 4: self.dimy=self.dims[2] self.dimx=self.dims[3] def subyrs(values, ncfile, varn): """ Function to retrieve a series of years from a file values = [year1]:[[year2]:...[yearn]] values for years [year1], [year2], ... [yearn] [yearI]-[yearE] values for the period between [yearI] and [yearN] ncfile = netCDF file name varn = variable name """ import datetime as dt import calendar as cal ofile = 'subsetyrs.nc' ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' subyrs: File does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) times = ncf.variables['time'] timevals = times[:] realdates = CFtimes_datetime(ncf, 'time') dimt = len(timevals) # Checking years ## isper = values.find('-') desiredvalues = np.array(dimt, dtype=bool) desiredvaluesi = np.array(dimt, dtype=bool) desiredvaluese = np.array(dimt, dtype=bool) if not isper == -1: # Years list given as a period [YYYYi]-[YYYYf] print ' subyrs: There is a period of years "' + values + '"' iyr = int(values.split('-')[0]) eyr = int(values.split('-')[1]) desiredvaluesi = np.array(realdates[:,0] >= iyr) desiredvaluese = np.array(realdates[:,0] <= eyr) desiredvalues = desiredvaluesi*desiredvaluese else: yrs = values.split(':') nyr = 1 for iyr in range(len(yrs)): desiredvaluesi = np.array(realdates[:,0] == int(yrs[iyr])) if nyr == 1: desiredvalues = desiredvaluesi else: desiredvalues = desiredvaluesi+desiredvalues nyr = nyr + 1 Ndesiredvalues = len(realdates[desiredvalues]) print ' subyrs: N values: ', Ndesiredvalues if Ndesiredvalues == 0: print errormsg print ' subyrs: No values found for "' + values + '"!!' ncf.close() quit(-1) # Variable values (assuming time as first dimension) ## var = ncf.variables[varn] # varvals = var[:] vardims = var.shape varshape = len(vardims) print ' subyrs: Shape of data: ',varshape, ':' , vardims if varshape == 1: vardesiredvalues = np.arange(Ndesiredvalues) vardesiredvalues = var[desiredvalues] elif varshape == 2: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]).reshape(Ndesiredvalues,vardims[1]) vardesiredvalues = var[desiredvalues, :] elif varshape == 3: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]).reshape(Ndesiredvalues, \ vardims[1],vardims[2]) vardesiredvalues = var[desiredvalues, :, :] elif varshape == 4: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]).reshape(Ndesiredvalues, \ vardims[1],vardims[2],vardims[3]) vardesiredvalues = var[desiredvalues, :, :, :] elif varshape == 5: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]*vardims[4]).reshape(Ndesiredvalues, \ vardims[1],vardims[2],vardims[3],vardims[4]) vardesiredvalues = var[desiredvalues, :, :, :, :] elif varshape == 6: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]*vardims[4]* \ vardims[5]).reshape(Ndesiredvalues,vardims[1],vardims[2],vardims[3],vardims[4],vardims[5]) vardesiredvalues = var[desiredvalues, :, :, :, :, :] else: print errormsg print ' subyrs: ', varshape, ' shape of matrix not prepared !' ncf.close() quit(-1) # print ' shape of desired values: ', vardesiredvalues.shape # Creation of file ## ncfo = NetCDFFile( ofile, 'w') vardims = var.dimensions vartype = var.dtype varattr = var.ncattrs() # Checking dimensions ## newdims = ncfo.dimensions for rdim in vardims: if not searchInlist(newdims, rdim): if not rdim == 'time': print ' subyrs: Adding dimension ' + rdim ncfo.sync() ncf.close() ncfo.close() fdimadd(ncfile + ',' + rdim, ofile) ncf = NetCDFFile(ncfile,'r') ncfo = NetCDFFile(ofile,'a') else: ncfo.createDimension('time', None) # Checking fill value ## if searchInlist(varattr, '_FillValue'): varfil = var._FillValue else: varfil = False newvar = ncfo.createVariable(varn, vartype, vardims, fill_value=varfil) if not varshape == 0: newvar[:] = vardesiredvalues newvar = ncfo.variables[varn] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = var.getncattr(attr) if not searchInlist(newvarattrs, attr): newvar.setncattr(attr, attrv) vardims = times.dimensions vartype = times.dtype varattr = times.ncattrs() newvar = ncfo.createVariable('time', vartype, vardims, fill_value=varfil) newvar = ncfo.variables['time'] newvar[:] = timevals[desiredvalues] ncf.close() ncfo.sync() ncfo.close() fattradd('time', ncfile + ':time', ofile) fvaradd(ncfile + ',lon', ofile) fvaradd(ncfile + ',lat', ofile) ncfo = NetCDFFile(ofile,'a') newvar = ncfo.variables['time'] newvarattrs = newvar.ncattrs() if searchInlist(newvarattrs, 'bounds'): if newvar.getncattr('bounds') == 'time_bnds': ncf = NetCDFFile(ncfile,'r') tbnds = ncf.variables['time_bnds'] vardims = tbnds.dimensions vartype = tbnds.dtype varattr = tbnds.ncattrs() ncfo.createDimension('bnds', 2) newvar = ncfo.createVariable('time_bnds', vartype, vardims, fill_value=varfil) newvar[:] = tbnds[desiredvalues,:] ncf.close() ncfo.sync() ncfo.close() fattradd('time_bnds', ncfile + ':time_bnds', ofile) else: ncfo.close() else: ncfo.close() fgaddattr(ncfile, ofile) print ' subyrs: File "' + ofile + '" with a subset of ' + values + ' has been created' def submns(values, ncfile, varn): """ Function to retrieve a series of months from a file values = [mon1]:[[mion2]:...[monn]] values for months [mon1], [mon2], ... [monn] [monI]-[monE] values for the period between [monI] and [monN] ncfile = netCDF file name varn = variable name """ import datetime as dt import calendar as cal ofile = 'subsetmns.nc' ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' submns: File "' + ncfile + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) times = ncf.variables['time'] timevals = times[:] realdates = CFtimes_datetime(ncf, 'time') dimt = len(timevals) # Checking months ## isper =values.find('-') desiredvalues = np.array(dimt, dtype=bool) desiredvaluesi = np.array(dimt, dtype=bool) desiredvaluese = np.array(dimt, dtype=bool) if not isper == -1: # Months list given as a period [MMi]-[MMf] print ' submns: There is a period of months "' + values + '"' imn = int(values.split('-')[0]) emn = int(values.split('-')[1]) desiredvaluesi = np.array(realdates[:,1] >= imn) desiredvaluese = np.array(realdates[:,1] <= emn) desiredvalues = desiredvaluesi*desiredvaluese else: mns = values.split(':') nmn = 1 for imn in range(len(mns)): desiredvaluesi = np.array(realdates[:,1] == int(mns[imn])) if nmn == 1: desiredvalues = desiredvaluesi else: desiredvalues = desiredvaluesi+desiredvalues nmn = nmn + 1 Ndesiredvalues = len(realdates[desiredvalues]) print ' submns: N values: ', Ndesiredvalues if Ndesiredvalues == 0: print errormsg print ' submns: No values found for "' + values + '"!!' ncf.close() quit(-1) # Variable values (assuming time as first dimension) ## var = ncf.variables[varn] # varvals = var[:] vardims = var.shape varshape = len(vardims) # print ' submns: Shape of data: ',varshape, ':' , vardims if varshape == 1: vardesiredvalues = np.arange(Ndesiredvalues) vardesiredvalues = var[desiredvalues] elif varshape == 2: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]).reshape(Ndesiredvalues,vardims[1]) vardesiredvalues = var[desiredvalues, :] elif varshape == 3: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]).reshape(Ndesiredvalues, \ vardims[1],vardims[2]) vardesiredvalues = var[desiredvalues, :, :] elif varshape == 4: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]).reshape(Ndesiredvalues, \ vardims[1],vardims[2],vardims[3]) vardesiredvalues = var[desiredvalues, :, :, :] elif varshape == 5: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]*vardims[4]).reshape(Ndesiredvalues, \ vardims[1],vardims[2],vardims[3],vardims[4]) vardesiredvalues = var[desiredvalues, :, :, :, :] elif varshape == 6: vardesiredvalues = np.arange(Ndesiredvalues*vardims[1]*vardims[2]*vardims[3]*vardims[4]* \ vardims[5]).reshape(Ndesiredvalues,vardims[1],vardims[2],vardims[3],vardims[4],vardims[5]) vardesiredvalues = var[desiredvalues, :, :, :, :, :] else: print errormsg print ' submns: ', varshape, ' shape of matrix not prepared !' ncf.close() quit(-1) # print ' shape of desired values: ', vardesiredvalues.shape # Creation of file ## ncfo = NetCDFFile( ofile, 'w') vardims = var.dimensions vartype = var.dtype varattr = var.ncattrs() # Checking dimensions ## newdims = ncfo.dimensions for rdim in vardims: if not searchInlist(newdims, rdim): if not rdim == 'time': print ' submns: Adding dimension ' + rdim ncfo.sync() ncf.close() ncfo.close() fdimadd(ncfile + ',' + rdim, ofile) ncf = NetCDFFile(ncfile,'r') ncfo = NetCDFFile(ofile,'a') else: ncfo.createDimension('time', None) # Checking fill value ## if searchInlist(varattr, '_FillValue'): varfil = var._FillValue else: varfil = False newvar = ncfo.createVariable(varn, vartype, vardims, fill_value=varfil) if not varshape == 0: newvar[:] = vardesiredvalues newvar = ncfo.variables[varn] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = var.getncattr(attr) if not searchInlist(newvarattrs, attr): newvar.setncattr(attr, attrv) vardims = times.dimensions vartype = times.dtype varattr = times.ncattrs() newvar = ncfo.createVariable('time', vartype, vardims, fill_value=varfil) newvar = ncfo.variables['time'] newvar[:] = timevals[desiredvalues] ncf.close() ncfo.sync() ncfo.close() fattradd('time', ncfile + ':time', ofile) fvaradd(ncfile + ',lon', ofile) fvaradd(ncfile + ',lat', ofile) ncfo = NetCDFFile(ofile,'a') newvar = ncfo.variables['time'] newvarattrs = newvar.ncattrs() if searchInlist(newvarattrs, 'bounds'): if newvar.getncattr('bounds') == 'time_bnds': ncf = NetCDFFile(ncfile,'r') tbnds = ncf.variables['time_bnds'] vardims = tbnds.dimensions vartype = tbnds.dtype varattr = tbnds.ncattrs() ncfo.createDimension('bnds', 2) newvar = ncfo.createVariable('time_bnds', vartype, vardims, fill_value=varfil) newvar[:] = tbnds[desiredvalues,:] ncf.close() ncfo.sync() ncfo.close() fattradd('time_bnds', ncfile + ':time_bnds', ofile) else: ncfo.close() else: ncfo.close() fgaddattr(ncfile, ofile) print ' submns: File "' + ofile + '" with a subset of ' + values + ' has been created' class statsValWeigthed(object): """Weigthed Statistics class providing: vals = values (can be a matrix) wgs = weights (can be a matrix) self.meanv: mean weigthed value self.mean2v: mean quadratic weigthed value self.stdv: weigthed standard deviation self.Nokvalue non None values of a list of values self.meanwgt: mean of the weigths self.mean2wgt: cuadratic mean of the weigths self.stdwgt: standard deviation of the weigths """ def __init__(self, vals, wgs): if vals is None: self.Nv = None self.meanv = None self.mean2v = None self.stdv = None self.Nokvalues = None self.meanwgt = None self.mean2wgt = None self.stdwgt = None else: values = vals.flatten() weights = wgs.flatten() self.Nv=len(values) self.meanv=0. self.mean2v=0. self.stdv=0. self.meanwgt = 0. self.mean2wgt = 0. self.stdwgt = 0. self.Nokvalues = 0 for inum in range(self.Nv): if not values[inum] is None: self.Nokvalues = self.Nokvalues + 1 self.meanv = self.meanv+values[inum]*weights[inum] self.mean2v = self.mean2v+values[inum]*weights[inum]*values[inum] self.meanwgt = self.meanwgt+weights[inum] self.mean2wgt = self.mean2wgt+weights[inum]*weights[inum] self.meanv = self.meanv/float(self.meanwgt) self.mean2v = self.mean2v/float(self.meanwgt) self.stdv = np.sqrt(self.mean2v-self.meanv*self.meanv) self.meanwgt = self.meanwgt/float(self.Nokvalues) self.mean2wgt = self.mean2wgt/float(self.Nokvalues) self.stdwgt = np.sqrt(self.mean2wgt-self.meanwgt*self.meanwgt) class statsValWeighted_missVal(object): """Weighted Statistics taking into account a missing value class providing: vals = values (can be a matrix) wgs = weights (can be a matrix) missVal= missing value self.meanv: mean weigthed value self.mean2v: mean quadratic weigthed value self.stdv: weigthed standard deviation self.Nokvalue non None values of a list of values self.meanwgt: mean of the weigths self.mean2wgt: cuadratic mean of the weigths self.stdwgt: standard deviation of the weigths self.quantilesv: quantiles of the weighted values """ def __init__(self, vals, wgs, missVal): fname='statsValWeigthed_missVal' if vals is None: self.Nv = None self.meanv = None self.mean2v = None self.stdv = None self.Nokvalues = None self.meanwgt = None self.mean2wgt = None self.stdwgt = None self.quantilesv = None else: Npt = 1 for idim in range(len(vals.shape)): Npt = Npt * vals.shape[idim] if np.sum(vals >= missVal) == Npt: print errormsg print ' ' + fname + ' all values get missing!!' print errormsg quit(-1) vals0 = np.where(vals >= missVal, None, vals) values = vals0.flatten() weights = wgs.flatten() self.Nv=Npt self.meanv=0. self.mean2v=0. self.stdv=0. self.meanwgt = 0. self.mean2wgt = 0. self.stdwgt = 0. self.Nokvalues = 0 valswgt = values for inum in range(self.Nv): if not values[inum] is None: self.Nokvalues = self.Nokvalues + 1 valswgt[inum] = valswgt[inum]*weights[inum] self.meanv = self.meanv+values[inum]*weights[inum] self.mean2v = self.mean2v+values[inum]*weights[inum]*values[inum] self.meanwgt = self.meanwgt+weights[inum] self.mean2wgt = self.mean2wgt+weights[inum]*weights[inum] self.meanv = self.meanv/float(self.meanwgt) self.mean2v = self.mean2v/float(self.meanwgt) self.stdv = np.sqrt(self.mean2v-self.meanv*self.meanv) valswgt = valswgt/np.float(self.meanwgt) self.meanwgt = self.meanwgt/float(self.Nokvalues) self.mean2wgt = self.mean2wgt/float(self.Nokvalues) self.stdwgt = np.sqrt(self.mean2wgt-self.meanwgt*self.meanwgt) valsq=Quantiles(valswgt, 20) self.quantilesv=valsq.quantilesv class stats2Val(object): """two variables Statistics class providing: vals1 = variable 1 vals2 = variable 2 power = power of the polynomial fitting to apply between both variables self.min[var], self.max[var], self.mean[var], self.mean2[var], self.std[var] of [var] = var1+var2[v1Av2], var1-var2[v1Sv2], var1/var2[v1Dv2], var1*var2[v1Pv2] self.Nokvalues1: number of correct values of variable 1 self.Nokvalues2: number of correct values of variable 2 self.Nokvalues12: number of correct coincident values of variable 1 and variable 2 self.mae=mean(abs(var1-var2)) self.rmse=sqrt((var1-var2)**2) self.correlation (and p-value) self.linRegress: linear regression [trend, intercept, regression coefficient, p_value, standard error] self.polRegress: polinomial Regresion of degree [power] [coef**[power], coef**[power-1], ...., coef**0] """ def __init__(self, vals1, vals2, power): import numpy as np from scipy import stats as sts if vals1 is None: self.Nv = None self.Nokvalues1 = None self.Nokvalues2 = None self.Nokvalues12 = None self.NDvalNone = None self.minv1Av2 = None self.maxv1Av2 = None self.meanv1Av2 = None self.mean2v1Av2 = None self.stdv1Av2 = None self.minv1Sv2 = None self.maxv1Sv2 = None self.meanv1Sv2 = None self.mean2v1Sv2 = None self.stdv1Sv2 = None self.minv1Dv2 = None self.maxv1Dv2 = None self.meanv1Dv2 = None self.mean2v1Dv2 = None self.stdv1Dv2 = None self.minv1Pv2 = None self.maxv1Pv2 = None self.meanv1Pv2 = None self.mean2v1Pv2 = None self.stdv1Pv2 = None self.mae = None self.rmse = None self.corr = None self.linRegress = None self.polRegress = None self.polRegressResidual = None self.polRegressRes = None self.polRegressSingVal = None else: values1 = vals1.flatten() values2 = vals2.flatten() if not len(values1) == len(values2): print errormsg print ' stats2Val: lengths of variables differ!! Lvar1: ', len(values1), ' Lvar2: ',len(values2),' statistics between them can not be computed!' quit(-1) self.Nv=len(values1) self.minv1Av2=10000000000. self.maxv1Av2=-self.minv1Av2 self.meanv1Av2=0. self.mean2v1Av2=0. self.stdv1Av2=0. self.minv1Sv2=self.minv1Av2 self.maxv1Sv2=-self.minv1Av2 self.meanv1Sv2=0. self.mean2v1Sv2=0. self.stdv1Sv2=0. self.minv1Dv2=self.minv1Av2 self.maxv1Dv2=-self.minv1Av2 self.meanv1Dv2=0. self.mean2v1Dv2=0. self.stdv1Dv2=0. self.minv1Pv2=self.minv1Av2 self.maxv1Pv2=-self.minv1Av2 self.meanv1Pv2=0. self.mean2v1Pv2=0. self.stdv1Pv2=0. self.mae = 0. self.rmse = 0. self.corr = np.array([0., 0.]) self.linRegress = np.zeros(5, float) self.polRegress = np.zeros(power+1, float) self.polRegressResidual = 0. self.polRegressSingVal = np.zeros(power+1, float) # v1 [+ / - / / / *] v2 ## self.Nokvalues1 = 0 self.Nokvalues2 = 0 self.Nokvalues12 = 0 self.NDvalNone = 0 for inum in range(self.Nv): if not values1[inum] is None: self.Nokvalues1 = self.Nokvalues1 + 1 if not values2[inum] is None: self.Nokvalues2 = self.Nokvalues2 + 1 if not values1[inum] is None and not values2[inum] is None: self.Nokvalues12 = self.Nokvalues12 + 1 Aval = values1[inum] + values2[inum] Sval = values1[inum] - values2[inum] Pval = values1[inum] * values2[inum] if np.isinf(values1[inum] / values2[inum]) or np.isnan(values1[inum] / values2[inum]): if self.NDvalNone < 1: print warnmsg print ' stats2Val: val1/val2 inf or Nan!!!!' Dval = None self.NDvalNone = self.NDvalNone + 1 else: Dval = values1[inum] / values2[inum] self.mae = self.mae + abs(Sval) self.rmse = self.rmse + Sval**2 if Aval < self.minv1Av2: self.minv1Av2 = Aval if Aval > self.maxv1Av2: self.maxv1Av2 = Aval if Sval < self.minv1Sv2: self.minv1Sv2 = Sval if Sval > self.maxv1Sv2: self.maxv1Sv2 = Sval if not Dval is None and Dval < self.minv1Dv2: self.minv1Dv2 = Dval if not Dval is None and Dval > self.maxv1Dv2: self.maxv1Dv2 = Dval if Pval < self.minv1Pv2: self.minv1Pv2 = Pval if Pval > self.maxv1Pv2: self.maxv1Pv2 = Pval self.meanv1Av2 = self.meanv1Av2+Aval self.mean2v1Av2 = self.mean2v1Av2+Aval*Aval self.meanv1Sv2 = self.meanv1Sv2+Sval self.mean2v1Sv2 = self.mean2v1Sv2+Sval*Sval if not Dval is None: self.meanv1Dv2 = self.meanv1Dv2+Dval self.mean2v1Dv2 = self.mean2v1Dv2+Dval*Dval self.meanv1Pv2 = self.meanv1Pv2+Pval self.mean2v1Pv2 = self.mean2v1Pv2+Pval*Pval ## print 'Nokvalues1: ', self.Nokvalues1, 'Nokvalues2: ', self.Nokvalues2, 'Nokvalues12: ', float(self.Nokvalues12), 'NDvalNone: ',self.NDvalNone self.meanv1Av2 = self.meanv1Av2/float(self.Nokvalues12) self.mean2v1Av2 = self.mean2v1Av2/float(self.Nokvalues12) self.stdv1Av2 = np.sqrt(self.mean2v1Av2-self.meanv1Av2*self.meanv1Av2) self.meanv1Sv2 = self.meanv1Sv2/float(self.Nokvalues12) self.mean2v1Sv2 = self.mean2v1Sv2/float(self.Nokvalues12) self.stdv1Sv2 = np.sqrt(self.mean2v1Sv2-self.meanv1Sv2*self.meanv1Sv2) if self.Nokvalues12 - self.NDvalNone == 0: self.meanv1Dv2 = None self.mean2v1Dv2 = None self.stdv1Dv2 = None print warnmsg print ' stats2Val: all values of val1/val2 are None!' else: self.meanv1Dv2 = self.meanv1Dv2/(float(self.Nokvalues12 - self.NDvalNone)) self.mean2v1Dv2 = self.mean2v1Dv2/(float(self.Nokvalues12 - self.NDvalNone)) self.stdv1Dv2 = np.sqrt(self.mean2v1Dv2-self.meanv1Dv2*self.meanv1Dv2) self.meanv1Pv2 = self.meanv1Pv2/float(self.Nokvalues12) self.mean2v1Pv2 = self.mean2v1Pv2/float(self.Nokvalues12) self.stdv1Pv2 = np.sqrt(self.mean2v1Pv2-self.meanv1Pv2*self.meanv1Pv2) self.mae = self.mae/self.Nokvalues12 self.rmse = np.sqrt(self.rmse/self.Nokvalues12) self.corr = sts.pearsonr(values1, values2) self.linRegress[0], self.linRegress[1], self.linRegress[2], self.linRegress[3], self.linRegress[4] = sts.linregress(values1, values2) polyfitvals=np.polyfit(values1, values2, power, full = True) self.polRegress = polyfitvals[0] self.polRegressRes = polyfitvals[1] self.polRegressSingVal = polyfitvals[3] class stats2Val_missVal(object): """two variables Statistics taking into account a missing value class providing: vals1 = variable 1 vals2 = variable 2 missVal = missing value power = power of the polynomial fitting to apply between both variables self.min[var], self.max[var], self.mean[var], self.mean2[var], self.std[var] of [var] = var1+var2[v1Av2], var1-var2[v1Sv2], var1/var2[v1Dv2], var1*var2[v1Pv2] self.Nokvalues1: number of correct values of variable 1 self.Nokvalues2: number of correct values of variable 2 self.Nokvalues12: number of correct coincident values of variable 1 and variable 2 self.mae=mean(abs(var1-var2)) self.rmse=sqrt((var1-var2)**2) self.correlation (and p-value) self.linRegress: linear regression [trend, intercept, regression coefficient, p_value, standard error] self.polRegress: polinomial Regresion of degree [power] [coef**[power], coef**[power-1], ...., coef**0] """ def __init__(self, vals1, vals2, power, missVal): import numpy as np from scipy import stats as sts fname='stats2Val_missVal' if vals1 is None: self.Nv = None self.Nokvalues1 = None self.Nokvalues2 = None self.Nokvalues12 = None self.NDvalNone = None self.minv1Av2 = None self.maxv1Av2 = None self.meanv1Av2 = None self.mean2v1Av2 = None self.stdv1Av2 = None self.minv1Sv2 = None self.maxv1Sv2 = None self.meanv1Sv2 = None self.mean2v1Sv2 = None self.stdv1Sv2 = None self.minv1Dv2 = None self.maxv1Dv2 = None self.meanv1Dv2 = None self.mean2v1Dv2 = None self.stdv1Dv2 = None self.minv1Pv2 = None self.maxv1Pv2 = None self.meanv1Pv2 = None self.mean2v1Pv2 = None self.stdv1Pv2 = None self.mae = None self.rmse = None self.corr = None self.linRegress = None self.polRegress = None self.polRegressResidual = None self.polRegressRes = None self.polRegressSingVal = None else: Npt1 = 1 for idim in range(len(vals1.shape)): Npt1 = Npt1 * vals1.shape[idim] if np.sum(vals1 >= missVal) == Npt1: print errormsg print ' ' + fname + ' all values 1 get missing!!' print errormsg quit(-1) Npt2 = 1 for idim in range(len(vals2.shape)): Npt2 = Npt2 * vals2.shape[idim] if np.sum(vals2 >= missVal) == Npt2: print errormsg print ' ' + fname + ' all values 2 get missing!!' print errormsg quit(-1) vals10 = np.where(abs(vals1) >= missVal, None, vals1) vals20 = np.where(abs(vals2) >= missVal, None, vals2) values1 = vals10.flatten() values2 = vals20.flatten() if not len(values1) == len(values2): print errormsg print ' stats2Val: lengths of variables differ!! Lvar1: ', len(values1), ' Lvar2: ',len(values2),' statistics between them can not be computed!' quit(-1) self.Nv=Npt1 self.minv1Av2=10000000000. self.maxv1Av2=-self.minv1Av2 self.meanv1Av2=0. self.mean2v1Av2=0. self.stdv1Av2=0. self.minv1Sv2=self.minv1Av2 self.maxv1Sv2=-self.minv1Av2 self.meanv1Sv2=0. self.mean2v1Sv2=0. self.stdv1Sv2=0. self.minv1Dv2=self.minv1Av2 self.maxv1Dv2=-self.minv1Av2 self.meanv1Dv2=0. self.mean2v1Dv2=0. self.stdv1Dv2=0. self.minv1Pv2=self.minv1Av2 self.maxv1Pv2=-self.minv1Av2 self.meanv1Pv2=0. self.mean2v1Pv2=0. self.stdv1Pv2=0. self.mae = 0. self.rmse = 0. self.corr = np.array([0., 0.]) self.linRegress = np.zeros(5, float) self.polRegress = np.zeros(power+1, float) self.polRegressResidual = 0. self.polRegressSingVal = np.zeros(power+1, float) # v1 [+ / - / / / *] v2 ## self.Nokvalues1 = 0 self.Nokvalues2 = 0 self.Nokvalues12 = 0 self.NDvalNone = 0 for inum in range(self.Nv): if not values1[inum] is None: self.Nokvalues1 = self.Nokvalues1 + 1 if not values2[inum] is None: self.Nokvalues2 = self.Nokvalues2 + 1 if not values1[inum] is None and not values2[inum] is None: self.Nokvalues12 = self.Nokvalues12 + 1 Aval = values1[inum] + values2[inum] Sval = values1[inum] - values2[inum] Pval = values1[inum] * values2[inum] if np.isinf(values1[inum] / values2[inum]) or np.isnan(values1[inum] / values2[inum]): if self.NDvalNone < 1: print warnmsg print ' stats2Val: val1/val2 inf or Nan!!!!' Dval = None self.NDvalNone = self.NDvalNone + 1 else: Dval = values1[inum] / values2[inum] self.mae = self.mae + abs(Sval) self.rmse = self.rmse + Sval**2 if Aval < self.minv1Av2: self.minv1Av2 = Aval if Aval > self.maxv1Av2: self.maxv1Av2 = Aval if Sval < self.minv1Sv2: self.minv1Sv2 = Sval if Sval > self.maxv1Sv2: self.maxv1Sv2 = Sval if not Dval is None and Dval < self.minv1Dv2: self.minv1Dv2 = Dval if not Dval is None and Dval > self.maxv1Dv2: self.maxv1Dv2 = Dval if Pval < self.minv1Pv2: self.minv1Pv2 = Pval if Pval > self.maxv1Pv2: self.maxv1Pv2 = Pval self.meanv1Av2 = self.meanv1Av2+Aval self.mean2v1Av2 = self.mean2v1Av2+Aval*Aval self.meanv1Sv2 = self.meanv1Sv2+Sval self.mean2v1Sv2 = self.mean2v1Sv2+Sval*Sval if not Dval is None: self.meanv1Dv2 = self.meanv1Dv2+Dval self.mean2v1Dv2 = self.mean2v1Dv2+Dval*Dval self.meanv1Pv2 = self.meanv1Pv2+Pval self.mean2v1Pv2 = self.mean2v1Pv2+Pval*Pval ## print 'Nokvalues1: ', self.Nokvalues1, 'Nokvalues2: ', self.Nokvalues2, 'Nokvalues12: ', float(self.Nokvalues12), 'NDvalNone: ',self.NDvalNone self.meanv1Av2 = self.meanv1Av2/float(self.Nokvalues12) self.mean2v1Av2 = self.mean2v1Av2/float(self.Nokvalues12) self.stdv1Av2 = np.sqrt(self.mean2v1Av2-self.meanv1Av2*self.meanv1Av2) self.meanv1Sv2 = self.meanv1Sv2/float(self.Nokvalues12) self.mean2v1Sv2 = self.mean2v1Sv2/float(self.Nokvalues12) self.stdv1Sv2 = np.sqrt(self.mean2v1Sv2-self.meanv1Sv2*self.meanv1Sv2) if self.Nokvalues12 - self.NDvalNone == 0: self.meanv1Dv2 = None self.mean2v1Dv2 = None self.stdv1Dv2 = None print warnmsg print ' stats2Val: all values of val1/val2 are None!' else: self.meanv1Dv2 = self.meanv1Dv2/(float(self.Nokvalues12 - self.NDvalNone)) self.mean2v1Dv2 = self.mean2v1Dv2/(float(self.Nokvalues12 - self.NDvalNone)) self.stdv1Dv2 = np.sqrt(self.mean2v1Dv2-self.meanv1Dv2*self.meanv1Dv2) self.meanv1Pv2 = self.meanv1Pv2/float(self.Nokvalues12) self.mean2v1Pv2 = self.mean2v1Pv2/float(self.Nokvalues12) self.stdv1Pv2 = np.sqrt(self.mean2v1Pv2-self.meanv1Pv2*self.meanv1Pv2) self.mae = self.mae/self.Nokvalues12 self.rmse = np.sqrt(self.rmse/self.Nokvalues12) vals1Nomiss = np.ones(len(values1), dtype=bool) vals2Nomiss = np.ones(len(values1), dtype=bool) for i in range(len(values1)): if values1[i] is None: vals1Nomiss[i] = False for i in range(len(values2)): if values2[i] is None: vals2Nomiss[i] = False v1 = np.array(values1[vals1Nomiss], dtype=float) v2 = np.array(values2[vals2Nomiss], dtype=float) if not v1.shape == v2.shape: print errormsg print ' ' + fname + ': variables without missing values v1: ',v1.shape , ' and v2: ',v2.shape ,' do not have the same shape! ' print errormsg quit(-1) self.corr = sts.pearsonr(v1, v2) self.linRegress[0], self.linRegress[1], self.linRegress[2], self.linRegress[3], self.linRegress[4] = sts.linregress(v1, v2) polyfitvals=np.polyfit(v1, v2, power, full = True) self.polRegress = polyfitvals[0] self.polRegressRes = polyfitvals[1] self.polRegressSingVal = polyfitvals[3] def mask_2masked(vals1, vals2): """ Function to provide the boolean matrix (in opposite way as it is in the mask) as combination of mask from to masked matrices """ import numpy.ma as ma fname = 'mask_2masked' # if len(vals1.shape) != len(vals2.shape): # print errormsg # print ' ' + fname + ' matrix 1 :', len(vals1.shape), ' and matrix 2 ', len(vals2.shape), ' have different size!' # print errormsg # quit(-1) # for idim in range(len(vals1.shape)): # if vals1.shape[idim] != vals2.shape[idim]: # print errormsg # print ' ' + fname + ' dimension ', idim,' from matrix 1 :', vals1.shape[idim], ' and matrix 2 ', \ # vals2.shape[idim], ' have different size!' # print errormsg # quit(-1) if type(vals1) == type(ma.array(1)): mask1array=np.where(ma.getmaskarray(vals1) == False, True, False) else: mask1array=np.ones(vals1.shape, dtype=bool) if type(vals2) == type(ma.array(1)): mask2array=np.where(ma.getmaskarray(vals2) == False, True, False) else: mask2array=np.ones(vals2.shape, dtype=bool) mask12 = mask1array*mask2array return mask12 def mask_pearsonr(xvals, yvals): """ Function to compute a pearson correlation from mask matrices """ from scipy import stats as sts fillVal = 1.e20 maskxy = mask_2masked(xvals, yvals) if np.sum(maskxy) > 1: pearsonr = sts.pearsonr(xvals[maskxy], yvals[maskxy]) if np.isnan(pearsonr[0]) or np.isnan(pearsonr[1]): pearsonr = ( fillVal, fillVal) else: pearsonr = (fillVal, fillVal) return pearsonr def mask_quantiles(maskmat, Nquants): """ Function to provide the quantiles of a masked array 20 for %5 bins (21 in total) """ import numpy.ma as ma fillValue = 1.e20 sortmat = maskmat.flatten().copy() sortmat.sort() quants = np.zeros(Nquants+1, dtype=type(maskmat[0])) Nv = ma.size(maskmat) NoMask=maskmat.count() if NoMask < Nquants: quants[:] = fillValue else: for iq in range(Nquants): quants[iq] = sortmat[int((NoMask-1)*iq/(Nquants))] quants[Nquants] = sortmat[NoMask-1] return quants def percendone(nvals,tot,percen,msg): """ Function to provide the percentage of an action across the matrix nvals=number of values tot=total number of values percen=percentage frequency for which the message is wanted msg= message """ from sys import stdout num = int(tot * percen/100) if (nvals%num == 0): print '\r ' + msg + '{0:8.3g}'.format(nvals*100./tot) + ' %', stdout.flush() return '' def mask_linregres(vals1, vals2): """ Function to compute a linear regression from masked data vals1: x-values for the regresion vals2: y-values for the regresion """ import numpy.ma as ma fname = 'mask_linregres' missval1 = vals1.get_fill_value() missval2 = vals2.get_fill_value() vals10 = np.where(abs(vals1) >= abs(missval1*0.9), None, vals1) vals20 = np.where(abs(vals2) >= abs(missval2*0.9), None, vals2) values1 = vals10.flatten() values2 = vals20.flatten() vals1Nomiss = np.ones(len(values1), dtype=bool) vals2Nomiss = np.ones(len(values2), dtype=bool) for i in range(len(values1)): if values1[i] is None: vals1Nomiss[i] = False for i in range(len(values2)): if values2[i] is None: vals2Nomiss[i] = False v1 = np.array(values1[vals1Nomiss], dtype=float) v2 = np.array(values2[vals2Nomiss], dtype=float) if len(v1) != len(v2): print errormsg print fname + ': length of masked matrices mat1:',len(v1),'and mat2:',len(v2),'does not match!' print errormsg quit(-1) linregres = np.array(sts.linregress(v1, v2), dtype= np.float64) return linregres def mask_space_stats(maskmat,statsvals,dim): """ Function to give back the multi-dimensional statisitcs of a given masked array maskmat=multidimensional masked array statsvals=[statn]:[values] [statn]: statistics to do: 'quant', quantiles [values]: value for the statistics: Nquantiles dim= dimension to run the statistics """ from sys import stdout fname = 'mask_space_stats' statn=statsvals.split(':')[0] if len(statsvals.split(':')) > 1: values=statsvals.split(':')[1] maskshape = maskmat.shape Ndims = len(maskshape) if statn == 'quant': if len(statsvals.split(':')) == 1: print errormsg print fname + ': statistics "' + statn + '" requires a value!!!' print errormsg quit(-1) Nquants=int(values) if Ndims == 2: if dim == 0: statval = np.ones((21, maskshape[1]), dtype=np.float64)*fillValue for i in range(maskshape[1]): percendone(i, maskshape[1], 5, 'quantiles') statval[:,i] = mask_quantiles(maskmat[:,i],Nquants) if dim == 1: statval = np.ones((21, maskshape[0]), dtype=np.float64)*fillValue for i in range(maskshape[0]): percendone(i, maskshape[0], 5, 'quantiles') statval[:,i] = mask_quantiles(statval[i,:],Nquants) elif Ndims == 3: if dim == 0: statval = np.ones((21, maskshape[1], maskshape[2]), dtype=np.float64)*fillValue for i in range(maskshape[1]): for j in range(maskshape[2]): percendone(i*maskshape[2] + j, maskshape[1]*maskshape[2], 5, 'quantiles') statval[:,i,j] = mask_quantiles(maskmat[:,i,j],Nquants) if dim == 1: statval = np.ones((21, maskshape[0], maskshape[2]), dtype=np.float64)*fillValue for i in range(maskshape[0]): for j in range(maskshape[2]): percendone(i*maskshape[2] + j, maskshape[0]*maskshape[2], 5, 'quantiles') statval[:,i,j] = mask_quantiles(maskmat[i,:,j],Nquants) if dim == 2: statval = np.ones((21, maskshape[0], maskshape[1]), dtype=np.float64)*fillValue for i in range(maskshape[0]): for j in range(maskshape[1]): percendone(i*maskshape[1] + j, maskshape[0]*maskshape[1], 5, 'quantiles') statval[:,i,j] = mask_quantiles(maskmat[i,j,:],Nquants) elif Ndims == 4: if dim == 0: statval = np.ones((21, maskshape[1], maskshape[2], maskshape[3]), dtype=np.float64)*fillValue for i in range(maskshape[1]): for j in range(maskshape[2]): for k in range(maskshape[3]): percendone(i*maskshape[1]*maskshape[2] + j*maskshape[2] + k, maskshape[1]*maskshape[2]*maskshape[3], 5, 'quantiles') statval[:,i,j,k] = mask_quantiles(maskmat[:,i,j,k],Nquants) if dim == 1: statval = np.ones((21, maskshape[0], maskshape[2], maskshape[3]), dtype=np.float64)*fillValue for i in range(maskshape[0]): for j in range(maskshape[2]): for k in range(maskshape[3]): percendone(i*maskshape[0]*maskshape[2] + j*maskshape[2] + k, maskshape[0]*maskshape[2]*maskshape[3], 5, 'quantiles') statval[:,i,j,k] = mask_quantiles(maskmat[i,:,j,k],Nquants) if dim == 2: statval = np.ones((21, maskshape[0], maskshape[1], maskshape[3]), dtype=np.float64)*fillValue for i in range(maskshape[0]): for j in range(maskshape[1]): for k in range(maskshape[3]): percendone(i*maskshape[0]*maskshape[1] + j*maskshape[1] + k, maskshape[0]*maskshape[1]*maskshape[3], 5, 'quantiles') statval[:,i,j,k] = mask_quantiles(maskmat[i,j,:,k],Nquants) if dim == 3: statval = np.ones((21, maskshape[0], maskshape[1], maskshape[2]), dtype=np.float64)*fillValue for i in range(maskshape[0]): for j in range(maskshape[1]): for k in range(maskshape[2]): percendone(i*maskshape[0]*maskshape[1] + j*maskshape[1] + k, maskshape[0]*maskshape[1]*maskshape[2], 5, 'quantiles') statval[:,i,j,k] = mask_quantiles(maskmat[i,j,k,:],Nquants) else: print errormsg print fname + ': size of matrix ', Ndims,'not ready!!!' print errormsg quit(-1) else: print errormsg print fname + ': statistics "' + statn + '" not ready!!!!' print errormsg quit(-1) print stdout.write("\n") return statval class statsVal(object): """Statistics class providing vals = variable self.Nv = number of values self.minv = minimum value self.maxv = maximum value self.meanv = mean value self.mean2v = cuadratic mean value self.stdv = standard deviation value self.Nokvalues = number of correct values of variable self.quantilesv = quantiles (%5 bins) of the variable """ def __init__(self, vals): if vals is None: self.Nv = None self.minv = None self.maxv = None self.meanv = None self.mean2v = None self.stdv = None self.Nokvalues = None self.quantilesv = None else: values = vals.flatten() self.Nv=len(values) self.minv=10000000000. self.maxv=-100000000. self.meanv=0. self.mean2v=0. self.stdv=0. sortedvalues = sorted(values) self.Nokvalues = 0 for inum in range(self.Nv): if not values[inum] is None: self.Nokvalues = self.Nokvalues + 1 if values[inum] < self.minv: self.minv = values[inum] if values[inum] > self.maxv: self.maxv = values[inum] self.meanv = self.meanv+values[inum] self.mean2v = self.mean2v+values[inum]*values[inum] self.meanv = self.meanv/float(self.Nokvalues) self.mean2v = self.mean2v/float(self.Nokvalues) self.stdv = np.sqrt(self.mean2v-self.meanv*self.meanv) self.quantilesv = [] for iq in range(20): self.quantilesv.append(sortedvalues[int((self.Nv-1)*iq/20)]) self.quantilesv.append(sortedvalues[self.Nv-1]) self.medianv = self.quantilesv[10] class Quantiles(object): """ Class to provide quantiles from a given arrayof values """ def __init__(self, values, Nquants): import numpy.ma as ma if values is None: self.quantilesv = None else: self.quantilesv = [] vals0 = values.flatten() Nvalues = len(vals0) vals = ma.masked_equal(vals0, None) Nvals=len(vals.compressed()) sortedvals = sorted(vals.compressed()) for iq in range(Nquants): self.quantilesv.append(sortedvals[int((Nvals-1)*iq/Nquants)]) self.quantilesv.append(sortedvals[Nvals-1]) class statsVal_missVal(object): """Statistics class tacking into account a missing value providing vals = variable missval = missing value self.Nv = number of values self.minv = minimum value self.maxv = maximum value self.meanv = mean value self.mean2v = cuadratic mean value self.stdv = standard deviation value self.Nokvalues = number of correct values of variable self.quantilesv = quantiles (%5 bins) of the variable """ def __init__(self, vals, missVal): fname='statsVal_missVal' if vals is None: self.Nv = None self.minv = None self.maxv = None self.meanv = None self.mean2v = None self.stdv = None self.Nokvalues = None self.quantilesv = None else: Npt = 1 for idim in range(len(vals.shape)): Npt = Npt * vals.shape[idim] if np.sum(vals >= missVal) == Npt: print errormsg print ' ' + fname + ' all values get missing!!' print errormsg quit(-1) vals1 = np.where(abs(vals) >= missVal, None, vals) vals0 = np.where(np.isnan(vals1), None, vals1) values = vals0.flatten() self.Nv=Npt self.minv=10000000000. self.maxv=-100000000. self.meanv=0. self.mean2v=0. self.stdv=0. sortedvalues = sorted(values) self.Nokvalues = 0 for inum in range(self.Nv): if not values[inum] is None: self.Nokvalues = self.Nokvalues + 1 if values[inum] < self.minv: self.minv = values[inum] if values[inum] > self.maxv: self.maxv = values[inum] self.meanv = self.meanv+values[inum] self.mean2v = self.mean2v+values[inum]*values[inum] self.meanv = self.meanv/float(self.Nokvalues) self.mean2v = self.mean2v/float(self.Nokvalues) self.stdv = np.sqrt(self.mean2v-self.meanv*self.meanv) self.quantilesv = [] for iq in range(20): self.quantilesv.append(sortedvalues[int((self.Nv-1)*iq/20)]) self.quantilesv.append(sortedvalues[self.Nv-1]) self.medianv = self.quantilesv[10] def spacemean(ncfile, varn): """ Function to retrieve a space mean series from a multidimensional variable of a file ncfile = netCDF file name varn = variable name """ import datetime as dt import calendar as cal ofile = 'spacemean_' + varn + '.nc' varfil=1.e20 statsn = ['min', 'max', 'mean', 'mean2', 'stdv', 'meanwgt', 'mean2wgt', 'stdvwgt', 'quant'] statslongn = ['minimum', 'maximum', 'mean', 'quadratic mean', 'standard deviation', 'weigthed mean', 'weigthed quadratic mean', 'weigthed standard deviation', 'quantiles'] ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' spacemean: File "' + ncfile + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) times = ncf.variables['time'] timevals = times[:] attvar = times.ncattrs() if not searchInlist(attvar, 'units'): print errormsg print ' spacemean: "time" does not have attribute: "units"' ncf.close() quit(-1) else: units = times.getncattr('units') txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: print ' spacemean: refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) dimt = len(timevals) realdates = np.zeros((dimt, 6), dtype=int) print realdates.shape ## Not in timedelta # if tunits == 'years': # for it in range(dimt): # realdate = refdate + dt.timedelta(years=float(times[it])) # realdates[it] = int(realdate.year) # elif tunits == 'months': # for it in range(dimt): # realdate = refdate + dt.timedelta(months=float(times[it])) # realdates[it] = int(realdate.year) if tunits == 'weeks': for it in range(dimt): realdate = refdate + dt.timedelta(weeks=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'days': for it in range(dimt): realdate = refdate + dt.timedelta(days=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'hours': for it in range(dimt): realdate = refdate + dt.timedelta(hours=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'minutes': for it in range(dimt): realdate = refdate + dt.timedelta(minutes=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'seconds': for it in range(dimt): realdate = refdate + dt.timedelta(seconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'milliseconds': for it in range(dimt): realdate = refdate + dt.timedelta(milliseconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) else: print errormsg print ' spacemean: time units "' + tunits + '" not ready!!!!' ncf.close() quit(-1) # Variable values (assuming time as first dimension) ## var = ncf.variables[varn] varinf = variable_inf(var) if searchInlist(varinf.attributes, 'scale_factor'): scalefact = var.getncattr('scale_factor') print ' spacemean: data has a scale factor of ', scalefact varinf.dtype=type(np.float(1.)) else: scalefact = 1. if searchInlist(varinf.attributes, 'add_offset'): offset = var.getncattr('add_offset') print ' spacemean: data has an offset of ', offset else: offset = 0. # varvals = var[:] vardims = var.shape varshape = len(vardims) vardimns = var.dimensions ## print ' spacemean: Shape of data: ',varshape, ':' , vardims dimt=vardims[0] vardnames = [] # Spatial average spatial weigthed ## lonvar = ncf.variables['lon'] latvar = ncf.variables['lat'] if not len(lonvar.shape) == 2: lonv = lonvar[:] latv = latvar[:] dx=lonvar.shape[0] dy=latvar.shape[0] lonval = np.zeros((dy, dx), dtype=np.float64) latval = np.zeros((dy, dx), dtype=np.float64) for iy in range(dy): lonval[iy,:] = lonv for ix in range(dx): latval[:,ix] = latv else: lonval = lonvar[:] latval = latvar[:] weightsv = abs(np.cos(latval*np.pi/180.)) if varinf.Ndims == 1: print errormsg print ' spacemean: You can not compute a space mean for a ', varinf.Ndims, 'D var!!!' ncf.close() quit(-1) elif varinf.Ndims== 2: print errormsg print ' spacemean: You can not compute a space mean for a ', varinf.Ndims, 'D var!!!' ncf.close() quit(-1) elif varinf.Ndims == 3: varstats = np.ones((dimt, 8), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((dimt, 21), dtype=np.float64) varquant = varquant*varfil vardnames.append(vardimns[0]) dy=vardims[1] dx=vardims[2] for it in range(dimt): if not varinf.FillValue is None: varvl = var[it,:,:] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[it,:,:] percendone(it,dimt,5,'computed space statistics') varst = statsVal(varval) varstwgt = statsValWeigthed(varval, weightsv) varstats[it,0] = varst.minv varstats[it,1] = varst.maxv varstats[it,2] = varst.meanv varstats[it,3] = varst.mean2v varstats[it,4] = varst.stdv varquant[it,:] = varst.quantilesv varstats[it,5] = varstwgt.meanv varstats[it,6] = varstwgt.mean2v varstats[it,7] = varstwgt.stdv elif varshape == 4: varstats = np.ones((dimt, vardims[1], 8), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((dimt, vardims[1], 21), dtype=np.float64) varquant = varquant*varfil vardimnames = (str(vardimns[0]), str(vardimns[1])) vardnames.append(vardimns[0]) vardnames.append(vardimns[1]) dy=vardims[2] dx=vardims[3] for it in range(dimt): for ik in range(vardims[1]): if not varinf.FillValue is None: varvl = var[it,ik,:,:] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[it,ik,:,:] percendone(it*vardims[1]+ik,dimt*vardims[1],5,'computed space statistics') varst = statsVal(varval) varstwgt = statsValWeigthed(varval, weightsv) varstats[it,ik,0] = varst.minv varstats[it,ik,1] = varst.maxv varstats[it,ik,2] = varst.meanv varstats[it,ik,3] = varst.mean2v varstats[it,ik,4] = varst.stdv varquant[it,ik,:] = varst.quantilesv varstats[it,ik,5] = varstwgt.meanv varstats[it,ik,6] = varstwgt.mean2v varstats[it,ik,7] = varstwgt.stdv elif varshape == 5: varstats = np.ones((dimt, vardims[1], vardims[2], 8), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((dimt, vardims[1], vardims[2], 21), dtype=np.float64) varquant = varquant*varfil vardnames.append(vardimns[0]) vardnames.append(vardimns[1]) vardnames.append(vardimns[2]) dy=vardims[3] dx=vardims[4] for it in range(dimt): for ik in range(vardims[1]): for il in range(vardims[2]): if not varinf.FillValue is None: varvl = var[it,ik,il,:,:] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[it,ik,il,:,:] percendone(it*vardims[1]*vardims[2]+ik*vardims[1]+il,dimt*vardims[1]*vardims[2],5,'computed space statistics') varst = statsVal(varval) varstwgt = statsValWeigthed(varval, weightsv) varstats[it,ik,il,0] = varst.minv varstats[it,ik,il,1] = varst.maxv varstats[it,ik,il,2] = varst.meanv varstats[it,ik,il,3] = varst.mean2v varstats[it,ik,il,4] = varst.stdv varquant[it,ik,il,:] = varst.quantilesv varstats[it,ik,il,5] = varstwgt.meanv varstats[it,ik,il,6] = varstwgt.mean2v varstats[it,ik,il,7] = varstwgt.stdv else: print errormsg print ' spacemean: ', varshape, ' shape of matrix not prepared !' ncf.close() quit(-1) vardimnames = tuple(vardnames) # print ' shape of desired values: ', vardesiredvalues.shape # Creation of file ## ncfo = NetCDFFile( ofile, 'w') vartype = var.dtype varattr = var.ncattrs() # Checking dimensions ## newdims = ncfo.dimensions Nvardnames = len(vardimnames) for idim in range(Nvardnames): rdim = vardimnames[idim] if not searchInlist(newdims, rdim): if not rdim == 'time': print ' spacemean: Adding dimension ' + rdim ncfo.sync() ncf.close() ncfo.close() fdimadd(ncfile + ',' + rdim, ofile) ncf = NetCDFFile(ncfile,'r') ncfo = NetCDFFile(ofile,'a') else: ncfo.createDimension('time', None) # Checking fill value ## if searchInlist(varattr, '_FillValue'): varfil = var._FillValue else: varfil = False Nstats = len(statsn) for ist in range(Nstats): if statsn[ist] == 'quant': print ist, statsn[ist]##, ': ', varquant[int(dimt/2),10] newdim = ncfo.createDimension('quant', 21) newvardnames = list(vardnames) newvardnames.append('quant') newvar = ncfo.createVariable(varn + statsn[ist], varinf.dtype, tuple(newvardnames), fill_value=varfil) newvar[:] = varquant else: print ist, statsn[ist]##, ': ', varstats[int(dimt/2),ist] newvar = ncfo.createVariable(varn + statsn[ist], varinf.dtype, vardimnames, fill_value=varfil) if varshape == 3: newvar[:] = varstats[:,ist]*1. elif varshape == 4: newvar[:] = varstats[:,:,ist]*1. elif varshape == 5: newvar[:] = varstats[:,:,:,ist]*1. newvar = ncfo.variables[varn + statsn[ist]] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = var.getncattr(attr) if not searchInlist(newvarattrs, attr): if attr != 'scale_factor' and attr != 'add_offset' and attr != 'valid_range' \ and attr != 'unpacked_valid_range' and attr != 'actual_range' : # if not statsn[ist] == 'stdv' and not statsn[ist] == 'stdvwgt': print 'attr:', attr newvar.setncattr(attr, attrv) # else: # newvar.setncattr(attr, attrv) newvar.setncattr('cell_methods', 'space ' + statslongn[ist] + ' all domain ' + str(dy) + 'x' + str(dx)) ## print ' Adding time variable' vartdims = times.dimensions vartype = times.dtype varattr = times.ncattrs() newvar = ncfo.createVariable('time', vartype, vartdims, fill_value=varfil) newvar = ncfo.variables['time'] newvar[:] = times ncf.close() ncfo.sync() ncfo.close() fattradd('time', ncfile + ':time', ofile) fvaradd(ncfile + ',lon', ofile) fvaradd(ncfile + ',lat', ofile) ncfo = NetCDFFile(ofile,'a') newvar = ncfo.variables['time'] newvarattrs = newvar.ncattrs() if searchInlist(newvarattrs, 'bounds'): if newvar.getncattr('bounds') == 'time_bnds': ncf = NetCDFFile(ncfile,'r') tbnds = ncf.variables['time_bnds'] vardims = tbnds.dimensions vartype = tbnds.dtype varattr = tbnds.ncattrs() ncfo.createDimension('bnds', 2) newvar = ncfo.createVariable('time_bnds', vartype, vardims, fill_value=varfil) newvar[:] = tbnds[:] ncf.close() ncfo.sync() ncfo.close() fattradd('time_bnds', ncfile + ':time_bnds', ofile) else: ncfo.close() else: ncfo.close() fgaddattr(ncfile, ofile) print ' spacemean: File "' + ofile + '" as space mean of "' + varn + '" has been created' def timemean(values, ncfile, varn): """ Function to retrieve a time mean series from a multidimensional variable of a file values = power of the polynomial fitting with time to be applied ncfile = netCDF file name varn = variable name """ import datetime as dt import calendar as cal powerv=int(values) ofile = 'timemean_' + varn + '.nc' varfil=1.e20 statsn = ['min', 'max', 'mean', 'mean2', 'stdv', 'quant','linregress','polregress'] statslongn = ['minimum', 'maximum', 'mean', 'quadratic mean', 'standard deviation', 'quantiles', \ 'linear regression', 'polynomial regression'] ncf = NetCDFFile(ncfile,'r') if not ncf.variables.has_key(varn): print errormsg print ' timemean: File "' + ncfile + '" does not have variable ' + varn + ' !!!!' print errormsg ncf.close() quit(-1) times = ncf.variables['time'] timevals = times[:] attvar = times.ncattrs() if not searchInlist(attvar, 'units'): print errormsg print ' timemean: "time" does not have attribute: "units"' ncf.close() quit(-1) else: units = times.getncattr('units') txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: print ' timemean: refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) dimt = len(timevals) realdates = np.zeros((dimt, 6), dtype=int) print realdates.shape ## Not in timedelta # if tunits == 'years': # for it in range(dimt): # realdate = refdate + dt.timedelta(years=float(times[it])) # realdates[it] = int(realdate.year) # elif tunits == 'months': # for it in range(dimt): # realdate = refdate + dt.timedelta(months=float(times[it])) # realdates[it] = int(realdate.year) if tunits == 'weeks': for it in range(dimt): realdate = refdate + dt.timedelta(weeks=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'days': for it in range(dimt): realdate = refdate + dt.timedelta(days=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'hours': for it in range(dimt): realdate = refdate + dt.timedelta(hours=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'minutes': for it in range(dimt): realdate = refdate + dt.timedelta(minutes=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'seconds': for it in range(dimt): realdate = refdate + dt.timedelta(seconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) elif tunits == 'milliseconds': for it in range(dimt): realdate = refdate + dt.timedelta(milliseconds=float(times[it])) realdates[it,0] = int(realdate.year) realdates[it,1] = int(realdate.month) realdates[it,2] = int(realdate.day) realdates[it,3] = int(realdate.hour) realdates[it,4] = int(realdate.second) realdates[it,5] = int(realdate.minute) else: print errormsg print ' timemean: time units "' + tunits + '" not ready!!!!' ncf.close() quit(-1) timesv = (realdates[:,0] - realdates[0,0])*12 + realdates[:,1] # Variable values (assuming time as first dimension) ## var = ncf.variables[varn] varinf = variable_inf(var) # varvals = var[:] vardims = varinf.dims varshape = varinf.Ndims vardimns = var.dimensions ## print ' timemean: Shape of data: ',varshape, ':' , vardims dimt=vardims[0] vardnames = [] if varshape == 1: print errormsg print ' timemean: You can not compute a time mean for a ', varshape, 'D var!!!' ncf.close() quit(-1) elif varshape == 2: dx=vardims[1] varstats = np.ones((5, dx), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((21, dx), dtype=np.float64) varquant = varquant*varfil varlregress = np.ones((5, dx), dtype=np.float64) varlregress = varlregress*varfil varpregress = np.ones((powerv+1, dx), dtype=np.float64) varpregress = varpregress*varfil varpregressres = np.ones((dx), dtype=np.float64) varpregressres = varpregressres*varfil varpregresssingval = varpregress.copy() varpregresssingval = varpregresssingval*varfil vardnames.append(vardimns[1]) for ix in range(dx): if not varinf.FillValue is None: varvl = var[:,ix] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[:,ix] percendone(ix,dx,5,'computed time statistics') varst = statsVal(varval) var2st = stats2Val(timesv, varval, powerv) varstats[0,ix] = varst.minv varstats[1,ix] = varst.maxv varstats[2,ix] = varst.meanv varstats[3,ix] = varst.mean2v varstats[4,ix] = varst.stdv varquant[:,ix] = varst.quantilesv varlregress[:,ix] = var2st.linRegress varpregress[:,ix] = var2st.polRegress varpregressres[ix] = var2st.polRegressRes varpregresssingval[:,ix] = var2st.polRegressSingVal elif varshape == 3: dy=vardims[1] dx=vardims[2] varstats = np.ones((5,dy, dx), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((21,dy, dx), dtype=np.float64) varquant = varquant*varfil varlregress = np.ones((5,dy, dx), dtype=np.float64) varlregress = varlregress*varfil varpregress = np.ones((powerv+1,dy, dx), dtype=np.float64) varpregress = varpregress*varfil varpregressres = np.ones((dy,dx), dtype=np.float64) varpregressres = varpregressres*varfil varpregresssingval = varpregress.copy() varpregresssingval = varpregresssingval*varfil vardnames.append(vardimns[1]) vardnames.append(vardimns[2]) for iy in range(dy): for ix in range(dx): if not varinf.FillValue is None: varvl = var[:,iy,ix] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[:,iy,ix] percendone(iy*dx+ix,dy*dx,5,'computed time statistics') varst = statsVal(varval) var2st = stats2Val(timesv, varval, powerv) varstats[0,iy,ix] = varst.minv varstats[1,iy,ix] = varst.maxv varstats[2,iy,ix] = varst.meanv varstats[3,iy,ix] = varst.mean2v varstats[4,iy,ix] = varst.stdv varquant[:,iy,ix] = varst.quantilesv varlregress[:,iy,ix] = var2st.linRegress varpregress[:,iy,ix] = var2st.polRegress varpregressres[iy,ix] = var2st.polRegressRes varpregresssingval[:,iy,ix] = var2st.polRegressSingVal elif varshape == 4: dz=vardims[1] dy=vardims[2] dx=vardims[3] varstats = np.ones((5, dz, dy, dx), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((21, dz, dy, dx), dtype=np.float64) varquant = varquant*varfil varlregress = np.ones((5, dz, dy, dx), dtype=np.float64) varlregress = varlregress*varfil varpregress = np.ones((powerv+1, dz, dy, dx), dtype=np.float64) varpregress = varpregress*varfil varpregressres = np.ones((dz,dy,dx), dtype=np.float64) varpregressres = varpregressres*varfil varpregresssingval = varpregress.copy() varpregresssingval = varpregresssingval*varfil vardnames.append(vardimns[1]) vardnames.append(vardimns[2]) vardnames.append(vardimns[3]) for iz in range(dz): for iy in range(dy): for ix in range(dx): if not varinf.FillValue is None: varvl = var[:,iz,iy,ix] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[:,iz,iy,ix] percendone(iz*dy*dx+iy*dx+ix,dz*dx*dy,5,'computed time statistics') varst = statsVal(varval) var2st = stats2Val(timesv, varval, powerv) varstats[0,iz,iy,ix] = varst.minv varstats[1,iz,iy,ix] = varst.maxv varstats[2,iz,iy,ix] = varst.meanv varstats[3,iz,iy,ix] = varst.mean2v varstats[4,iz,iy,ix] = varst.stdv varquant[:,iz,iy,ix] = varst.quantilesv varlregress[:,iz,iy,ix] = var2st.linRegress varpregress[:,iz,iy,ix] = var2st.polRegress varpregressres[iz,iy,ix] = var2st.polRegressRes varpregresssingval[:,iz,iy,ix] = var2st.polRegressSingVal elif varshape == 5: dn=vardims[1] dz=vardims[2] dy=vardims[3] dx=vardims[4] varstats = np.ones((5, dn, dz, dy, dx), dtype=np.float64) varstats = varstats*varfil varquant = np.ones((21, dn, dz, dy, dx), dtype=np.float64) varquant = varquant*varfil varlregress = np.ones((5, dn, dz, dy, dx), dtype=np.float64) varlregress = varlregress*varfil varpregress = np.ones((powerv+1, dn, dz, dy, dx), dtype=np.float64) varpregress = varpregress*varfil varpregressres = np.ones((dn,dz,dy,dx), dtype=np.float64) varpregressres = varpregressres*varfil varpregresssingval = varpregress.copy() varpregresssingval = varpregresssingval*varfil vardnames.append(vardimns[1]) vardnames.append(vardimns[2]) vardnames.append(vardimns[3]) vardnames.append(vardimns[4]) for iN in range(dn): for iz in range(dz): for iy in range(dy): for ix in range(dx): if not varinf.FillValue is None: varvl = var[:,iN,iz,iy,ix] varval = np.where(varvl == varinf.FillValue, None, varvl) else: varval = var[:,iN,iz,iy,ix] percendone(iN*dy*dx*dz+iz*dy*dx+iy*dx+ix,dn*dz*dx*dy,5,'computed time statistics') varst = statsVal(varval) var2st = stats2Val(timesv, varval, powerv) varstats[0,iN,iz,iy,ix] = varst.minv varstats[1,iN,iz,iy,ix] = varst.maxv varstats[2,iN,iz,iy,ix] = varst.meanv varstats[3,iN,iz,iy,ix] = varst.mean2v varstats[4,iN,iz,iy,ix] = varst.stdv varquant[:,iN,iz,iy,iiN,x] = varst.quantilesv varlregress[:,iN,iz,iy,ix] = var2st.linRegress varpregress[:,iN,iz,iy,ix] = var2st.polRegress varpregressres[iN,iz,iy,ix] = var2st.polRegressRes varpregresssingval[:,iN,iz,iy,ix] = var2st.polRegressSingVal else: print errormsg print ' timemean: ', varshape, ' shape of matrix not prepared !' ncf.close() quit(-1) vardimnames = tuple(vardnames) # print ' shape of desired values: ', vardesiredvalues.shape # Creation of file ## ncfo = NetCDFFile( ofile, 'w') vartype = var.dtype varattr = var.ncattrs() # Checking dimensions ## newdims = ncfo.dimensions Nvardnames = len(vardimnames) for idim in range(Nvardnames): rdim = vardimnames[idim] if not searchInlist(newdims, rdim): if not rdim == 'time': ## print ' timemean: Adding dimension ' + rdim ncfo.sync() ncf.close() ncfo.close() fdimadd(ncfile + ',' + rdim, ofile) ncf = NetCDFFile(ncfile,'r') ncfo = NetCDFFile(ofile,'a') else: print ' timemean: No time dimension!' # Checking fill value ## if searchInlist(varattr, '_FillValue'): varfil = var._FillValue else: varfil = False Nstats = len(statsn) for ist in range(Nstats): newvardnames = [] if statsn[ist] == 'quant': print ist, statsn[ist]##, ': ', varquant[int(dimt/2),10] newdim = ncfo.createDimension('quant', 21) newvardnames.append('quant') newvardnames = newvardnames + list(vardnames) newvar = ncfo.createVariable(varn + statsn[ist], vartype, tuple(newvardnames), fill_value=varfil) newvar[:] = varquant elif statsn[ist] == 'linregress': print ist, statsn[ist]##, ': ', varquant[int(dimt/2),10] newdim = ncfo.createDimension('lregress', 5) newvar = ncfo.createVariable('lregressn', str, ('lregress',)) newvar[0] = 'slope' newvar[1] = 'intercept' newvar[2] = 'r_value' newvar[3] = 'p_value' newvar[4] = 'std_err' newvardnames.append('lregress') newvardnames = newvardnames + list(vardnames) newvar = ncfo.createVariable(varn + statsn[ist], 'f4', tuple(newvardnames), fill_value=varfil) newvar[:] = varlregress*1. elif statsn[ist] == 'polregress': print ist, statsn[ist]##, ': ', varquant[int(dimt/2),10] newdim = ncfo.createDimension('pregress', powerv+1) newvar = ncfo.createVariable('pregressn', str, ('pregress',)) for ip in range(powerv+1): newvar[ip]='coefficient**' + str(powerv-ip) newvardnames.append('pregress') newvardnames = newvardnames + list(vardnames) newvar = ncfo.createVariable(varn + statsn[ist], 'f4', tuple(newvardnames), fill_value=varfil) newvar[:] = varpregress*1. newvar.setncattr('power',powerv) newvar.setncattr('values','Polynomial coefficients, highest power first') newvar = ncfo.createVariable(varn + statsn[ist] + '_Residual', vartype, tuple(vardnames), fill_value=varfil) newvar[:] = varpregressres newvar.setncattr('power',powerv) newvar.setncattr('values','Polynomial residuals') newvar = ncfo.createVariable(varn + statsn[ist] + '_VandermondeSingularVector', vartype, tuple(newvardnames), fill_value=varfil) newvar[:] = varpregresssingval newvar.setncattr('power',powerv) newvar.setncattr('values','Polynomial coefficients, highest power first') else: print ist, statsn[ist]##, ': ', varstats[int(dimt/2),ist] if statsn[ist] == 'mean' or statsn[ist] == 'stdv' or statsn[ist] == 'mean2' or statsn[ist] == 'polregress_Residual' \ or statsn[ist] == 'polregress_VandermondeSingularVector' and searchInlist(varattr, 'scale_factor'): newvar = ncfo.createVariable(varn + statsn[ist], 'f4', vardimnames, fill_value=varfil) if varshape == 2: newvar[:] = varstats[ist,:]*1. elif varshape == 3: newvar[:] = varstats[ist,:,:]*1. elif varshape == 4: newvar[:] = varstats[ist,:,:,:]*1. elif varshape == 5: newvar[:] = varstats[ist,:,:,:,:]*1. else: newvar = ncfo.createVariable(varn + statsn[ist], vartype, vardimnames, fill_value=varfil) if varshape == 2: newvar[:] = varstats[ist,:] elif varshape == 3: newvar[:] = varstats[ist,:,:] elif varshape == 4: newvar[:] = varstats[ist,:,:,:] elif varshape == 5: newvar[:] = varstats[ist,:,:,:,:] newvar = ncfo.variables[varn + statsn[ist]] for attr in varattr: newvarattrs = newvar.ncattrs() attrv = var.getncattr(attr) if not searchInlist(newvarattrs, attr): if attr == 'scale_factor' or attr == 'add_offset' or attr == 'valid_range' \ or attr == 'unpacked_valid_range' or attr == 'actual_range' : if not statsn[ist] == 'mean' and not statsn[ist] == 'stdv' and not statsn[ist] == 'mean2' and not \ statsn[ist] == 'linregress' and not statsn[ist] == 'polregress' \ and not statsn[ist] == 'polregress_Residual' and not \ statsn[ist] == 'polregress_VandermondeSingularVector': newvar.setncattr(attr, attrv) else: newvar.setncattr(attr, attrv) newvar.setncattr('cell_methods', 'time ' + statslongn[ist] + ' all period in file ' + str(dimt) + ' time steps') ncfo.sync() ncfo.close() fvaradd(ncfile + ',lon', ofile) fvaradd(ncfile + ',lat', ofile) fgaddattr(ncfile, ofile) print ' timemean: File "' + ofile + '" as time mean of "' + varn + '" has been created' def printing_class(classobj): """ Function to print all the values of a given class """ valscls = vars(classobj) for attrcls in valscls: print attrcls, ':', valscls[attrcls] def fmtprinting_class(classobj): """ Function to print all the values of a given class """ valscls = vars(classobj) for attrcls in valscls: print '@' + attrcls + '@', ':', valscls[attrcls] return def flipdim(values, filename, varn): """ flips the value following one dimension [Nflipdim] values=[Nflipdim]:[flipdim] [Nflipdim]: number of the dimension to flip [flipdim]: has also the corresondant variable dimensinon be flipped? (yes/no) filename= netCDF file name varn= variable name """ fname = 'flipdim' if values == 'h': print fname + '_____________________________________________________________' print flipdim.__doc__ quit() arguments = '[Nflipdim]:[flipdim]' check_arguments(fname, values, arguments, ':') ncf = NetCDFFile(filename,'a') Nflipdim=int(values.split(':')[0]) flipdim=values.split(':')[1] if not ncf.variables.has_key(varn): print errormsg print ' flipdim: File "' + filename + '" does not have variable ' + varn + ' !!!!' print errormsg ncf.close() quit(-1) varnc = ncf.variables[varn] varinf = variable_inf(varnc) if varinf.Ndims < Nflipdim: print errormsg print ' flipdim: variable "' + varn + '" has less dimensions ', varinf.Ndims, 'than the required to flip "', Nflipdim, '!!!!' print errormsg quit(-1) if flipdim == 'yes': flipdimname = varinf.dimns[Nflipdim] print ' flipdim: Flipping also associated dimension variable "' + flipdimname + '"' flipdim = ncf.variables[flipdimname] if len(flipdim.shape) == 1: newdim = flipdim[:] flipdim[:] = newdim[::-1] elif len(flipdim.shape) == 2: if Nflipdim == varinf.Ndims: for i in range(flipdim.shape[0]): newdim = flipdim[i,:] flipdim[i,:] = newdim[::-1] if Nflipdim == varinf.Ndims - 1: for i in range(flipdim.shape[1]): newdim = flipdim[:,i] flipdim[:,:] = newdim[::-1] else: print errormsg print ' flipdim: dimension to flip has ', len(flipdim.shape), ' dimensions, not ready to be flipped !!!' print errormsg quit(-1) newvar = np.ones(varinf.dims, dtype=varinf.dtype) if varinf.Ndims == 1: newvar = varnc[:] varnc[:] = newvar[::-1] elif varinf.Ndims == 2: if Nflipdim == 1: for i in range(varinf.dims[0]): newvar[i,:] = varnc[i,:] varnc[i,:] = newvar[i,::-1] elif Nflipdim == 0: for i in range(varinf.dims[1]): newvar[:,i] = varnc[:,i] varnc[:,i] = newvar[::-1,i] elif varinf.Ndims == 3: if Nflipdim == 2: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): newvar[i,j,:] = varnc[i,j,:] varnc[i,j,:] = newvar[i,j,::-1] elif Nflipdim == 1: for i in range(varinf.dims[0]): for j in range(varinf.dims[2]): newvar[i,:,j] = varnc[i,:,j] varnc[i,:,j] = newvar[i,::-1,j] elif Nflipdim == 0: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): newvar[:,i,j] = varnc[:,i,j] varnc[:,i,j] = newvar[::-1,i,j] elif varinf.Ndims == 4: if Nflipdim == 3: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[2]): newvar[i,j,k,:] = varnc[i,j,k,:] varnc[i,j,k,:] = newvar[i,j,k,::-1] if Nflipdim == 2: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[3]): newvar[i,j,:,k] = varnc[i,j,:,k] varnc[i,j,:,k] = newvar[i,j,::-1,k] if Nflipdim == 1: for i in range(varinf.dims[0]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): newvar[i,:,j,k] = varnc[i,:,j,k] varnc[i,:,j,k] = newvar[i,::-1,j,k] if Nflipdim == 0: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): newvar[:,i,j,k] = varnc[:,i,j,k] varnc[:,i,j,k] = newvar[::-1,i,j,k] elif varinf.Ndims == 5: if Nflipdim == 4: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[2]): for l in range(varinf.dims[3]): newvar[i,j,k,l,:] = varnc[i,j,k,l,:] varnc[i,j,k,l,:] = newvar[i,j,k,l,::-1] if Nflipdim == 3: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[2]): for l in range(varinf.dims[4]): newvar[i,j,k,:,l] = varnc[i,j,k,:,l] var[i,j,k,:,l] = newvar[i,j,k,::-1,l] if Nflipdim == 2: for i in range(varinf.dims[0]): for j in range(varinf.dims[1]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): newvar[i,j,:,k,l] = varnc[i,j,:,k,l] varnc[i,j,:,k,l] = newvar[i,j,::-1,k,l] if Nflipdim == 1: for i in range(varinf.dims[0]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): newvar[i,:,j,k,l] = varnc[i,:,j,k,l] varnc[i,:,j,k,l] = newvar[i,::-1,j,k,l] if Nflipdim == 0: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): newvar[:,i,j,k,l] = varnc[:,i,j,k,l] varnc[:,i,j,k,l] = newvar[::-1,i,j,k,l] else: print errormsg print ' flipdim: Number of dimensions ', varinf.Ndims, ' not ready!!!!' print errormsg quit(-1) ncf.sync() ncf.close() def cycl_incr(cyclval,cycle,ninc): """ Function to increment a cyclic value [cyclval] with a cycle [cycle] a given number [ninc] of times >>> cycl_incr(1,4,1) 2 >>> cycl_incr(3,4,1) 0 >>> cycl_incr(1,4,10) 3 """ if ninc >= cycle: print 'cycl_incr: WARNING -- warning -- WARNING -- warning' print ' given increment: ', ninc,' is larger than the cycle !!' ninc = ninc - cycle*int(ninc/cycle) print ' reducing it to: ', ninc val=cyclval + ninc if val >= cycle: val=cyclval + ninc - cycle return val def times_4seasons(tvals, integrity): """ Function to split a time series in matricial date format ([:,year,mon,day,hour,minute,second]) in the four stations DJF,MAM,JJA,SON tvals= matrix withe times as [:,year,mon,day,hour,minute,second] integrity= only give values for entire seasons [True, 3 months for the season], or just split the values by seasons [False] """ fillVal=1.e20 # print tvals dt=tvals.shape[0] seasons=np.ones((dt,4),dtype=bool) seasons=seasons*False monseas=[12,3,6,9] firstseas=False for it in range(dt): if not firstseas: if integrity: for iseas in range(4): if tvals[it,1] == monseas[iseas]: nseas=iseas seasons[it,nseas]=True firstseas=True begseas=it else: for iseas in range(4): for imon in range(3): if tvals[it,1] == cycl_incr(monseas[iseas],12,imon): nseas=iseas seasons[it,nseas]=True firstseas=True else: newseas=cycl_incr(nseas,4,1) if tvals[it,1] == monseas[newseas]: seasons[it,newseas] = True nseas=newseas begseas=it else: seasons[it,nseas] = True endseas = it ## print 'Last season: ',nseas,' beginnig: ',begseas,' ending: ',endseas # Checking integrity of last season (has to have 3 months) ## if integrity: fullseas=True Nmon=np.unique(tvals[begseas:endseas+1,1]) for it in range(begseas,endseas): fullseas=fullseas*seasons[it,nseas] if len(Nmon) < 3 or not fullseas: seasons[begseas:endseas+1,nseas] = False return seasons def realdatetime_CFcompilant(times, Srefdate, tunits): """ Function to transform a matrix with real time values ([year, month, day, hour, minute, second]) to a netCDF one times= matrix with times Srefdate= reference date ([YYYY][MM][DD][HH][MI][SS] format) tunits= units of time respect to Srefdate >>> realdatetime_CFcompilant(np.array([ [1976, 2, 17, 8, 20, 0], [1976, 2, 18, 8, 20, 0]], dtype=int), '19491201000000', 'hours') [ 229784.33333333 229808.33333333] """ import datetime as dt yrref=int(Srefdate[0:4]) monref=int(Srefdate[4:6]) dayref=int(Srefdate[6:8]) horref=int(Srefdate[8:10]) minref=int(Srefdate[10:12]) secref=int(Srefdate[12:14]) refdate=dt.datetime(yrref, monref, dayref, horref, minref, secref) dimt=times.shape[0] cfdates = np.zeros((dimt), dtype=np.float64) if tunits == 'weeks': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = (cfdate.days + cfdate.seconds/(3600.*24.))/7. elif tunits == 'days': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days + cfdate.seconds/(3600.*24.) elif tunits == 'hours': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*24. + cfdate.seconds/3600. elif tunits == 'minutes': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*24.*60. + cfdate.seconds/60. elif tunits == 'seconds': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*24.*3600. + cfdate.seconds elif tunits == 'milliseconds': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*1000.*24.*3600. + cfdate.seconds*1000. elif tunits == 'microseconds': for it in range(dimt): cfdate = dt.datetime(times[it,0], times[it,1], times[it,2], times[it,3], times[it,4], times[it,5]) - refdate cfdates[it] = cfdate.days*1000000.*24.*3600. + cfdate.seconds*1000000. else: print errormsg print ' ' + fname + ': time units "' + tunits + '" is not ready!!!' quit(-1) return cfdates def realdatetime1_CFcompilant(time, Srefdate, tunits): """ Function to transform a matrix with a real time value ([year, month, day, hour, minute, second]) to a netCDF one time= matrix with time Srefdate= reference date ([YYYY][MM][DD][HH][MI][SS] format) tunits= units of time respect to Srefdate >>> realdatetime1_CFcompilant([1976, 2, 17, 8, 20, 0], '19491201000000', 'hours') 229784.33333333 """ import datetime as dt yrref=int(Srefdate[0:4]) monref=int(Srefdate[4:6]) dayref=int(Srefdate[6:8]) horref=int(Srefdate[8:10]) minref=int(Srefdate[10:12]) secref=int(Srefdate[12:14]) refdate=dt.datetime(yrref, monref, dayref, horref, minref, secref) if tunits == 'weeks': cfdate = dt.datetime(time[0],time[1],time[2],time[3],time[4],time[5])-refdate cfdates = (cfdate.days + cfdate.seconds/(3600.*24.))/7. elif tunits == 'days': cfdate = dt.datetime(time[0],time[1],time[2],time[3],time[4],time[5]) - refdate cfdates = cfdate.days + cfdate.seconds/(3600.*24.) elif tunits == 'hours': cfdate = dt.datetime(time[0],time[1],time[2],time[3],time[4],time[5]) - refdate cfdates = cfdate.days*24. + cfdate.seconds/3600. elif tunits == 'minutes': cfdate = dt.datetime(time[0],time[1],time[2],time[3],time[4],time[5]) - refdate cfdates = cfdate.days*24.*60. + cfdate.seconds/60. elif tunits == 'seconds': cfdate = dt.datetime(time[0],time[1],time[2],time[3],time[4],time[5]) - refdate cfdates = cfdate.days*24.*3600. + cfdate.seconds elif tunits == 'milliseconds': cfdate = dt.datetime(time[0],time[1],time[2],time[3],time[4],time[5]) - refdate cfdates = cfdate.days*1000.*24.*3600. + cfdate.seconds*1000. elif tunits == 'microseconds': cfdate = dt.datetime(time[0],time[1],time[2],time[3],time[4],times[5]) - refdate cfdates = cfdate.days*1000000.*24.*3600. + cfdate.seconds*1000000. else: print errormsg print ' ' + fname + ': time units "' + tunits + '" is not ready!!!' quit(-1) return cfdates def netCDFdatetime_realdatetime(units, tcalendar, times): """ Function to transfrom from netCDF CF-compilant times to real time [units]= CF time units [tunits] since [YYYY]-[MM]-[HH] [[HH]:[MI]:[SS]] [tcalendar]= time calendar [times]= CF time values """ import datetime as dt txtunits = units.split(' ') tunits = txtunits[0] Srefdate = txtunits[len(txtunits) - 1] # Calendar type ## is360 = False if tcalendar is not None: print ' netCDFdatetime_realdatetime: There is a calendar attribute' if tcalendar == '365_day' or tcalendar == 'noleap': print ' netCDFdatetime_realdatetime: No leap years!' isleapcal = False elif tcalendar == 'proleptic_gregorian' or tcalendar == 'standard' or tcalendar == 'gregorian': isleapcal = True elif tcalendar == '360_day': is360 = True isleapcal = False else: print errormsg print ' netCDFdatetime_realdatetime: Calendar "' + tcalendar + '" not prepared!' quit(-1) # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## timeval = Srefdate.find(':') if not timeval == -1: print ' netCDFdatetime_realdatetime: refdate with time!' refdate = datetimeStr_datetime(txtunits[len(txtunits) - 2] + '_' + Srefdate) else: refdate = dateStr_date(Srefdate) dimt = len(times) # datetype = type(dt.datetime(1972,02,01)) # realdates = np.array(dimt, datetype) # print realdates ## Not in timedelta # if tunits == 'years': # for it in range(dimt): # realdate = refdate + dt.timedelta(years=float(times[it])) # realdates[it] = int(realdate.year) # elif tunits == 'months': # for it in range(dimt): # realdate = refdate + dt.timedelta(months=float(times[it])) # realdates[it] = int(realdate.year) # realdates = [] realdates = np.zeros((dimt, 6), dtype=int) if tunits == 'weeks': for it in range(dimt): realdate = refdate + dt.timedelta(weeks=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'days': for it in range(dimt): realdate = refdate + dt.timedelta(days=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'hours': for it in range(dimt): realdate = refdate + dt.timedelta(hours=float(times[it])) # if not isleapcal: # Nleapdays = cal.leapdays(int(refdate.year), int(realdate.year)) # realdate = realdate - dt.timedelta(days=Nleapdays) # if is360: # Nyears360 = int(realdate.year) - int(refdate.year) + 1 # realdate = realdate -dt.timedelta(days=Nyears360*5) # realdates[it] = realdate # realdates = refdate + dt.timedelta(hours=float(times)) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'minutes': for it in range(dimt): realdate = refdate + dt.timedelta(minutes=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'seconds': for it in range(dimt): realdate = refdate + dt.timedelta(seconds=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'milliseconds': for it in range(dimt): realdate = refdate + dt.timedelta(milliseconds=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] elif tunits == 'microseconds': for it in range(dimt): realdate = refdate + dt.timedelta(microseconds=float(times[it])) realdates[it,:]=[realdate.year, realdate.month, realdate.day, realdate.hour, realdate.minute, realdate.second] else: print errormsg print ' netCDFdatetime_realdatetime: time units "' + tunits + '" is not ready!!!' quit(-1) return realdates class cls_yearly_means(object): """ Class to compute the yearly mean of a time series of values tvals=time values (year, month, day, hour, minute, second) format values=time-series of values dateref= reference date [YYYY][MM][DD][HH][MI][SS] format tunits= time units self.dimt = Number of values self.yrv = yearly mean values self.yrt = years of the mean values >>> timesv = netCDFdatetime_realdatetime('days since 1949-12-01 00:00:00', 'standard', np.arange(1129)) >>> valuesv = np.zeros((1129), dtype=np.float) >>> valuesv[0:31] = 0. >>> valuesv[31:394] = 1. >>> valuesv[395:761] = 2. >>> valuesv[761:1127] = 3. >>> valuesv[1127:1129] = 4. >>> yrlmeans = cls_yearly_means(timesv, valuesv, '19491201000000', 'days') >>> print yrlmeans.dimt,yrlmeans.yrv, yrlmeans.yrt 5 [ 0. 1. 2. 3. 4.] [ 1949. 1950. 1951. 1952. 1953.] """ def __init__(self, tvals, values, dateref, tunits): import numpy.ma as ma fillVal=1.e20 fname = 'cls_yearly_means' if tvals is None: self.dimt = None self.yrv = None self.yrt = None elif tvals == 'h': print fname + '_____________________________________________________________' print cls_yearly_means.__doc__ quit() else: dt=len(values) if dt > 0: years = np.unique(tvals[:,0]) Nyrs = len(years) yrmean = np.zeros((Nyrs), dtype=np.float) yrt = np.zeros((Nyrs), dtype=np.float) iiyr = tvals[0,0] yrmean[0] = values[0] yrt[0] = iiyr iit = 0 for iyr in range(Nyrs): nvals = 1 for it in range(iit+1,dt): # print iyr, iiyr, it, tvals[it,0],':',values[it],'->',yrmean[iyr] if tvals[it,0] == iiyr: yrmean[iyr] = yrmean[iyr] + values[it] nvals = nvals + 1 else: yrmean[iyr] = yrmean[iyr] / (nvals*1.) iiyr = tvals[it,0] yrmean[iyr + 1] = values[it] yrt[iyr+1] = iiyr iit = it break yrmean[Nyrs-1] = yrmean[Nyrs-1]/nvals self.dimt = Nyrs self.yrv = yrmean self.yrt = yrt else: print errormsg print ' ' + fname + ': No values passed!' print ' values:', values quit(-1) class cls_seasonal_means(object): """ Class to compute the seasonal mean of a time series of values tvals=time values (year, month, day, hour, minute, second) format values=time-series of values dateref= reference date [YYYY][MM][DD][HH][MI][SS] format tunits= time units self.dimt = Number of values self.seasv = seasonal mean values self.seast = seasonal time mean values """ def __init__(self, tvals, values, dateref, tunits): import numpy.ma as ma fillVal=1.e20 fname = 'cls_seasonal_means' if tvals is None: self.dimt = None self.seasv = None self.seast = None else: tseasons=times_4seasons(tvals, True) dt=len(values) seasvals=np.ones((dt/12,4), dtype=np.float64)*fillVal seastimes=np.zeros((dt/12,4), dtype=np.float64) dates = realdatetime_CFcompilant(tvals, dateref, tunits) for iseas in range(4): for it in range(dt): if tseasons[it,iseas]: tfirstseas=int(it/11) firstseas=True ## print ' first time-step of season ', iseas, ' is: ',it,' position: ',tfirstseas break for itt in range(it, dt-1,12): seasvals[int(itt/12),iseas]=(values[itt] + values[itt+1] + values[itt+2])/3. seastimes[int(itt/12),iseas]=dates[itt] + (dates[itt+2] - dates[itt])/2. ## print itt, values[itt], values[itt+1], values[itt+2], '--->', seasvals[int(itt/12),iseas] self.dimt = dt/12 self.seasv = ma.masked_equal(seasvals, fillVal) self.seast = ma.masked_equal(seastimes, fillVal) class cls_seasonal_accums(object): """ Class to compute the seasonal accumulations of a time series of values tvals=time values (year, month, day, hour, minute, second) format values=time-series of values dateref= reference date [YYYY][MM][DD][HH][MI][SS] format tunits= time units self.dimt = Number of values self.seasv = seasonal mean values self.seast = seasonal time mean values """ def __init__(self, tvals, values, dateref, tunits): import numpy.ma as ma fillVal=1.e20 if tvals is None: self.dimt = None self.seasv = None self.seast = None else: tseasons=times_4seasons(tvals, True) dt=len(values) seasvals=np.ones((dt/12,4), dtype=np.float64)*fillVal seastimes=np.zeros((dt/12,4), dtype=np.float64) dates = realdatetime_CFcompilant(tvals, dateref, tunits) for iseas in range(4): for it in range(dt): if tseasons[it,iseas]: tfirstseas=int(it/11) firstseas=True # print ' first time-step of season ', iseas, ' is: ',it,' position: ',tfirstseas break for itt in range(it, dt-1,12): seasvals[int(itt/12),iseas]=values[itt] + values[itt+1] + values[itt+2] seastimes[int(itt/12),iseas]=dates[itt] + (dates[itt+2] - dates[itt])/2. self.dimt = dt/12 self.seasv = ma.masked_equal(seasvals, fillVal) self.seast = ma.masked_equal(seastimes, fillVal) def seasonal_means(tvals, values): """ Function to compute the seasonal mean of a time series of values tvals=time values (year, month, day, hour, minute, second) format values=time-series of values """ fillVal=1.e20 tseasons=times_4seasons(tvals, True) dt=len(values) seasvals=np.ones((dt/4,4), dtype=np.float64) seasvals=seasvals*fillVal for iseas in range(4): for it in range(dt): if tseasons[it,iseas]: tfirstseas=int(it/11) firstseas=True # print ' first time-step of season ', iseas, ' is: ',it,' position: ',tfirstseas break for itt in range(it, dt-1,12): seasvals[int(itt/12),iseas]=(values[itt] + values[itt+1] + values[itt+2])/3. return seasvals def seasonal_accum(tvals, values): """ Function to compute the seasonal accumulation of a time series of values tvals=time values (year, month, day, hour, minute, second) format values=time-series of values """ fillVal=1.e20 tseasons=times_4seasons(tvals, True) dt=len(values) seasvals=np.ones((dt/4,4), dtype=np.float64) seasvals=seasvals*fillVal for iseas in range(4): for it in range(dt): if tseasons[it,iseas]: tfirstseas=int(it/11) firstseas=True # print ' first time-step of season ', iseas, ' is: ',it,' position: ',tfirstseas break for itt in range(it, dt-1,12): seasvals[int(itt/11),iseas]=values[itt] + values[itt+1] + values[itt+2] return seasvals def seasmean(timename, filename, varn): """ Function to compute the seasonal mean of a variable timename= name of the variable time in the file filename= netCDF file name varn= name of the variable """ fillValue=1.e20 ncf = NetCDFFile(filename,'a') ofile = 'seasmean_' + varn + '.nc' if not ncf.variables.has_key(varn): print errormsg print ' seasmean: File "' + filename + '" does not have variable ' + varn + ' !!!!' print errormsg ncf.close() quit(-1) varobj = ncf.variables[varn] varinf = variable_inf(varobj) timeobj = ncf.variables[timename] timeinf = cls_time_information(ncf, timename) timevals=timeobj[:] netcdftimes = netCDFdatetime_realdatetime(timeinf.units + ' since ' + timeinf.Urefdate, timeinf.calendar, timevals) timeseasons=times_4seasons(netcdftimes, True) timeseasvals=seasonal_means(netcdftimes, timevals) ncfnew = NetCDFFile(ofile,'w') ncfnew.createDimension('seastime', timeinf.dimt/4) ncfnew.createDimension('seas', 4) newvarobj = ncfnew.createVariable('seastime', 'f4', ('seastime', 'seas',), fill_value=fillValue) newvarobj[:]=timeseasvals attr = newvarobj.setncattr('calendar', timeinf.calendar) attr = newvarobj.setncattr('units', timeinf.units) newvarobj = ncfnew.createVariable('seasn', str, ('seas')) attr = newvarobj.setncattr('standard_name', 'seasn') attr = newvarobj.setncattr('long_name', 'name of the season') attr = newvarobj.setncattr('units', '3-months') newvarobj[0]='DJF' newvarobj[1]='MAM' newvarobj[2]='JJA' newvarobj[3]='SON' newdims=[] idim = 0 for ndim in varinf.dimns: if ndim == timename: newdims.append(unicode('seastime')) if not idim == 0: print errormsg print ' seasmean: File "' + filename + '" does not have time dimension "' + timename + '" as the first one. It has it as # ', idim, ' !!!!' print errormsg ncf.close() quit(-1) else: newdims.append(ndim) if not ncfnew.dimensions.has_key(ndim): ncfnew.sync() ncfnew.close() ncf.close() fdimadd(filename + ',' + ndim, ofile) ncf = NetCDFFile(filename,'r') ncfnew = NetCDFFile(ofile,'a') print ncfnew.dimensions namedims=tuple(newdims) print namedims varobj = ncf.variables[varn] varinf = variable_inf(varobj) newvarobj=ncfnew.createVariable(varn + '_seasmean', 'f4', namedims, fill_value=fillValue) attr = newvarobj.setncattr('standard_name', varn + '_seasmean') attr = newvarobj.setncattr('long_name', 'seasonal mean of ' + varn) attr = newvarobj.setncattr('units', varinf.units) newvar = np.ones(varinf.dims[0], dtype=varinf.dtype) if varinf.Ndims == 1: newvar = newvarobj[:] newvarobj[:] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 2: for i in range(varinf.dims[1]): newvar = newvarobj[:,i] newvarobj[:,i] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 3: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): newvar = newvarobj[:,i,j] newvarobj[:,i,j] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 4: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): newvar = newvarobj[:,i,j,k] newvarobj[:,i,j,k] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 5: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): newvar = newvarobj[:,i,j,k,l] newvarobj[:,i,j,k,l] = seasonal_means(netcdftimes, newvar) if varinf.Ndims == 6: for i in range(varinf.dims[1]): for j in range(varinf.dims[2]): for k in range(varinf.dims[3]): for l in range(varinf.dims[4]): for m in range(varinf.dims[5]): newvar = newvarobj[:,i,j,k,l,m] newvarobj[:,i,j,k,l,m] = seasonal_means(netcdftimes, newvar) else: print errormsg print ' seasmean: number of dimensions ', varinf.Ndims, ' not ready !!!!' print errormsg ncf.close() quit(-1) ncfnew.sync() ncfnew.close() print 'seasmean: Seasonal mean file "' + ofile + '" written!' def load_variable_lastdims(var, prevdims, Nlastdims): """ Function to load the last [Nlastdims] dimensions of a variable [var] at the other dimensions at [prevdims] >>> load_variable_lastdims(np.array(range(5*5*5)).reshape(5,5,5), [1,2], 1) [35 36 37 38 39] """ fname='load_variable_lastdims' dims=var.shape Ndims=len(dims) Nprevdims = len(prevdims) if not Nprevdims + Nlastdims == Ndims: print erromsg print ' ' + fname + ': number of dimensions previous (',Nprevdim,') and last (',Nlastdims, ') does not match with variable size: ',Ndims,' !!!!' print errormsg quit(-1) if Nlastdims > Ndims-1: print errormsg print ' ' + fname + ': number of last dimensions ', Nlastdims,' >= than the number of dimensions of the variable ', Ndims,' !!!!' print errormsg quit(-1) if Ndims == 1: loadvar = var[:] elif Ndims == 2: loadvar = var[prevdims[0], :] elif Ndims == 3: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], :] else: loadvar = var[prevdims[0], :, :] elif Ndims == 4: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], :, :] else: loadvar = var[prevdims[0], :, :, :] elif Ndims == 5: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :, :] elif Nlastdims == 3: loadvar = var[prevdims[0], prevdims[1], :, :, :] else: loadvar = var[prevdims[0], :, :, :, :] elif Ndims == 6: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], prevdims[4], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :, :] elif Nlastdims == 3: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :, :, :] elif Nlastdims == 4: loadvar = var[prevdims[0], prevdims[1], :, :, :, :] else: loadvar = var[prevdims[0], :, :, :, :] else: print errormsg print ' ' + fname + ' variable size ', Ndims, ' not ready!!!' print errormsg quit(-1) return loadvar def load_ncvariable_lastdims(ncf, varn, prevdims, Nlastdims): """ Function to load the last [Nlastdims] dimensions of a variable [varn] from a netCDF object at the other dimensions at [prevdims] ncf= netCDF object varn= variable name prevdims= values at the previous dimensions Nlastims= number of last dimensions """ fname='load_ncvariable_lastdims' var=ncf.variables[varn] dims=var.shape Ndims=len(dims) Nprevdims = len(prevdims) if not Nprevdims + Nlastdims == Ndims: print erromsg print ' ' + fname + ': number of dimensions previous (',Nprevdim,') and last (',Nlastdims, ') does not match with variable size: ',Ndims,' !!!!' print errormsg quit(-1) if Nlastdims > Ndims-1: print errormsg print ' ' + fname + ': number of last dimensions ', Nlastdims,' >= than the number of dimensions of the variable ', Ndims,' !!!!' print errormsg quit(-1) if Ndims == 1: loadvar = var[:] elif Ndims == 2: loadvar = var[prevdims[0], :] elif Ndims == 3: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], :] else: loadvar = var[prevdims[0], :, :] elif Ndims == 4: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], :, :] else: loadvar = var[prevdims[0], :, :, :] elif Ndims == 5: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :, :] elif Nlastdims == 3: loadvar = var[prevdims[0], prevdims[1], :, :, :] else: loadvar = var[prevdims[0], :, :, :, :] elif Ndims == 6: if Nlastdims == 1: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], prevdims[4], :] elif Nlastdims == 2: loadvar = var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :, :] elif Nlastdims == 3: loadvar = var[prevdims[0], prevdims[1], prevdims[2], :, :, :] elif Nlastdims == 4: loadvar = var[prevdims[0], prevdims[1], :, :, :, :] else: loadvar = var[prevdims[0], :, :, :, :] else: print errormsg print ' ' + fname + ' variable size ', Ndims, ' not ready!!!' print errormsg quit(-1) return loadvar def fill_ncvariable_lastdims(ncf, varn, prevdims, Nlastdims, fillvals): """ Function to fill the last [Nlastdims] dimensions of a variable [varn] from a netCDF object at the other dimensions at [prevdims] with as given set of [fillvals] values ncf= netCDF object varn= variable name prevdims= values at the previous dimensions Nlastims= number of last dimensions fillvals= values to use to fill """ fname='fill_ncvariable_lastdims' var=ncf.variables[varn] dims=var.shape Ndims=len(dims) Nprevdims = len(prevdims) if not Nprevdims + Nlastdims == Ndims: print erromsg print ' ' + fname + ': number of dimensions previous (',Nprevdim,') and last (',Nlastdims, ') does not match with variable size: ',Ndims,' !!!!' print errormsg quit(-1) if Nlastdims > Ndims-1: print errormsg print ' ' + fname + ': number of last dimensions ', Nlastdims,' >= than the number of dimensions of the variable ', Ndims,' !!!!' print errormsg quit(-1) if Ndims == 1: var[:] = fillvals elif Ndims == 2: var[prevdims[0], :] = fillvals elif Ndims == 3: if Nlastdims == 1: var[prevdims[0], prevdims[1], :] = fillvals else: var[prevdims[0], :, :] = fillvals elif Ndims == 4: if Nlastdims == 1: var[prevdims[0], prevdims[1], prevdims[2], :] = fillvals elif Nlastdims == 2: var[prevdims[0], prevdims[1], :, :] = fillvals else: var[prevdims[0], :, :, :] = fillvals elif Ndims == 5: if Nlastdims == 1: var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :] = fillvals elif Nlastdims == 2: var[prevdims[0], prevdims[1], prevdims[2], :, :] = fillvals elif Nlastdims == 3: var[prevdims[0], prevdims[1], :, :, :] = fillvals else: var[prevdims[0], :, :, :, :] = fillvals elif Ndims == 6: if Nlastdims == 1: var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], prevdims[4], :] = fillvals elif Nlastdims == 2: var[prevdims[0], prevdims[1], prevdims[2], prevdims[3], :, :] = fillvals elif Nlastdims == 3: var[prevdims[0], prevdims[1], prevdims[2], :, :, :] = fillvals elif Nlastdims == 4: var[prevdims[0], prevdims[1], :, :, :, :] = fillvals else: var[prevdims[0], :, :, :, :] = fillvals else: print errormsg print ' ' + fname + ' variable size ', Ndims, ' not ready!!!' print errormsg quit(-1) def maskvar(values, filename, varn): """ Function to mask a variable using a mask. It is assumed that mask[...,dimM,dimJ,dimK] and var[...,dimM,dimJ,dimK] share the last dimensions values= [maskfilename]:[maskvar]:[dimsmask]:[timename] [maskfilename]= file with the mask [maskvar]= name of the variable with the mask [dimsmask]= dimensions d1,d2,d3,... to use from variable [mask] (ahead of the sharing ones with [var]) [timename]= name of the variable time in [filename] filename= netCF file name varn= variable name """ fname='maskvar' fillValue=1.e20 ofile = 'mask_' + varn + '.nc' maskfilename = values.split(':')[0] maskvar = values.split(':')[1] maskdims = np.array(values.split(':')[2].split(','), dtype=int) timename = values.split(':')[3] ncf = NetCDFFile(filename,'r') if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ': File "' + filename + '" does not have variable "' + varn + '" !!!!' print errormsg ncf.close() quit(-1) varobj = ncf.variables[varn] varinf = variable_inf(varobj) timeobj = ncf.variables[timename] timeinf = cls_time_information(ncf, timename) ncfmask = NetCDFFile(maskfilename,'r') if not ncfmask.variables.has_key(maskvar): print errormsg print ' ' + fname + ': File "' + maskfilename + '" does not have variable mask "' + maskvar + '" !!!!' print errormsg ncf.close() ncfmask.close() quit(-1) varmaskobj = ncfmask.variables[maskvar] varmaskinf = variable_inf(varmaskobj) sdims=[] for ivdim in range(varinf.Ndims): vardim = varinf.Ndims - ivdim - 1 maskdim = varmaskinf.Ndims - ivdim - 1 if varinf.dims[vardim] == varmaskinf.dims[vardim]: sdims.append(varinf.dims[vardim]) else: break samedims = sdims[::-1] print ' ' + fname + ': variable: ', varinf.dims, ' and mask: ', varmaskinf.dims,' share: ', samedims # masking ## Nsamedims=len(samedims) if not Nsamedims + len(maskdims) == varmaskinf.Ndims: print errormsg print ' ' + fname + ': desired dimensions of mask ', maskdims, ' and coincident dims with variable ',Nsamedims, ' does not match mask dims ',varmaskinf.Ndims,' !!!!' print errormsg ncf.close() ncfmask.close() quit(-1) if varmaskinf.Ndims > Nsamedims: maskvals = load_ncvariable_lastdims(ncfmask, maskvar, maskdims, Nsamedims) else: maskvals = maskvarobj[:] ncfmask.close() # Creation of the new file ## ncfnew = NetCDFFile(ofile,'w') for idim in range(varinf.Ndims): newdimv = ncf.dimensions[varinf.dimns[idim]] if newdimv.isunlimited(): ncfnew.createDimension(varinf.dimns[idim], None) else: ncfnew.createDimension(varinf.dimns[idim], varinf.dims[idim]) newvar = ncfnew.createVariable(varn, varinf.dtype, varinf.dimns, fill_value=fillValue) for attrn in varinf.attributes: if not attrn == '_FillValue': attr = newvar.setncattr(attrn, varobj.getncattr(attrn)) attr = newvar.setncattr('mask', 'variable masked using ' + values.split(':')[2] + ' of variable ' + maskvar + ' from file ' + maskfilename) ncfnew.sync() ncfnew.close() for varns in ncf.variables: if not varns == varn: fvaradd(filename + ',' + varns, ofile) ncfnew = NetCDFFile(ofile,'a') Nptmask = 1 for isdim in range(Nsamedims): Nptmask = Nptmask * samedims[isdim] Nvarloop = 1 varsecdims = [] for insdim in range(varinf.Ndims - Nsamedims): Nvarloop = Nvarloop * varinf.dims[insdim] varsecdims.append(0) Nsecdims = len(varsecdims) for iloop in range(Nvarloop): varval = load_ncvariable_lastdims(ncf, varn, varsecdims, Nsamedims) newvarval = np.where(maskvals == 1, varval, fillValue) if np.sum(newvarval == fillValue) == Nptmask: print errormsg print ' ' + fname + ': all values got masked!!!' print erromsg quit(-1) fill_ncvariable_lastdims(ncfnew,varn,varsecdims,Nsamedims,newvarval) varsecdims[Nsecdims -1] = varsecdims[Nsecdims - 1] + 1 if varsecdims[Nsecdims - 1] > varinf.dims[Nsecdims - 1] - 1: varsecdims[Nsecdims - 1] = 0 for jdim in range(Nsecdims - 1): varsecdims[Nsecdims - 2 - jdim] = varsecdims[Nsecdims - 2 - jdim] + 1 if varsecdims[Nsecdims - 2 - jdim] <= varinf.dims[Nsecdims - 2 - jdim] - 1: break else: varsecdims[Nsecdims - 2 - jdim] = 0 print 'variable: ', varinf.dims, ' and mask: ', varmaskinf.dims,' share: ', samedims, Nvarloop ncfnew.sync() ncfnew.close() ncf.close() print 'masked file "' + ofile + '" generated !!!' def lonlat2D(ncfobj, lonn, latn): """ Function to create 2D matricies of longitudes and latitudes from the file ncfobj= netCDF object file lonn= longitude name latn= latitude name """ fname='lonlat2D' lonvar = ncfobj.variables[lonn] latvar = ncfobj.variables[latn] loninf = variable_inf(lonvar) latinf = variable_inf(latvar) if loninf.Ndims == 1: lonv = lonvar[:] latv = latvar[:] dx=loninf.dims[0] dy=latinf.dims[0] lonval = np.zeros((dy, dx), dtype=float) latval = np.zeros((dy, dx), dtype=float) for iy in range(dy): lonval[iy,:] = lonv for ix in range(dx): latval[:,ix] = latv elif loninf.Ndims == 2: lonval = lonvar[:] latval = latvar[:] elif loninf.Ndims == 3: lonval = lonvar[0,:,:] latval = latvar[0,:,:] else: print errormsg print ' ' + fname + ' dimension ', lonvar.Ndims, ' of the lon/lat matrices not ready !!!!' return lonval, latval def indices_mask(mask): """ Function to retrieve the indices from a 2D mask mask= 2D mask matrix mat=np.array(range(100), dtype=float).reshape((10,10)) >>> mask1= mat >= 45. >>> mask2= mat <= 55. >>> mask = mask1 * mask2 indices_mask(mask) [[4 5] [4 6] [4 7] [4 8] [4 9] [5 0] [5 1] [5 2] [5 3] [5 4] [5 5]] """ dy=mask.shape[0] dx=mask.shape[1] Nptmask = np.sum(mask) maskindices = np.zeros((Nptmask,2), dtype=int) ij=0 for j in range(dy): for i in range(dx): if mask[j,i] == True: maskindices[ij,0] = j maskindices[ij,1] = i ij = ij + 1 return maskindices def chgtimestep(values, origfile, varN): """ Function to change the values of a given time-step of a variable inside a netCDF for values from a nother file [values]=[origtime_step]:[newfile]:[newtime_step] [origtime_step]: desired original time step to change: 'first', 'last', 'middle', '[num]' [newfile]: new netCDF file name [newtime_step]: desired new time step to change: 'first', 'last', 'middle', '[num]' 'zeros', 'fillVal' 'first': first time-step 'last': last time-step 'middle': middle time-step '[num]': [num] time-step 'zeros': fill time-step with zeros 'fillVal': fill time-step with fill_Values [FillValue] [varN]= variable name [origfile]= original netCDF file name """ fname = 'chgtimestep' ermsg = fname + ': ERROR -- error -- ERROR -- error' vals = values.split(':') origtime_step = vals[0] newfile = vals[1] newtime_step = vals[2] if len(vals) == 4: FillValue = np.float(vals[3]) filexist(origfile, ermsg, 'original') filexist(newfile, ermsg, 'new') print 'origfile: ',origfile ncofile = NetCDFFile(origfile,'a') varinfile(ncofile, origfile, ermsg, 'change', varN) ncnfile = NetCDFFile(newfile,'r') varinfile(ncofile, newfile, ermsg, 'new', varN) ncnfile.close() if newtime_step == 'zeros': newvals = uploading_timestep(newfile, varN, 'first') newvals = 0. elif newtime_step == 'fillVal': newvals = uploading_timestep(newfile, varN, 'first') newvals = FillValue else: newvals = uploading_timestep(newfile, varN, newtime_step) varobj = ncofile.variables[varN] Ndims = len(varobj.shape) dimt = varobj.shape[0] if origtime_step == 'first': oval = 0 elif origtime_step == 'middle': oval = int(dimt/2) elif origtime_step == 'last': oval = dimt-1 else: oval = int(origtime_step) if Ndims == 1: varobj[oval] = newvals elif Ndims == 2: varobj[oval,:] = newvals elif Ndims == 3: varobj[oval,:,:] = newvals elif Ndims == 4: varobj[oval,:,:,:] = newvals elif Ndims == 5: varobj[oval,:,:,:,:] = newvals elif Ndims == 6: varobj[oval,:,:,:,:,:] = newvals else: print ermsg print ' variable size', Ndims, 'not ready !' print ermsg quit(-1) ncofile.sync() ncofile.close() def uploading_timestep(ncfile, varN, time_step): """ Function to upload a given time-step of a variable inside a netCDF [ncfile]= netCDF file name [varN]= variable name [time_step]= desired time step: 'first', 'last', 'middle', '[num]' 'first': first time-step 'last': last time-step 'middle': middle time-step '[num]': [num] time-step """ fname = 'uploading_timestep' ermsg = fname + ': ERROR -- error -- ERROR -- error' filexist(ncfile, ermsg, 'uploading') ncufile = NetCDFFile(ncfile,'r') varinfile(ncufile, ncfile, ermsg, 'uploading', varN) varobj = ncufile.variables[varN] Ndims = len(varobj.shape) dimt = varobj.shape[0] if time_step == 'first': upval = 0 elif time_step == 'middle': upval = int(dimt/2) elif time_step == 'last': upval = dimt-1 else: upval = int(time_step) if Ndims == 1: uploadvar = varobj[upval] elif Ndims == 2: uploadvar = varobj[upval,:] elif Ndims == 3: uploadvar = varobj[upval,:,:] elif Ndims == 4: uploadvar = varobj[upval,:,:,:] elif Ndims == 5: uploadvar = varobj[upval,:,:,:,:] elif Ndims == 6: uploadvar = varobj[upval,:,:,:,:,:] else: print ermsg print ' variable size', Ndims, 'not ready !' print ermsg quit(-1) return uploadvar def timeshiftvar(values, fileN, varN, FillVal=1.e20): """ Function to temporaly shift a number of time-steps a given variable inside a netCDF file [values]=[nsteps]:[[FillValue]] [nsteps]: desired number of time step to move [FillValue]: value to fill the empty time-steps [fileobj]= original netCDF object [varN]= variable name """ from sys import stdout fname = 'timesfhitvar' ermsg = fname + ': ERROR -- error -- ERROR -- error' nsteps = int(values.split(':')[0]) if len(values.split(':')) == 2: FillVal = np.float(values.split(':')[1]) fileobj = NetCDFFile(fileN,'a') varobj = fileobj.variables[varN] vardims = varobj.shape vartype = varobj.dtype Ndims = len(vardims) dimt = vardims[0] if nsteps > 0: inimov = nsteps endmov = dimt iniplace = 0 endplace = dimt - nsteps else: inimov = 0 endmov = dimt + nsteps iniplace = -nsteps endplace = dimt print ' ' + fname + ': moving', nsteps, '___ __ _' print ' from (', inimov, ',', endmov, ') to (',iniplace,',',endplace,')' if Ndims == 1: vals = np.ones(vardims, dtype=vartype)*FillVal vals[iniplace:endplace] = varobj[inimov:endmov] varobj[:] = vals elif Ndims == 2: vals = np.ones(vardims, dtype=vartype)*FillVal vals[iniplace:endplace,:] = varobj[inimov:endmov,:] varobj[:] = vals elif Ndims == 3: vals = np.ones((vardims[0], vardims[1]), dtype=vartype)*FillVal for i in range(vardims[2]): if (i%10 == 0): print '\r shifted ' + '{0:5.2g}'.format(i*100./vardims[2]) + ' %', stdout.flush() vals[iniplace:endplace,:] = varobj[inimov:endmov,:,i] varobj[:,:,i] = vals elif Ndims == 4: vals = np.ones((vardims[0], vardims[1]), dtype=vartype)*FillVal for i in range(vardims[2]): for j in range(vardims[3]): nvals=i*vardims[3] + j if (nvals%100 == 0): print '\r shifted ' + '{0:5.2g}'.format(nvals*100./(vardims[2]*vardims[3])) + ' %', stdout.flush() vals[iniplace:endplace,:] = varobj[inimov:endmov,:,i,j] varobj[:,:,i,j] = vals elif Ndims == 5: vals = np.ones((vardims[0], vardims[1]), dtype=vartype)*FillVal for i in range(vardims[2]): for j in range(vardims[3]): for k in range(vardims[4]): nvals=i*vardims[3]*vardims[4] + j*vardims[4] + k if (nvals%1000 == 0): print '\r shifted ' + '{0:5.2g}'.format(nvals*100./(vardims[2]*vardims[3]*vardims[4])) + ' %', stdout.flush() vals[iniplace:endplace,:] = varobj[inimov:endmov,:,i,j,k] varobj[:,:,i,j,k] = vals elif Ndims == 6: vals = np.ones((vardims[0], vardims[1]), dtype=vartype)*FillVal for i in range(vardims[2]): for j in range(vardims[3]): for k in range(vardims[4]): for l in range(vardims[5]): nvals=i*vardims[3]*vardims[4]*vardims[5] + j*vardims[4]*vardims[5] + k*vardims[5] + l if (nvals%10000 == 0): print '\r shifted ' + \ '{0:5.2g}'.format(nvals*100./(vardims[2]*vardims[3]*vardims[4]*vardims[5])) + ' %', stdout.flush() vals[iniplace:endplace,:] = varobj[inimov:endmov,:,i,j,k,l] varobj[:,:,i,j,k,l] = vals else: print ermsg print ' variable size', Ndims, 'not ready !' print ermsg quit(-1) print stdout.write("\n") fileobj.sync() fileobj.close() def typemod(value, typeval): """ Function to give back a value in a given dtype >>> print(typemod(8.2223, 'np.float64')) >>> print(typemod(8.2223, 'tuple')) """ fname='typemod' if typeval == 'int': return int(value) elif typeval == 'long': return long(value) elif typeval == 'float': return float(value) elif typeval == 'complex': return complex(value) elif typeval == 'str': return str(value) elif typeval == 'bool': return bool(value) elif typeval == 'list': newval = [] newval.append(value) return newval elif typeval == 'dic': newval = {} newval[value] = value return newval elif typeval == 'tuple': newv = [] newv.append(value) newval = tuple(newv) return newval elif typeval == 'np.int8': return np.int8(value) elif typeval == 'np.int16': return np.int16(value) elif typeval == 'np.int32': return np.int32(value) elif typeval == 'np.int64': return np.int64(value) elif typeval == 'np.uint8': return np.uint8(value) elif typeval == 'np.uint16': return np.uint16(value) elif typeval == 'np.np.uint32': return np.uint32(value) elif typeval == 'np.uint64': return np.uint64(value) elif typeval == 'np.float': return np.float(value) elif typeval == 'np.float16': return np.float16(value) elif typeval == 'np.float32': return np.float32(value) elif typeval == 'float32': return np.float32(value) elif typeval == 'np.float64': return np.float64(value) elif typeval == 'float64': return np.float64(value) elif typeval == 'np.complex': return np.complex(value) elif typeval == 'np.complex64': return np.complex64(value) elif typeval == 'np.complex128': return np.complex128(value) else: print errormsg print fname + ': data type "' + typeval + '" is not ready !' print errormsg quit(-1) def addvals(values, filename, varN): """ Function to add values to a given variable at a given dimension values = [dimension]:[position]:[Nvals]:[addval] [dimension]: dimension at which the variable wants to be increased [position]: position within the dimension at which the variable wants to be increased 'first': first position of the dimension 'middle': middle position of the dimension 'last': last position of the dimension 'num': before given value [Nvals]: number of values [addval]: value to give to the added values 'val': a given value 'missing': as a missing value filename = name of the netCDF file varN = name of the variable """ import subprocess as sub fname = 'addvals' if values == 'h': print fname + '_____________________________________________________________' print addvals.__doc__ quit() newf = 'addvals.nc' dimension=int(values.split(':')[0]) position=values.split(':')[1] Nvals=int(values.split(':')[2]) addval=values.split(':')[3] filexist(filename, errormsg, 'add values') ncfa = NetCDFFile(filename,'r') varinfile(ncfa, filename, errormsg, 'adding', varN) varobj=ncfa.variables[varN] varinf=variable_inf(varobj) newdims=list(varinf.dims) newdims[dimension] = newdims[dimension] + Nvals newdims=tuple(newdims) dmodname = varinf.dimns[dimension] print ' ' + fname + ' increassed dimension "' + dmodname + '"' if position == 'first': pos=0 elif position == 'middle': pos=(varinf.dims[dimension])/2 elif position == 'last': pos=varinf.dims[dimension] else: pos=int(position) if pos > varinf.dims[dimension]: print errormsg print fname + ': position given ', pos,' is too big for the size of the dimension ',varinf.dims[dimension], '!!!!' print errormsg quit(-1) ## print 'starting position: ', pos if addval == 'missing': addvaln = 1.e20 else: addvaln = addval addvalv=typemod(addvaln, str(varinf.dtype)) newnc = NetCDFFile(newf,'w') # Dimensions ## for dim in ncfa.dimensions.values(): dname = dim._name dsize = dim.__len__() if dim.isunlimited(): dsize=None else: if dname == varinf.dimns[dimension]: dsize=newdims[dimension] newnc.createDimension(dname, size=dsize) # Variables ## for var in ncfa.variables.values(): vname = var._name vkind = var.dtype vdims = var.dimensions varattrs = var.ncattrs() if searchInlist(varattrs, '_FillValue'): fillval = var.getncattr('_FillValue') newvarobj = newnc.createVariable(vname, vkind, vdims, fill_value=fillval) elif vname == varN and addval == 'missing': newvarobj = newnc.createVariable(vname, vkind, vdims, fill_value=typemod(addvaln, str(varinf.dtype))) else: newvarobj = newnc.createVariable(vname, vkind, vdims) varobj=ncfa.variables[vname] vals = varobj[:] if vname == varN: for ival in range(Nvals): ## print ' ' + fname + ': adding',ival+1,' values' newvar = np.insert(vals, pos, addvalv, axis=dimension) vals=newvar.copy() newvarobj[:] = newvar else: if vname == dmodname: # Increassing dimension variable values ## dvals=ncfa.variables[dmodname][:] if position == 'first': valB = -(dvals[1] - dvals[0]) valA=dvals[pos] + valB elif position == 'last': valB=-(dvals[1] - dvals[0]) valA=dvals[pos-1] - valB*Nvals else: valB=-(dvals[pos] - dvals[pos-1])/(Nvals+1) valA=dvals[pos-1] - valB*Nvals for ival in range(Nvals): addvalvar=valA + valB*ival ## print ' ' + fname + ' adding to dimension variable "', dmodname ,'" value ',addvalvar newvar = np.insert(vals, pos, addvalvar, axis=0) vals=newvar.copy() newvarobj[:] = newvar else: vardims=list(var.dimensions) if searchInlist(vardims,varinf.dimns[dimension]): ## print ' ' + fname + ' variable ' + vname + ' has also the increased dimension!' vardim=vardims.index(varinf.dimns[dimension]) for ival in range(Nvals): addvalvar = addval newvar = np.insert(vals, pos, addvalvar, axis=vardim) vals=newvar.copy() newvarobj[:] = newvar else: newvarobj[:] = vals for vattr in varobj.ncattrs(): if not vattr == '_FillValue': vattrval = var.getncattr(vattr) newvarobj = set_attribute(newvarobj, vattr, vattrval) # Global attributes ## for gattr in ncfa.ncattrs(): gattrval = ncfa.getncattr(gattr) newnc = set_attribute(newnc, gattr, gattrval) newnc.sync() newnc.close() ncfa.close() try: ins = 'cp ' + newf + ' ' + filename syscode = sub.call(ins, shell=True) if not syscode == 0: print errmsg print " Copy to the new file failed!" print ins print syscode quit() else: sub.call('rm ' + newf, shell=True) except OSError, e: print errmsg print " Copy failed!" print e quit(-1) def TimeInf(filename, tname): """ Function to print all the information from the variable time filename= netCDF file name tname= name of the variable time """ ncfobj = NetCDFFile(filename,'r') timeinf = cls_time_information(ncfobj, tname) fmtprinting_class(timeinf) ncfobj.close() def sorttimesmat(filename, tname): """ Function to sort the time values of a given file tname, name of the variable time filename= netCDF file name """ fname = 'sorttimesmat' filexist(filename, errormsg, 'to sort time') ncft = NetCDFFile(filename,'a') varinfile(ncft, filename, errormsg, 'time', tname) timeobj = ncft.variables[tname] timeinf = cls_time_information(ncft, tname) tvals = timeobj[:] Ltvals = list(tvals) sortedtvals = sorted(Ltvals) tinds = np.zeros((timeinf.dimt), dtype=int) for it in range(timeinf.dimt): tinds[it] = Ltvals.index(sortedtvals[it]) for var in ncft.variables: varobj = ncft.variables[var] if searchInlist(varobj.dimensions, tname): print fname + ': time-sorting variable: "' + var + '" ' varvals = varobj[:] newvarvals = varvals.copy() for it in range(timeinf.dimt): if var == 'time': print it, tinds[it], tvals[tinds[it]] newvarvals[it,] = varvals[tinds[it],] varobj[:] = newvarvals ncft.sync() ncft.close() def diffdate_units(diffdate, units): """ Function to transform a difference of dates to a certain units diffdate = difference of dates (as timedelta) units = units to transform: 'weeks', 'days', 'hours', 'minutes', 'seconds', 'miliseconds', 'nanoseconds' >>> diffdate_units(dt.datetime(1976, 2, 17, 8, 32, 5) - dt.datetime(1949, 12, 1, 0, 0, 0), 'seconds') 827224325.0 """ fname = 'diffdate_units' if units == 'weeks': diffunits = diffdate.days/7. + diffdate.seconds/(3600.*24.*7.) elif units == 'days': diffunits = diffdate.days + diffdate.seconds/(3600.*24.) elif units == 'hours': diffunits = diffdate.days*24. + diffdate.seconds/(3600.) elif units == 'minutes': diffunits = diffdate.days*24.*60. + diffdate.seconds/(60.) elif units == 'seconds': diffunits = diffdate.days*24.*3600. + diffdate.seconds elif units == 'miliseconds': diffunits = diffdate.days*24.*3600.*1000. + diffdate.seconds*1000. elif units == 'nanoseconds': diffunits = diffdate.days*24.*3600.*1000000. + diffdate.seconds*1000000. else: print errormsg print fname + ': time units "' + units + '" not ready!!!' print errormsg quit(-1) return diffunits def days_month(year,month): """ Function to give the number of days of a month of a given year >>> days_month(1976,2) 29 """ import datetime as dt date1=dt.date(year,month,1) date2=dt.date(year,month+1,1) diffdate = date2-date1 return diffdate.days def mid_period(yr1,mon1,day1,hour1,min1,sec1,yr2,mon2,day2,hour2,min2,sec2): """ Function to give the mid poiint of a period >>> mid_period(1976,2,1,0,0,0,1976,3,1,0,0,0) [1976 2 15 12 0 0] """ import datetime as dt date1=dt.datetime(yr1,mon1,day1,hour1,min1,sec1) date2=dt.datetime(yr2,mon2,day2,hour2,min2,sec2) diffdate = date2-date1 diffseconds = diffdate.days*24*3600 + diffdate.seconds diff2 = dt.timedelta(seconds=diffseconds/2) datenew = date1 + diff2 datenewvals = np.array([datenew.year, datenew.month, datenew.day, datenew.hour, \ datenew.minute, datenew.second]) return datenewvals def days_year(year): """ Function to give the number of days of a year >>> days_year(1976) 366 """ import datetime as dt date1=dt.date(year,1,1) date2=dt.date(year+1,1,1) diffdate = date2-date1 return diffdate.days def days_period(yeari,yearf): """ Function to give the number of days for a period >>> days_period(1976,1980) 1827 """ ndays=0 for iyr in range(yearf-yeari+1): ndays=ndays+days_year(yeari+iyr) return ndays def check_times_file(values, filename, timen): """ Function to check time-steps of a given file values=[FirstDate]:[LastDate][freq]:[auxmonth] [FirstDate] = first date within file (in [YYYY][MM][DD][HH][MI][SS] format) [LastDate] = last date within file (in [YYYY][MM][DD][HH][MI][SS] format) [freq] = frequency of the data within the file (in time units) or 'month' (monthly values) [auxmonth] = in case of 'month' midmon: dates are given according to the mid time value of the month real: dates are given increassing a month from the previous date filename: name of the file timen: name of the variable time """ import datetime as dt fname = 'check_times_file' FirstDate = values.split(':')[0] LastDate = values.split(':')[1] freq = values.split(':')[2] ncft = NetCDFFile(filename, 'r') varinfile(ncft, filename, errormsg, '', timen) timeobj=ncft.variables[timen] timeinf = cls_time_information(ncft, timen) timevals = timeobj[:] ncft.close() yrf=int(FirstDate[0:4]) monf=int(FirstDate[4:6]) dayf=int(FirstDate[6:8]) horf=int(FirstDate[8:10]) minf=int(FirstDate[10:12]) secf=int(FirstDate[12:14]) firstdate = dt.datetime(yrf, monf, dayf, horf, minf, secf) yrl=int(LastDate[0:4]) monl=int(LastDate[4:6]) dayl=int(LastDate[6:8]) horl=int(LastDate[8:10]) minl=int(LastDate[10:12]) secl=int(LastDate[12:14]) lastdate = dt.datetime(yrl, monl, dayl, horl, minl, secl) firstT = firstdate - timeinf.refdate t0 = diffdate_units(firstT, timeinf.units) period = lastdate - firstdate per = diffdate_units(period, timeinf.units) if freq != 'month': theordimt = np.int(per / np.float64(freq) + 1) else: theordimt = np.int((yrl - yrf + 1) * 12) theordates = np.zeros((theordimt), dtype=np.float64) theordates[0] = t0 if freq != 'month': theordates = t0 + np.arange(1,theordimt) * np.float64(freq) else: auxmonth = values.split(':')[3] if auxmonth == 'midmon': t01 = np.array([yrf, monf, 1, 0, 0, 0]) else: t01 = np.array([yrf, monf, dayf, horf, minf, secf]) for it in range(1,theordimt): t02 = t01.copy() t02[1] = t02[1] + 1 if t02[1] > 12: t02[1] = 1 t02[0] = t02[0] + 1 date01 = dt.datetime(t01[0], t01[1], t01[2], t01[3], t01[4], t01[5]) if auxmonth == 'midmon': date02 = dt.datetime(t02[0], t02[1], t02[2], t02[3], t02[4], t02[5]) diff012 = date02 - date01 t01 = diffdate_units(date01 - timeinf.refdate, timeinf.units) theordates[it] = t01 + diffdate_units(diff012, timeinf.units) / 2. elif auxmonth == 'real': date02 = dt.datetime(t02[0], t02[1], t01[2], t01[3], t01[4], t01[5]) diff012 = date02 - date01 theordates[it] = theordates[it-1] + diffdate_units(diff012, timeinf.units) else: print errormsg print fname + ': frequency "month" required third argument "' + auxmonth + '" not ready!!!!' print errormsg quit(-1) t01 = t02.copy() equaldates = sorted(list(set(theordates) & set(timevals))) missdates = sorted(list(set(theordates) - set(timevals))) Nequal = len(equaldates) Ndiff = len(missdates) if theordimt != timeinf.dimt: print warnmsg print fname + ': there are not the correspondant times (', theordimt ,') in file "' + filename + '" with ', timeinf.dimt,' time-steps !!!!!!!' print ' ' + fname + ': Number equal times-steps: ', Nequal, 'Number different time-steps: ', Ndiff if Nequal >= Ndiff or Nequal == 0: matdates = netCDFdatetime_realdatetime(timeinf.unitsval, timeinf.calendar, missdates) print ' file does not have dates: ' for it in range(Ndiff): print matdates[it,:] else: matdates = netCDFdatetime_realdatetime(timeinf.unitsval, timeinf.calendar, equaldates) print ' file only has: ' for it in range(Nequal): print matdates[it,:] def writing_str_nc(varo, values, Lchar): """ Function to write string values in a netCDF variable as a chain of 1char values varo= netCDF variable object values = list of values to introduce Lchar = length of the string in the netCDF file """ Nvals = len(values) for iv in range(Nvals): if values[iv] is None: stringv = 'None' else: stringv = values[iv] charvals = np.chararray(Lchar) Lstr = len(stringv) charvals[Lstr:Lchar] = '' for ich in range(Lstr): charvals[ich] = stringv[ich:ich+1] varo[iv,:] = charvals return def writing_1str_nc(varo, values, Lchar): """ Function to write 1 string value in a netCDF variable as a chain of 1char values varo= netCDF variable object values = list of values to introduce Lchar = length of the string in the netCDF file """ stringv=values charvals = np.chararray(Lchar) Lstr = len(stringv) charvals[Lstr:Lchar] = '' for ich in range(Lstr): charvals[ich] = stringv[ich:ich+1] varo[:] = charvals return def get_1str_nc(varo, Lchar): """ Function to get 1 string value in a netCDF variable as a chain of 1char values varo= netCDF variable object Lchar = length of the string in the netCDF file """ string = '' for ich in range(Lchar): charval = str(stringv[ich:ich+1]).replace('[','').replace(']','') if charval == '': break else: if ich == 0: string = charval else: string = string + charval return string def get_str_nc(varo, Lchar): """ Function to get string values in a netCDF variable as a chains of 1char values varo= netCDF variable object Lchar = length of the string in the netCDF file """ Nvalues = varo.shape[0] strings = [] for ival in range(Nvalues): stringv = varo[ival] string = str('') for ich in range(Lchar): charval = str(stringv[ich:ich+1]).replace('[','').replace(']','') if charval == '--': break else: if ich == 0: string = charval else: string = string + charval strings.append(string) return strings class ysubsetstats(object): """ Class to compute multi-year statistics of a given subset of values values= values to use dates=dates of the values in matrix format ([year], [month], [day], [hour], [minute], [second]) sec=section to use for the period='year', 'month', 'day', 'hour', 'minute', 'second' vali=initial value of the period valf=final value of the period Nq= number of quantiles (20 for 5% bins, it is going to produce 21 values) missval= missing value self.Nvalues = Number of non masked values self.min = minimum non masked value self.max = maximum non masked value self.mean = mean non masked value self.mean2 = quandratic mean non masked value self.stdev = standard deviation non masked value self.quantiles = quantiles non masked value """ def __init__(self, values, dates, sec, vali, valf, Nq, missVal): import numpy.ma as ma fname = 'ysubsetstats' if values is None: self.Nvalues = None self.min = None self.max = None self.mean = None self.mean2 = None self.stdev = None self.quantiles = None else: Nvalues = len(values) Ndates = len(dates) if Nvalues != Ndates: print errormsg print fname + ': number of values ', Nvalues,' does not coincide with the number of dates ',Ndates, '!!!!!!' print errormsg quit(-1) initialsub = np.zeros(Nvalues, dtype=bool) endsub = initialsub.copy() if missVal > 0.: valuesmask = ma.masked_greater_equal(values, missVal*0.9) else: valuesmask = ma.masked_less_equal(values, missVal*0.9) if sec == 'year': isec = 0 elif sec == 'month': isec = 1 elif sec == 'day': isec = 2 elif sec == 'hour': isec = 3 elif sec == 'minute': isec = 4 elif sec == 'second': isec = 5 else: print errormsg print fname + ': sction "' + sec + '" not ready!!!!' print errormsg quit(-1) if vali < valf: timesmaskperi = ma.masked_less(dates[:,isec], vali) timesmaskperf = ma.masked_greater(dates[:,isec], valf) else: timesmaskperi = ma.masked_less(dates[:,isec], valf) timesmaskperf = ma.masked_greater(dates[:,isec], vali) finalmask = valuesmask.mask+timesmaskperi.mask+timesmaskperf.mask finalvalues = ma.array(values, mask=finalmask) finalvalues2 = finalvalues*finalvalues self.Nvalues = finalvalues.count() self.min = finalvalues.min() self.max = finalvalues.max() self.mean = finalvalues.mean() self.mean2 = finalvalues2.mean() self.stdev = finalvalues.std() self.quantiles = mask_quantiles(finalvalues, Nq) def remapnn(values, filename, varname): """ Function to remap to the nearest neightbor a variable using projection from another file values=[newprojectionfile]|[newlonname,newlatname]|[newlondim,newlatdim]|[oldlonname,oldlatname]| [oldlondim,oldlatdim] [newprojectionfile]: name of the file with the new projection [newlonname,newlatname]: name of the longitude and the latitude variables in the new file [newlondim,newlatdim]: name of the dimensions for the longitude and the latitude in the new file [oldlonname,oldlatname]: name of the longitude and the latitude variables in the old file [oldlondim,oldlatdim]: name of the dimensions for the longitude and the latitude in the old file filename= netCDF file name varn= variable name ('all', for all variables) """ fname = 'remapnn' if values == 'h': print fname + '_____________________________________________________________' print remapnn.__doc__ quit() expectargs = '[newprojectionfile]|[newlonname,newlatname]|[newlondim,' + \ 'newlatdim]|[oldlonname,oldlatname]|[oldlondim,oldlatdim]' check_arguments(fname,values,expectargs,'|') newprojectionfile = values.split('|')[0] newlonlatname = values.split('|')[1] newlonlatdim = values.split('|')[2].split(',') oldlonlatname = values.split('|')[3] oldlonlatdim = values.split('|')[4].split(',') if varname != 'all': ofile = 'remapnn_' + varname + '.nc' else: ofile = 'remapnn_allvars.nc' filexist(filename, errormsg, 'old file') filexist(newprojectionfile, errormsg, 'new file') oldlon=oldlonlatname.split(',')[0] oldlat=oldlonlatname.split(',')[1] oldncfile = NetCDFFile(filename,'r') varinfile(oldncfile, filename, errormsg, 'old', oldlon) varinfile(oldncfile, filename, errormsg, 'old', oldlat) newlon=newlonlatname.split(',')[0] newlat=newlonlatname.split(',')[1] newncfile = NetCDFFile(newprojectionfile,'r') varinfile(newncfile, newprojectionfile, errormsg, 'new', newlon) varinfile(newncfile, newprojectionfile, errormsg, 'new', newlat) oldlon2d, oldlat2d = lonlat2D(oldncfile.variables[oldlon], \ oldncfile.variables[oldlat]) newlon2d, newlat2d = lonlat2D(newncfile.variables[newlon], \ newncfile.variables[newlat]) newdimx = newlon2d.shape[1] newdimy = newlon2d.shape[0] olddimx = oldlon2d.shape[1] olddimy = oldlon2d.shape[0] # We are going to do the search in two phases # 1.- Searching on grid points every 10% of the size of the origina domain # 2.- Within the correspondant box of 20%x20% percent # 1.- first pairs old-->new ## dx10 = int(olddimx/10) dy10 = int(olddimy/10) oldlon2d10 = oldlon2d[slice(dy10,olddimy,dy10), slice(dx10,olddimx,dx10)] oldlat2d10 = oldlat2d[slice(dy10,olddimy,dy10), slice(dx10,olddimx,dx10)] firstmappairs = np.ones((2, newdimy, newdimx), dtype=int) print ' ' + fname + ' first searching nearest neighbor by',dx10,',',dy10,'...' for jnew in range(newdimy): for inew in range(newdimx): dist=np.sqrt((newlon2d[jnew, inew] - oldlon2d10)**2. + \ (newlat2d[jnew, inew] - oldlat2d10)**2.) mindist = np.min(dist) minydist10, minxdist10 = index_mat(dist,mindist) firstmappairs[:,jnew,inew] = [int(dy10*(1+minydist10)), \ int(dx10*(1+minxdist10))] # 2.- Looking around the closest coarse point ## mappairs = np.ones((2, newdimy, newdimx), dtype=int) dx2 = dx10/2 dy2 = dy10/2 print ' ' + fname + ' searching nearest neighbor...' for jnew in range(newdimy): for inew in range(newdimx): i10 = firstmappairs[1,jnew,inew] j10 = firstmappairs[0,jnew,inew] imin = np.max([i10-dx2, 0]) imax = np.min([i10+dx2, olddimx+1]) jmin = np.max([j10-dy2, 0]) jmax = np.min([j10+dy2, olddimy+1]) oldlon2d2 = oldlon2d[slice(jmin,jmax), slice(imin,imax)] oldlat2d2 = oldlat2d[slice(jmin,jmax), slice(imin,imax)] dist=np.sqrt((newlon2d[jnew, inew] - oldlon2d2)**2. + \ (newlat2d[jnew, inew] - oldlat2d2)**2.) # print 'oldlon2d2________________',i10,j10 # print oldlon2d2 # print 'oldlat2d2________________' # print oldlat2d2 # print 'dist________________' # print dist mindist = np.min(dist) minydist, minxdist = index_mat(dist,mindist) mappairs[:,jnew,inew] = [int(j10+minydist), int(i10+minxdist)] # print jnew,inew,'|',firstmappairs[:,jnew,inew],':',minxdist,minydist, \ # 'final:',mappairs[:,jnew,inew],'lon,lat old:',oldlon2d[minydist,minxdist],\ # oldlat2d[minydist,minxdist], 'new:', newlon2d[jnew, inew], \ # newlat2d[jnew, inew],'mindist:',mindist # quit() # Creation of the new file ## ncfnew = NetCDFFile(ofile,'w') for dn in oldncfile.dimensions: if dn == oldlon: ncfnew.createDimension(newlonlatdim[0], newdimx) elif dn == oldlat: ncfnew.createDimension(newlonlatdim[1], newdimy) else: if oldncfile.dimensions[dn].isunlimited(): ncfnew.createDimension(dn, None) else: ncfnew.createDimension(dn, len(oldncfile.dimensions[dn])) ncfnew.sync() # looping along all the variables ## if varname == 'all': oldvarnames = oldncfile.variables else: oldvarnames = [varname] for varn in oldvarnames: print ' adding:',varn,'...' ovar = oldncfile.variables[varn] vardims = ovar.dimensions vartype = ovar.dtype varattrs = ovar.ncattrs() # Looking if variable has lon and/or lat: varlonlat = False if searchInlist(vardims, oldlonlatdim[0]): varlonlat = True if searchInlist(vardims, oldlonlatdim[1]): varlonlat = True if varn != oldlon and varn != oldlat and varlonlat: newvarshape = [] newvardimn = [] newslice = [] oldslice = [] for dn in vardims: if dn == oldlat: newvarshape.append(newdimy) newvardimn.append(newlonlatdim[1]) elif dn == oldlon: newvarshape.append(newdimx) newvardimn.append(newlonlatdim[0]) else: newvarshape.append(len(oldncfile.dimensions[dn])) newvardimn.append(dn) newslice.append(slice(0,len(oldncfile.dimensions[dn]))) oldslice.append(slice(0,len(oldncfile.dimensions[dn]))) newvarval = np.zeros(tuple(newvarshape), dtype=vartype) for jnew in range(newdimy): for inew in range(newdimx): oldslice0 = oldslice[:] newslice0 = newslice[:] if searchInlist(vardims,oldlat): oldslice0.append(mappairs[0,jnew,inew]) newslice0.append(jnew) if searchInlist(vardims,oldlon): oldslice0.append(mappairs[1,jnew,inew]) newslice0.append(inew) # if ((jnew*newdimx + inew)%100 == 0): print ' ' + fname + ': ', \ # '{0:5.2f}'.format(float((jnew*newdimx + inew)*100./(newdimx*newdimy))),\ # '% done' # print 'slices new:',newslice0,'old:',oldslice0 newvarval[tuple(newslice0)] = ovar[tuple(oldslice0)] elif varn == oldlon: ovar = newncfile.variables[newlon] vardims = ovar.dimensions vartype = ovar.dtype varattrs = ovar.ncattrs() newvardimn = (newlonlatdim[1], newlonlatdim[0]) newvarval = newlon2d elif varn == oldlat: ovar = newncfile.variables[newlat] vardims = ovar.dimensions vartype = ovar.dtype varattrs = ovar.ncattrs() newvardimn = (newlonlatdim[1], newlonlatdim[0]) newvarval = newlat2d else: newvardimn = vardims newvarval = ovar[:] if searchInlist(varattrs, '_FillValue'): fvalv = ovar.getncattr('_FillValue') newvar = ncfnew.createVariable(varn, vartype, tuple(newvardimn), \ fill_value=fvalv) else: newvar = ncfnew.createVariable(varn, vartype, tuple(newvardimn)) for ncattr in varattrs: if ncattr != '_FillValue': attrv = ovar.getncattr(ncattr) newattr = set_attribute(newvar,ncattr,attrv) attr = newvar.setncattr('mask', 'variable remapped at nearest neighbors ' + \ 'using ' + values.split(':')[0] + ' using as longitude and latitude values ') newvar[:] = newvarval ncfnew.sync() ncfnew.close() newncfile.close() oldncfile.close() # Adding attributes fgaddattr(filename, ofile) print 'remapnn: new file "' + ofile + '" with remapped variable written !!' #remapnn('met_em.d01.1979-01-01_00:00:00.nc|XLONG_M,XLAT_M|east_weast,south_north|longitude,latitude|longitude,latitude', \ # 'Albedo.nc', 'all') #quit() def remapnn_old(values, filename, varn, ddegL, ddegl): """ Function to remap to the nearest neightbor a variable using projection from another file values=[newprojectionfile]:[newlonname,newlatname]:[oldlonname,oldlatname] [newprojectionfile]: name of the file with the new projection [newlonname,newlatname]: name of the longitude and the latitude variables in the new file [oldlonname,oldlatname]: name of the longitude and the latitude variables in the old file filename= netCDF file name varn= variable name ddegL= Longitude range around the old grid point to speed up the search ddegl= latitude range around the old grid point to speed up the search """ fname = 'remapnn' ofile = 'remapnn_' + varn + '.nc' fillValue = 1.e20 newprojectionfile = values.split(':')[0] newlonlatname = values.split(':')[1] oldlonlatname = values.split(':')[2] filexist(filename, errormsg, 'old file') filexist(newprojectionfile, errormsg, 'new file') oldlon=oldlonlatname.split(',')[0] oldlat=oldlonlatname.split(',')[1] oldncfile = NetCDFFile(filename,'r') varinfile(oldncfile, filename, errormsg, 'old', oldlon) varinfile(oldncfile, filename, errormsg, 'old', oldlat) newlon=newlonlatname.split(',')[0] newlat=newlonlatname.split(',')[1] newncfile = NetCDFFile(newprojectionfile,'r') varinfile(newncfile, newprojectionfile, errormsg, 'old', newlon) varinfile(newncfile, newprojectionfile, errormsg, 'old', newlat) oldlon2d, oldlat2d = lonlat2D(oldncfile, oldlon, oldlat) newlon2d, newlat2d = lonlat2D(newncfile, newlon, newlat) newdimx = newlon2d.shape[1] newdimy = newlon2d.shape[0] olddimx = oldlon2d.shape[1] olddimy = oldlon2d.shape[0] # pairs old-->new ## mappairs = np.ones((2, newdimy, newdimx), dtype=int) mappairs = mappairs*fillValue print ' ' + fname + ' searching nearest neighbor...' for jnew in range(newdimy): for inew in range(newdimx): if ((jnew*newdimx + inew)%100 == 0): print ' ' + fname + ' found: ', '{0:5.2f}'.format(float(jnew*newdimx + inew)*100/(newdimx*newdimy*1)), '%' masklonoldpt1 = oldlon2d >= newlon2d[jnew, inew] - float(ddegL) masklonoldpt2 = oldlon2d <= newlon2d[jnew, inew] + float(ddegL) masklonoldpt = masklonoldpt1 * masklonoldpt2 masklatoldpt1 = oldlat2d >= newlat2d[jnew, inew] - float(ddegl) masklatoldpt2 = oldlat2d <= newlat2d[jnew, inew] + float(ddegl) masklatoldpt = masklatoldpt1 * masklatoldpt2 Nmasklon = np.sum(masklonoldpt) Nmasklat = np.sum(masklatoldpt) if Nmasklon > 0 and Nmasklat > 0: ## print newlon2d[jnew, inew], newlat2d[jnew, inew], ' ### ', np.min(oldlon2d), np.max(oldlon2d), ':', np.min(oldlat2d), np.max(oldlat2d) ## print masklonoldpt.shape, masklatoldpt.shape, Nmasklon, Nmasklat maskindlon = indices_mask(masklonoldpt) maskindlat = indices_mask(masklatoldpt) mindist=1000000. # for jold in range(olddimy): # for iold in range(olddimx): # dist=np.sqrt((oldlon2d[jold, iold] - newlon2d[jnew, inew])**2. + \ # (oldlat2d[jold, iold] - newlat2d[jnew, inew])**2.) for jold in range(Nmasklat): for iold in range(Nmasklon): if maskindlon[iold,0] == maskindlat[jold,0] and maskindlon[iold,1] == maskindlat[jold,1]: dist=np.sqrt((newlon2d[jnew, inew] - oldlon2d[maskindlon[iold,0], maskindlon[iold,1]])**2. + \ (newlat2d[jnew, inew] - oldlat2d[maskindlat[jold,0], maskindlat[jold,1]])**2.) ## print jold, iold, ':' , maskindlon[iold,:], '-->', maskindlat[jold,:], '**** L:', \ ## oldlon2d[maskindlon[iold,0], maskindlon[iold,1]], 'l', oldlat2d[maskindlat[jold,0], maskindlat[jold,1]], \ ## 'dist:',dist, 'mindist: ', mindist if dist < mindist: mindist = dist ## print jold, iold, 'dist: ', dist, 'L', maskindlon[iold,:], 'l', maskindlat[jold,:] mappairs[0, jnew, inew] = maskindlon[iold,1] mappairs[1, jnew, inew] = maskindlat[jold,0] ## quit() varobj = oldncfile.variables[varn] varinf = variable_inf(varobj) # Creation of the new file ## ncfnew = NetCDFFile(ofile,'w') for idim in range(varinf.Ndims): newdimv = oldncfile.dimensions[varinf.dimns[idim]] if dimensions[varinf.dimns[idim]] == oldlon or dimensions[varinf.dimns[idim]] == oldlat: if dimensions[varinf.dimns[idim]] == oldlon: ncfnew.createDimension(varinf.dimns[idim], newdimx) else: ncfnew.createDimension(varinf.dimns[idim], newdimy) else: if newdimv.isunlimited(): ncfnew.createDimension(varinf.dimns[idim], None) else: ncfnew.createDimension(varinf.dimns[idim], varinf.dims[idim]) newvar = ncfnew.createVariable(varn, varinf.dtype, varinf.dimns, fill_value=fillValue) for attrn in varinf.attributes: if not attrn == '_FillValue': attr = newvar.setncattr(attrn, varobj.getncattr(attrn)) attr = newvar.setncattr('mask', 'variable remapped at nearest neighbors using ' + values.split(':')[0] + ' using ' + \ ' as longitude and latitude valules ') ncfnew.sync() ncfnew.close() for varns in ncf.variables: if not varns == varn: fvaradd(filename + ',' + varns, ofile) elif varns == oldlon: fvaradd(newprojectionfile + ',' + newlon, ofile) elif varns == oldlat: fvaradd(newprojectionfile + ',' + newlat, ofile) chvarname(oldlon, ofile, newlon) chvarname(oldlat, ofile, newlat) ncfnew = NetCDFFile(ofile,'a') # looping on all the variable ## varsecdims = [] Nvarloop=1 for insdim in range(varinf.Ndims - 2): Nvarloop = Nvarloop * varinf.dims[insdim] varsecdims.append(0) Nsecdims = len(varsecdims) for iloop in range(Nvarloop): varval = load_ncvariable_lastdims(oldncfile, varn, varsecdims, 2) for jnew in range(newdimy): for inew in range(newdimx): if ((jnew*newdimx + inew)%100 == 0): print ' ' + fname + ': ', '{0:5.2f}'.format(float((jnew*newdimx + inew)*100./(newdimx*newdimy))), '% done' newvarval[jnew, inew] = varval[mappairs[jnew, inew, 0], mappairs[jnew, inew, 1]] fill_ncvariable_lastdims(ncfnew,varn,varsecdims,2,newvarval) varsecdims[Nsecdims -1] = varsecdims[Nsecdims - 1] + 1 if varsecdims[Nsecdims - 1] > varinf.dims[Nsecdims - 1] - 1: varsecdims[Nsecdims - 1] = 0 for jdim in range(Nsecdims - 1): varsecdims[Nsecdims - 2 - jdim] = varsecdims[Nsecdims - 2 - jdim] + 1 if varsecdims[Nsecdims - 2 - jdim] <= varinf.dims[Nsecdims - 2 - jdim] - 1: break else: varsecdims[Nsecdims - 2 - jdim] = 0 ncfnew.sync() ncfnew.close() print 'remapnn: new file "' + ofile + '" with remapped variable written !!' #remapnn('data/R1_CCRC_NARCliM_MOM_1950-2009_pracc.nc:lon,lat:lon,lat', 'data/AWAP_pre_1950-2009_pre_mask.nc', 'pre', 1. , 1.) #quit() def checkallvars(values, filen): """ Function to check all variables of a file values=[dimn1],[[dimn2],...,[dimnN]]:[dim1],[[dim2],...,[dimN]] [dimn1/N]: name of dimension [dim1/N]: value for the given dimension [0-N]: dimension value -1: last value -2: half of the dimension size -3: all values filen=name of the netCDF file to check """ fname = "'checkallvars'" dimns = values.split(':')[0].split(',') dims0 = np.array(values.split(':')[1].split(',')) Ndims = len(dimns) dims = np.zeros( (Ndims), dtype=np.int) dimvals = np.zeros( (Ndims), dtype=np.int) ncobj = NetCDFFile(filen,'r') for idim in range(Ndims): dims[idim] = np.int(dims0[idim]) if not ncobj.dimensions.has_key(dimns[idim]): print errormsg print ' ' + fname + " file '" + filen + "' does not have dimension '" + dimns[idim] + "' !!!!" quit(-1) sizedim = len(ncobj.dimensions[dimns[idim]]) if dims[idim] == -3: dimvals[idim] = -1 elif dims[idim] == -2: dimvals[idim] = sizedim/2 elif dims[idim] == -1: dimvals[idim] = sizedim-1 else: dimvals[idim] = dims[idim] print " dimension '" + dimns[idim] + "' of size: ",sizedim,' shown at :',\ dimvals[idim] ivar = 1 for varn in ncobj.variables: varobj = ncobj.variables[varn] varinf = variable_inf(varobj) dimsvar = np.zeros( (varinf.Ndims), dtype=np.int) dimsvar[:] = -9 for ivardim in range(varinf.Ndims): vardimn = varinf.dimns[ivardim] for idim in range(Ndims): if vardimn == dimns[idim]: dimsvar[ivardim] = dimvals[idim] if dimsvar[ivardim] == -9: print errormsg print ' ' + fname + " No value given to the dimension '" + vardimn +\ "' !!" quit(-1) print ivar, ':', varn + ' with: ', varinf.dimns,' outputted as: ',dimsvar,' ___________________' if varinf.Ndims == 1: if dimsvar == -1: varvals = varobj[:] else: varvals = varobj[dimsvar] elif varinf.Ndims == 2: if dimsvar[0] == -1 and dimsvar[1] == -1: varvals = varobj[:] elif dimsvar[0] == -1 and dimsvar[1] != -1: varvals = varobj[:,dimsvar[1]] elif dimsvar[0] != -1 and dimsvar[1] == -1: varvals = varobj[dimsvar[0],:] else: varvals = varobj[dimsvar[0],dimsvar[1]] elif varinf.Ndims == 3: if dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] == -1: varvals = varobj[:] elif dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] != -1: varvals = varobj[:,:,dimsvar[2]] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] == -1: varvals = varobj[:,dimsvar[1],:] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] == -1: varvals = varobj[dimsvar[0],:,:] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] != -1: varvals = varobj[:,dimsvar[1],dimsvar[2]] elif dimsvar[0] != -1 and dimsvar[1] != -1 and dimsvar[2] == -1: varvals = varobj[dimsvar[0],dimsvar[1],:] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] != -1: varvals = varobj[dimsvar[0],:,dimsvar[2]] else: varvals = varobj[dimsvar[0],dimsvar[1],dimsvar[2]] elif varinf.Ndims == 4: if dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] == -1 and dimsvar[3] == -1: varvals = varobj[:] elif dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] == -1 and dimsvar[3] != -1: varvals = varobj[:,:,:,dimsvar[2]] elif dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] != -1 and dimsvar[3] == -1: varvals = varobj[:,:,dimsvar[2],:] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] == -1 and dimsvar[3] == -1: varvals = varobj[:,dimsvar[1],:,:] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] == -1 and dimsvar[3] == -1: varvals = varobj[dimsvar[0],:,:,:] elif dimsvar[0] == -1 and dimsvar[1] == -1 and dimsvar[2] != -1 and dimsvar[3] != -1: varvals = varobj[:,:,dimsvar[2],dimsvar[3]] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] == -1 and dimsvar[3] != -1: varvals = varobj[:,dimsvar[1],:,dimsvar[3]] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] == -1 and dimsvar[3] != -1: varvals = varobj[dimsvar[1],:,:,dimsvar[3]] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] != -1 and dimsvar[3] == -1: varvals = varobj[:,dimsvar[1],dimsvar[2],:] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] != -1 and dimsvar[3] == -1: varvals = varobj[dimsvar[0],:,dimsvar[2],:] elif dimsvar[0] != -1 and dimsvar[1] != -1 and dimsvar[2] == -1 and dimsvar[3] == -1: varvals = varobj[dimsvar[0],dimsvar[1],:,:] elif dimsvar[0] == -1 and dimsvar[1] != -1 and dimsvar[2] != -1 and dimsvar[3] != -1: varvals = varobj[:,dimsvar[1],dimsvar[2],dimsvar[3]] elif dimsvar[0] != -1 and dimsvar[1] == -1 and dimsvar[2] != -1 and dimsvar[3] != -1: varvals = varobj[dimsvar[0],:,dimsvar[2],dimsvar[3]] elif dimsvar[0] != -1 and dimsvar[1] != -1 and dimsvar[2] == -1 and dimsvar[3] != -1: varvals = varobj[dimsvar[0],dimsvar[1],:,dimsvar[3]] elif dimsvar[0] != -1 and dimsvar[1] != -1 and dimsvar[2] != -1 and dimsvar[3] == -1: varvals = varobj[dimsvar[0],dimsvar[1],dimsvar[2],:] else: varvals = varobj[dimsvar[0],dimsvar[1],dimsvar[2],dimvals[3]] else: print errormsg print ' ' + fname + ' number of dimensions ',varinf.Ndims,' not ready!!' quit(-1) print varvals ivar = ivar + 1 ncobj.close() def give_fix1dim_values(mat,iddim,dimval): """ Function to return matrix values when one dimension has been removed at a given value mat = matrix iddim = dimesion to remove (starting from 0 tacking the right most) dimval = value of the dimension >>> matrix = np.array( (range(27)), dtype=np.float).reshape(3,3,3) give_fix1dim_values(matrix,2,2) [[[ 0. 1. 2.] [ 3. 4. 5.] [ 6. 7. 8.]] [[ 9. 10. 11.] [ 12. 13. 14.] [ 15. 16. 17.]]] """ fname='give_fix1dim_values' # Not working ma.squeeze ## import numpy.ma as ma ## print ma.squeeze(mat, axis = (idim,)) matshape = mat.shape mattype = mat.dtype matmask = np.ones( tuple(matshape), dtype=bool) zerodims=np.ones( (6), dtype=int) Ndims = len(matshape) if Ndims > 6: print errmsg print ' ' + fname + ' number of dimensions: ',Ndims,' not ready!!!!!' quit(-1) zerodims[5-Ndims+1:6] = matshape newdims = list(matshape) newdims[Ndims-1-iddim] = matshape[iddim]-1 mask = np.ones(tuple(zerodims), dtype=bool) vals = np.zeros(tuple(zerodims), dtype=mattype) vals[0:zerodims[0],0:zerodims[1],0:zerodims[2],0:zerodims[3],0:zerodims[4], 0:zerodims[5]] = mat for i5 in range(zerodims[5]): for i4 in range(zerodims[4]): for i3 in range(zerodims[3]): for i2 in range(zerodims[2]): for i1 in range(zerodims[1]): for i0 in range(zerodims[0]): # print 'iddim:',iddim,'dimval:',dimval,'-->',i0,i1,i2,i3,i4,i5 if iddim == 5 and dimval == i0: mask[i0,i1,i2,i3,i4,i5] = False elif iddim == 4 and dimval == i1: mask[i0,i1,i2,i3,i4,i5] = False elif iddim == 3 and dimval == i2: mask[i0,i1,i2,i3,i4,i5] = False elif iddim == 2 and dimval == i3: mask[i0,i1,i2,i3,i4,i5] = False elif iddim == 1 and dimval == i4: mask[i0,i1,i2,i3,i4,i5] = False elif iddim == 0 and dimval == i5: mask[i0,i1,i2,i3,i4,i5] = False newmat = vals[mask].reshape(tuple(newdims)) # print 'newmat dim:',iddim,'val:',dimval,':',newdims,'______' # print newmat return newmat def portion_fix1dim_values(mat,iddim,dimval): """ Function to return that portion of the matrix values when one dimension has been removed at a given value mat = matrix iddim = dimesion to remove (starting from 0 tacking the right most) dimval = value of the dimension >>> matrix = np.array( (range(27)), dtype=np.float).reshape(3,3,3) portion_fix1dim_values(matrix,2,2) [[ 18. 19. 20.] [ 21. 22. 23.] [ 24. 25. 26.]] """ fname='portion_fix1dim_values' # Not working ma.squeeze ## import numpy.ma as ma ## print ma.squeeze(mat, axis = (idim,)) matshape = mat.shape mattype = mat.dtype matmask = np.zeros( tuple(matshape), dtype=bool) zerodims=np.ones( (6), dtype=int) Ndims = len(matshape) if Ndims > 6: print errmsg print ' ' + fname + ' number of dimensions: ',Ndims,' not ready!!!!!' quit(-1) zerodims[5-Ndims+1:6] = matshape newdims = [] for idim in range(Ndims): if idim != iddim: newdims.append(matshape[idim]) newdims.reverse() mask = np.zeros(tuple(zerodims), dtype=bool) vals = np.zeros(tuple(zerodims), dtype=mattype) vals[0:zerodims[0],0:zerodims[1],0:zerodims[2],0:zerodims[3],0:zerodims[4], 0:zerodims[5]] = mat for i5 in range(zerodims[5]): for i4 in range(zerodims[4]): for i3 in range(zerodims[3]): for i2 in range(zerodims[2]): for i1 in range(zerodims[1]): for i0 in range(zerodims[0]): # print 'iddim:',iddim,'dimval:',dimval,'-->',i0,i1,i2,i3,i4,i5 if iddim == 5 and dimval == i0: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 4 and dimval == i1: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 3 and dimval == i2: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 2 and dimval == i3: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 1 and dimval == i4: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 0 and dimval == i5: mask[i0,i1,i2,i3,i4,i5] = True # print 'newmat dim:',iddim,'val:',dimval,':',newdims,'______' newmat = vals[mask].reshape(tuple(newdims)) # print newmat return newmat def valmod_dim(values, ncfile, varn): """ Function to modify the value of a variable at a given dimension and value values = dimn,dimval,modins,modval1,[modval2,...] dimn = name of the dimension dimval = value of the dimension at which change the values modins = instruction: 'sumc', add [modval1] 'subc', substraction [modval1] 'mulc', multiply by [modval1] 'divc', divide by [modval1] 'lowthres': modify all values below [modval1] to [modval2] 'upthres': modify all values above [modval1] to [modval2] ncfile = netCDF file name varn = name of the variable >>> valmod_dim('num_metgrid_levels,10,mulc,-1','aqua_met-em_control.nc','PRES') """ fname = 'valmod_dim' if values == '-h': print """ Function to modify the value of a variable at a given dimension and value values = dimn,dimval,modins,modval1,[modval2,...] dimn = name of the dimension dimval = value of the dimension at which change the values modins = instruction: 'sumc', add [modval1] 'subc', substraction [modval1] 'mulc', multiply by [modval1] 'divc', divide by [modval1] 'lowthres': modify all values below [modval1] to [modval2] 'upthres': modify all values above [modval1] to [modval2] ncfile = netCDF file name varn = name of the variable >>> valmod_dim('num_metgrid_levels,10,mulc,-1','aqua_met-em_control.nc','PRES') """ quit() ## mat = np.array( (range(27)), dtype=np.float).reshape(3,3,3) vals = values.split(',') Lvals = len(vals) dimn = vals[0] dimval = np.int(vals[1]) modins = vals[2] modval = np.float(vals[3]) newvalues = ','.join(vals[2:Lvals]) if not os.path.isfile(ncfile): print errormsg print ' valmod: File "' + ncfile + '" does not exist !!' print errormsg quit(-1) ncf = NetCDFFile(ncfile,'a') if ncf.dimensions.has_key('plev'): # removing pressure level from variable name varn = re.sub("\d+", "", varn) if not ncf.dimensions.has_key(dimn): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not have dimension "' + \ dimn + '" !!!!' ncf.close() quit(-1) if not ncf.variables.has_key(varn): print errormsg print ' ' + fname + ': File "' + ncfile + '" does not have variable "' + \ varn + '" !!!!' ncf.close() quit(-1) varobj = ncf.variables[varn] varinf = variable_inf(varobj) varvals = varobj[:] # Dimension id iddim=varinf.Ndims - 1 for idimn in varinf.dimns: # print iddim, idimn if dimn == idimn: break iddim = iddim - 1 matshape = varvals.shape mattype = varvals.dtype matmask = np.zeros( tuple(matshape), dtype=bool) zerodims=np.ones( (6), dtype=int) Ndims = len(matshape) if Ndims > 6: print errmsg print ' ' + fname + ' number of dimensions: ',Ndims,' not ready!!!!!' quit(-1) zerodims[5-Ndims+1:6] = matshape newdims = [] for idim in range(Ndims): # print idim,'matshape[idim]:',matshape[Ndims-1-idim],'iddim:',iddim if idim != iddim: newdims.append(matshape[Ndims-1-idim]) newdims.reverse() mask = np.zeros(tuple(zerodims), dtype=bool) vals = np.zeros(tuple(zerodims), dtype=mattype) newvals = np.zeros(tuple(zerodims), dtype=mattype) vals[0:zerodims[0],0:zerodims[1],0:zerodims[2],0:zerodims[3],0:zerodims[4], \ 0:zerodims[5]] = varvals # print 'vals shape:',vals.shape,':',newdims for i5 in range(zerodims[5]): for i4 in range(zerodims[4]): for i3 in range(zerodims[3]): for i2 in range(zerodims[2]): for i1 in range(zerodims[1]): for i0 in range(zerodims[0]): # print 'iddim:',iddim,'dimval:',dimval,'-->',i0,i1,i2,i3,i4,i5 if iddim == 5 and dimval == i0: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 4 and dimval == i1: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 3 and dimval == i2: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 2 and dimval == i3: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 1 and dimval == i4: mask[i0,i1,i2,i3,i4,i5] = True elif iddim == 0 and dimval == i5: mask[i0,i1,i2,i3,i4,i5] = True # print 'newmat dim:',iddim,'val:',dimval,':',newdims,'______' newmat = vals[mask].reshape(tuple(newdims)) varmod = np.zeros(tuple(newdims), dtype=mattype) varshape = newmat.shape vartype = newmat.dtype Ndims = len(varshape) if Ndims <= 2: varmod = valmodoper(newmat, newvalues) elif Ndims == 3: for i in range(varshape[0]): varmod[i,:,:] = valmodoper(newmat[i,:,:], newvalues) elif Ndims == 4: for i in range(varshape[0]): for j in range(varshape[1]): varmod[i,j,:,:] = valmodoper(newmat[i,j,:,:], newvalues) elif Ndims == 5: for i in range(varshape[0]): for j in range(varshape[1]): for k in range(varshape[2]): varmod[i,j,k,:,:] = valmodoper(newmat[i,j,k,:,:], newvalues) newvals = np.where( mask, varmod, vals) zerdims = np.where( zerodims == 1, 0, zerodims ) finalvals = newvals[0:zerodims[0],0:zerodims[1],0:zerodims[2],0:zerodims[3],\ 0:zerodims[4],0:zerodims[5]] finalvals = np.squeeze(finalvals) if len(varobj.shape) != len(finalvals.shape): print ' Modified values have a different shape:', finalvals.shape, \ 'than variable:', varobj.shape print ' Assuming that extra dimensions are at the beggining' newfinalvals = finalvals.reshape(varobj.shape) varobj[:] = newfinalvals[:] else: varobj[:] = finalvals[:] ncf.sync() ncf.close() def checkNaNs(values, filen): """ Function to check for NaN values over all variables in a file values = absolute lower threshold for NaN value filen=name of the netCDF file to check """ fname = "'checkNaNs'" valthres = np.float(values) ncobj = NetCDFFile(filen,'r') S1val = np.dtype('a1') Sval = np.dtype(str) Uval = np.dtype(unicode) ivar = 1 print fname + ' checking with: ',valthres,' in _______' for varn in ncobj.variables: varobj = ncobj.variables[varn] varinf = variable_inf(varobj) varvals = varobj[:] var1 = varvals.flatten()[0] if varinf.dtype != type('c') and varinf.dtype != type('Str') and \ varinf.dtype != Sval.type and varinf.dtype != Uval.type and \ varinf.dtype != S1val.type and type(var1) != S1val.type and \ type(var1) != Sval.type and type(var1) != Uval.type: print ' ' + varn + ':',varinf.dims found=0 if varinf.Ndims == 1: for i0 in range(varinf.dims[0]): if abs(varvals[i0]) > valthres or np.isnan(varvals[i0]): print ' ' + fname + ': NaN value found on variable "' + \ varn + '" at: ',i0,' value: ',varvals[i0] break elif varinf.Ndims == 2: for i1 in range(varinf.dims[0]): for i0 in range(varinf.dims[1]): if abs(varvals[i1,i0]) > valthres or np.isnan(varvals[i1,i0]): print ' ' + fname + ': NaN value found on variable "' + \ varn + '" at: ',i1,', ',i0,' value: ',varvals[i1,i0] found=1 break if found == 1: break elif varinf.Ndims == 3: for i2 in range(varinf.dims[0]): for i1 in range(varinf.dims[1]): for i0 in range(varinf.dims[2]): if abs(varvals[i2,i1,i0]) > valthres or \ np.isnan(varvals[i2,i1,i0]): print ' ' + fname + \ ': NaN value found on variable "' + varn + \ '" at: ', i2,', ',i1,', ',i0,' value: ', \ varvals[i2,i1,i0] found=1 break if found == 1: break if found == 1: break elif varinf.Ndims == 4: for i3 in range(varinf.dims[0]): for i2 in range(varinf.dims[1]): for i1 in range(varinf.dims[2]): for i0 in range(varinf.dims[3]): if abs(varvals[i3,i2,i1,i0]) > valthres or \ np.isnan(varvals[i3,i2,i1,i0]): print ' ' + fname + \ ': NaN value found on variable "' + varn + \ '" at: ', i3,', ', i2,', ',i1,', ',i0, \ ' value: ',varvals[i3,i2,i1,i0] found=1 break if found == 1: break if found == 1: break if found == 1: break elif varinf.Ndims == 5: for i4 in range(varinf.dims[0]): for i3 in range(varinf.dims[1]): for i2 in range(varinf.dims[2]): for i1 in range(varinf.dims[3]): for i0 in range(varinf.dims[4]): if abs(varvals[i4,i3,i2,i1,i0]) > valthres or \ np.isnan(varvals[i4,i3,i2,i1,i0]): print ' ' + fname + \ ': NaN value found on variable "' + varn + \ '" at: ', i4,', ', i3,', ', i2,', ',i1, \ ', ',i0,' value: ',varvals[i4,i3,i2,i1,i0] found=1 break if found == 1: break if found == 1: break if found == 1: break if found == 1: break else: print errormsg print ' ' + fname + ' number of dimensions ',varinf.Ndims, \ ' not ready!!' quit(-1) ivar = ivar + 1 ncobj.close() ##checkNaNs(10.e10, '/d4/lflmd/etudes/WRF_LMDZ/WaquaL/WRF/control/wrfout_d01_1979-12-01_00:00:00') ##checkNaNs(10.e10, '/d4/lflmd/etudes/FF/tests/em_quarter_ss/run/wrfout_d01_0001-01-01_00:00:00') def checkAllValues(values, filen): """ Function to check for variables with along all their dimensions with the same value in a file values = value to check filen=name of the netCDF file to check """ fname = "'checkAllValues'" valcheck = np.float(values) ncobj = NetCDFFile(filen,'r') S1val = np.dtype('a1') Sval = np.dtype(str) Uval = np.dtype(unicode) ivar = 1 print fname + ' checking with: ',valcheck,' in _______' for varn in ncobj.variables: varobj = ncobj.variables[varn] varinf = variable_inf(varobj) varvals = varobj[:] var1 = varvals.flatten()[0] if varinf.dtype != type('c') and varinf.dtype != type('Str') and \ varinf.dtype != Sval.type and varinf.dtype != Uval.type and \ varinf.dtype != S1val.type and type(var1) != S1val.type and \ type(var1) != Sval.type and type(var1) != Uval.type: print ' ' + varn + ':',varinf.dims if np.all(varvals == valcheck): print ' ' + fname + ': variable "' + varn + \ '" with all its values == ',valcheck ivar = ivar + 1 ncobj.close() ##checkAllValues(0., '/d4/lflmd/etudes/WRF_LMDZ/WaquaL/WRF/control/wrfout_d01_1979-12-01_00:00:00') ##checkAllValues(0., '/d4/lflmd/etudes/FF/tests/em_quarter_ss/run/wrfout_d01_0001-01-01_00:00:00') ##checkAllValues(0., '/home/lluis/etudes/WRF_LMDZ/tests/wrf_input/AR40.0/wrfout_d01_1979-01-01_00:00:00') def DataSetSection(values, filen): """ Function to get a section (values along a dimension) of a given data-set values= [dimn],[beg],[end],[int] [dimn]: name of the dimension [beg],[end],[int]: beginning, end and interval along the dimension-axis [end] = -1, values until the end filen= netCDF with the data-set DataSetSection('time,3,15,1','test.nc') """ fname = 'DataSetSection' if values == 'h': print fname + '_____________________________________________________________' print DataSetSection.__doc__ quit() dimn = values.split(',')[0] begv = np.int(values.split(',')[1]) endv = np.int(values.split(',')[2]) intv = np.int(values.split(',')[3]) ofile=filen.split('.')[0] + '_' + dimn + '_B' + str(begv) + '-E' + str(endv) + \ '-I' + str(intv) + '.nc' nciobj = NetCDFFile(filen,'r') ncoobj = NetCDFFile(ofile,'w') # Creating dimensions ## for dims in nciobj.dimensions: objdim = nciobj.dimensions[dims] if objdim.isunlimited(): dimsize = None else: if dims == dimn: if begv > len(objdim): print errormsg print ' ' + fname + ': beginning value' ,begv, \ 'is larger than dimension (', len(objdim), ')!!!!' quit(-1) if endv > len(objdim): print errormsg print ' ' + fname + ': endining value' ,endv, \ 'is larger than dimension (', len(objdim), ')!!!!' quit(-1) if endv == -1: endv = len(objdim) dimsize = (endv - begv + 1)/intv - 1 else: dimsize = len(objdim) print ' ' + fname + ': adding dimension: ' + dims + ' size:',dimsize dim = ncoobj.createDimension(dims, dimsize) ncoobj.sync() # Creating variables ## for varns in nciobj.variables: print ' ' + fname + ': adding variable "' + varns + '"...' varobj = nciobj.variables[varns] vardims = varobj.dimensions if not searchInlist(list(vardims),dimn): varvals = varobj[:] else: varslice = [] for dimname in varobj.dimensions: if dimname == dimn: varslice.append(slice(begv,endv,intv)) else: Ldim = len(nciobj.dimensions[dimname]) varslice.append(slice(0,Ldim)) varvals = varobj[tuple(varslice)] # Adding fill value attribute ## varattrs = varobj.ncattrs() if searchInlist(varattrs, '_FillValue'): varfil = varobj._FillValue else: varfil = False vartype = varobj.dtype newvar = ncoobj.createVariable(varns, vartype, vardims, fill_value=varfil) newvar[:] = varvals for attrs in varattrs: if not attrs == '_FillValue': attrv = varobj.getncattr(attrs) attr = set_attribute(newvar, attrs, attrv) # Global attributes ## for attrs in nciobj.ncattrs(): attrv = nciobj.getncattr(attrs) attr = set_attribute(ncoobj, attrs, attrv) nciobj.close() ncoobj.sync() ncoobj.close() print ' ' + fname + ' succesfull creation of file "' + ofile + '" !!!' return def DataSetSection_multidims(values, filen, varn): """ Function to get a section (values along multiple dimensions) of a given data-set values= [dimn1],[beg1],[end1],[int1]@[...[[dimnM],[begM],[endM],[intM]]] [dimni]: name of the dimension [begi],[endi],[inti]: beginning, end and interval along the dimension-axis [endi] = -1, last value NOTE: dimensions without values are taken allover their size filen= netCDF with the data-set varn= ',' list of variables, 'all' for all variables in data-set DataSetSection_multidims('Time,-1,-1,1@bottom_top,6,6,1','wrfout_d01_1979-12-01_00:00:00','all') """ fname = 'DataSetSection_multidims' if values == 'h': print fname + '_____________________________________________________________' print DataSetSection_multidims.__doc__ quit() Ndims = len(values.split('@')) dimns = [] ofiletile = '' begvs = np.zeros((Ndims), dtype=int) endvs = np.zeros((Ndims), dtype=int) intvs = np.zeros((Ndims), dtype=int) for idim in range(Ndims): val = values.split('@')[idim] dimns.append(val.split(',')[0]) begvs[idim] = int(val.split(',')[1]) endvs[idim] = int(val.split(',')[2]) intvs[idim] = int(val.split(',')[3]) ofiletile = ofiletile + '_' + dimns[idim] + '_B' + str(begvs[idim]) + '-E' + \ str(endvs[idim]) + '-I' + str(intvs[idim]) ofile = ofile=filen.split('.')[0] + ofiletile + '.nc' nciobj = NetCDFFile(filen,'r') ncoobj = NetCDFFile(ofile,'w') # Creating dimensions ## print ' ' + fname + ': adding dimensions...' newdimsizes = {} dimslices = {} varslice = [] for dims in nciobj.dimensions: objdim = nciobj.dimensions[dims] dimsize = -1 if not searchInlist(dimns, dims): if objdim.isunlimited(): dimsize = None else: dimsize = len(objdim) varslice.append(slice(0,len(objdim))) else: dimid = dimns.index(dims) if begvs[dimid] > len(objdim): print errormsg print ' ' + fname + ': beginning value' ,begvs[dimid], \ 'is larger than dimension (', len(objdim), ')!!!!' quit(-1) if endvs[dimid] > len(objdim): print errormsg print ' ' + fname + ': endining value' ,endvs[dimid], \ 'is larger than dimension (', len(objdim), ')!!!!' quit(-1) if begvs[dimid] != endvs[dimid]: if endvs[dimid] != -1: dimsize = (endvs[dimid]-begvs[dimid]+1)/intvs[dimid] dimslices[dims] = [begvs[dimid],endvs[dimid]+1,intvs[dimid]] else: print 'here!' dimsize = len(objdim) dimslices[dims] = [begvs[dimid],len(objdim),1] else: dimsize = 1 if begvs[dimid] == -1: dimslices[dims] = [len(objdim)-1,len(objdim),1] else: dimslices[dims] = [begvs[dimid],begvs[dimid]+1,1] newdimsizes[dims] = dimsize if dimsize != -1: if len(objdim) == 1 or dimsize != 1: print ' ' + fname + ': adding dimension: '+ dims +' size:',dimsize if objdim.isunlimited(): dimsize=None dim = ncoobj.createDimension(dims, dimsize) ncoobj.sync() print ' ' + fname + ': Slice according to dimension selection: ',dimslices # Creating variables ## print ' ' + fname + ': adding variables...' if varn == 'all': getvarns = nciobj.variables else: getvarns = varn.split(',') for varns in getvarns: print ' ' + fname + ': adding variable "' + varns + '"...' varobj = nciobj.variables[varns] varorigdims = varobj.dimensions vardims = [] # From: http://stackoverflow.com/questions/1388818/how-can-i-compare-two-lists-in-python-and-return-matches coinc = list(set(varorigdims) & set(dimns)) if len(coinc) == 0: varvals = varobj[:] vardims = varorigdims else: varslice = [] for dimname in varorigdims: if searchInlist(dimns, dimname): varslice.append(slice(dimslices[dimname][0], \ dimslices[dimname][1], dimslices[dimname][2])) if newdimsizes[dimname] != 1: vardims.append(dimname) else: Ldim = len(nciobj.dimensions[dimname]) varslice.append(slice(0,Ldim)) vardims.append(dimname) varvals = varobj[tuple(varslice)] # Adding fill value attribute ## # print ' final variable dimensions:',vardims varattrs = varobj.ncattrs() if searchInlist(varattrs, '_FillValue'): varfil = varobj._FillValue else: varfil = False vartype = varobj.dtype newvar = ncoobj.createVariable(varns, vartype, tuple(vardims), \ fill_value=varfil) # print 'newvar:',newvar.shape,'varvals:',varvals.shape newvar[:] = varvals for attrs in varattrs: if not attrs == '_FillValue': attrv = varobj.getncattr(attrs) attr = set_attribute(newvar, attrs, attrv) # Global attributes ## for attrs in nciobj.ncattrs(): attrv = nciobj.getncattr(attrs) attr = set_attribute(ncoobj, attrs, attrv) slicevalS = '' for dimid in range(Ndims): dn = dimns[dimid] if dimslices[dn][2] is not None: slicevalS = slicevalS + dn + ' (' + str(dimslices[dn][0]) + ',' + \ str(dimslices[dn][1]) + ',' + str(dimslices[dn][2]) + '); ' else: slicevalS = slicevalS + dn + ' (' + str(dimslices[dn][0]) + ',' + \ str(dimslices[dn][1]) + ',1); ' attr = set_attribute(ncoobj, 'sliced_dimensions', slicevalS) nciobj.close() ncoobj.sync() ncoobj.close() print ' ' + fname + ' succesfull creation of file "' + ofile + '" !!!' return #DataSetSection_multidims('Time,-1,-1,1@bottom_top,6,6,1','wrfout_d01_1979-12-01_00:00:00') #DataSetSection_multidims('bottom_top,6,6,1@south_north,3,3,1@west_east,26,26,1','wrfout_d01_1979-12-01_00:00:00') def sellonlatbox(values, ncfile, varn): """ Function to select a lotlan box from a data-set values= [lonName],[latName],[lonSW],[latSW],[lonNE],[latNE] [lonName]: name of the variable with the longitudes [latName]: name of the variable with the latitudes [lonSW],[latSW]: longitude and latitude of the SW vertex of the box [lonNE],[latNE]: longitude and latitude of the NE vertex of the box ncfile= netCDF file varn= ',' list of names of the variables ('all', for all variables) """ import numpy.ma as ma fname = 'sellonlatbox' if values == 'h': print fname + '_____________________________________________________________' print sellonlatbox.__doc__ quit() arguments = '[lonName],[latName],[lonSW],[latSW],[lonNE],[latNE]' check_arguments(fname,values,arguments,',') lonn = values.split(',')[0] latn = values.split(',')[1] lonSW = np.float(values.split(',')[2]) latSW = np.float(values.split(',')[3]) lonNE = np.float(values.split(',')[4]) latNE = np.float(values.split(',')[5]) objfile = NetCDFFile(ncfile, 'r') if not objfile.variables.has_key(lonn): print errormsg print ' ' + fname + ": file '" + ncfile + "' des not have longitude " + \ "variable '" + lonn + "' !!" quit(-1) if not objfile.variables.has_key(latn): print errormsg print ' ' + fname + ": file '" + ncfile + "' des not have latitude " + \ "variable '" + lonn + "' !!" quit(-1) lonobj = objfile.variables[lonn] latobj = objfile.variables[latn] loninf = variable_inf(lonobj) latinf = variable_inf(latobj) lonv = lonobj[:] latv = latobj[:] if varn == 'all': varns = objfile.variables filevarn = varn elif varn.find(',') != -1: varns = varn.split(',') filevarn = varn.replace(',','-') else: varns = [varn] filevarn = varn ofile = 'sellonlatbox_' + filevarn + '.nc' # Lon/lat box nalon = lonv.min() nalat = latv.min() xalon = lonv.max() xalat = latv.max() if lonSW < nalon: print errormsg print ' ' + fname + ': longitude SW corner:', lonSW,' too small!!' print ' min lon data:', nalon, 'increase SW corner longitude' quit(-1) if latSW < nalat: print errormsg print ' ' + fname + ': latitude SW corner:', latSW,' too small!!' print ' min lat data:', nalat, 'increase SW corner latitude' quit(-1) if lonNE > xalon: print errormsg print ' ' + fname + ': longitude NE corner:', lonNE,' too large!!' print ' max lon data:', xalon, 'decrease NE corner longitude' quit(-1) if latNE > xalat: print errormsg print ' ' + fname + ': latitude NE corner:', latNE,' too large!!' print ' max lat data:', xalat, 'decrease NE corner latitude' quit(-1) malon = ma.masked_outside(lonv,lonSW,lonNE) malat = ma.masked_outside(latv,latSW,latNE) if len(lonv.shape) == 1: dlonn = loninf.dimns[0] dlatn = latinf.dimns[0] dimx = lonv.shape[0] dimy = latv.shape[0] malonlat = np.zeros((dimy,dimx), dtype=lonv.dtype) for i in range(dimx): malonlat[:,i] = malon[i] + malat elif len(lonv.shape) == 2: dlonn = loninf.dimns[1] dlatn = loninf.dimns[0] dimx = lonv.shape[1] dimy = lonv.shape[0] malonlat = malon.mask + malat.mask malonv = ma.array(lonv, mask=malonlat) malatv = ma.array(latv, mask=malonlat) elif len(lonv.shape) == 3: print warnmsg print ' ' + fname + ': assuming that lon/lat is constant to the first dimension!!' print ' shape lon:', lonv.shape, 'lat:', latv.shape dlonn = loninf.dimns[2] dlatn = loninf.dimns[1] dimx = lonv.shape[2] dimy = lonv.shape[1] malonlat = np.zeros((dimy,dimx), dtype=bool) malonlat = malon.mask[0,:,:] + malat.mask[0,:,:] malonv = ma.array(lonv, mask=malon) malatv = ma.array(latv, mask=malon) else: print errormsg print ' ' + fname + ': matrix shapes not ready!!' print ' shape lon:',lonv.shape,'lat:',latv.shape quit(-1) # Size of the new domain nlon = malon.min() nlat = malat.min() xlon = malon.max() xlat = malat.max() print ' ' + fname + ': data box: SW', nlon, nlat,'NE:',xlon,xlat ilon = dimx for j in range(dimy): for i in range(dimx): if malonlat[j,i] == False and i < ilon: ilon = i elon = 0 for j in range(dimy): for i in range(dimx): if malonlat[j,i] == False and i > elon: elon = i ilat = dimy for j in range(dimy): for i in range(dimx): if malonlat[j,i] == False and j < ilat: ilat = j elat = 0 for j in range(dimy): for i in range(dimx): if malonlat[j,i] == False and j > elat: elat = j newdimx = elon - ilon + 1 newdimy = elat - ilat + 1 if len(malon.shape) == 1: datalonSW = lonv[ilon] datalatSW = latv[ilat] datalonNE = lonv[elon] datalatNE = latv[elat] elif len(malon.shape) == 2: datalonSW = lonv[ilat,ilon] datalatSW = latv[ilat,ilon] datalonNE = lonv[elat,elon] datalatNE = latv[elat,elon] elif len(malon.shape) == 3: datalonSW = lonv[0,ilat,ilon] datalatSW = latv[0,ilat,ilon] datalonNE = lonv[0,elat,elon] datalatNE = latv[0,elat,elon] print ' ' + fname + ': final SW:', datalonSW, ',', datalatSW, 'NE:', datalonNE, \ ',', datalatNE print ' ' + fname + ': new lon size:', ilon, ',', elon, 'new dimx:', newdimx print ' ' + fname + ': new lat size:', ilat, ',', elat, 'new dimy:', newdimy newncobj = NetCDFFile(ofile, 'w') # Creation of lon,lat related dims newdim = newncobj.createDimension(dlonn, newdimx) newdim = newncobj.createDimension(dlatn, newdimy) # Adding lonn/latn to the output file if not searchInlist(varns,lonn): varns.append(lonn) if not searchInlist(varns,latn): varns.append(latn) for vn in varns: print 'Adding "' + vn + "' ..." if not objfile.variables.has_key(lonn): print errormsg print ' ' + fname + ": file '" + ncfile + "' des not have variable '" + \ vn + "' !!" quit(-1) if not searchInlist(objfile.variables, vn): print errormsg print ' ' + fname + ': variable name "' + vn + '" is not in file "' + \ ncfile + '" !!!!!' quit(-1) varobj = objfile.variables[vn] varinf = variable_inf(varobj) # Adding variable dimensions varslice = [] for dimn in varinf.dimns: if not searchInlist(newncobj.dimensions, dimn): newncobj.createDimension(dimn, len(objfile.dimensions[dimn])) if dimn == dlonn: varslice.append(slice(ilon,elon+1)) elif dimn == dlatn: varslice.append(slice(ilat,elat+1)) else: varslice.append(slice(0,len(objfile.dimensions[dimn]))) if varinf.FillValue is not None: newvar = newncobj.createVariable(vn, nctype(varinf.dtype), varinf.dimns) else: newvar = newncobj.createVariable(vn, nctype(varinf.dtype), varinf.dimns, \ fill_value = varinf.FillValue) newvar[:] = varobj[tuple(varslice)] for atrn in varinf.attributes: if atrn != '_FillValue': attrv = varobj.getncattr(atrn) newattr = set_attributek(newvar, atrn, attrv, type(attrv)) newncobj.sync() for atrn in objfile.ncattrs(): attrv = objfile.getncattr(atrn) newattr = set_attributek(newncobj, atrn, attrv, type(attrv)) # global attributes newncobj.setncattr('author', 'L. Fita') newattr = set_attributek(newncobj, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') newncobj.setncattr('university', 'Pierre Marie Curie - Jussieu') newncobj.setncattr('center', 'Centre National de Recherches Scientifiques') newncobj.setncattr('city', 'Paris') newncobj.setncattr('country', 'France') newncobj.setncattr('script', 'nc_var_tools.py') newncobj.setncattr('function', 'sellonlatbox') newncobj.setncattr('version', '1.0') newncobj.setncattr('original_file', ncfile) newnattr = set_attributek(newncobj, 'lonSW', lonSW, 'R') newnattr = set_attributek(newncobj, 'latSW', latSW, 'R') newnattr = set_attributek(newncobj, 'lonNE', lonNE, 'R') newnattr = set_attributek(newncobj, 'latNE', latNE, 'R') newnattr = set_attributek(newncobj, 'data_lonSW', datalonSW, 'R') newnattr = set_attributek(newncobj, 'data_latSW', datalatSW, 'R') newnattr = set_attributek(newncobj, 'data_lonNE', datalonNE, 'R') newnattr = set_attributek(newncobj, 'data_latNE', datalatNE, 'R') newnattr = set_attributek(newncobj, 'data_pt_lonSW', ilon, 'I') newnattr = set_attributek(newncobj, 'data_pt_latSW', ilat, 'I') newnattr = set_attributek(newncobj, 'data_pt_lonNE', elon, 'I') newnattr = set_attributek(newncobj, 'data_pt_latNE', elat, 'I') objfile.close() newncobj.sync() newncobj.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return def sellonlatboxold(values, ncfile, varn): """ Function to select a lotlan box from a data-set values= [lonName],[latName],[lonSW],[latSW],[lonNE],[latNE] [lonName]: name of the variable with the longitudes [latName]: name of the variable with the latitudes [lonSW],[latSW]: longitude and latitude of the SW vertex of the box [lonNE],[latNE]: longitude and latitude of the NE vertex of the box ncfile= netCDF file varn= ',' list of names of the variables ('all', for all variables) """ fname = 'sellonlatboxold' if values == 'h': print fname + '_____________________________________________________________' print sellonlatbox.__doc__ quit() ofile = 'sellonlatbox_' + varn + '.nc' lonn = values.split(',')[0] latn = values.split(',')[1] lonSW = np.float(values.split(',')[2]) latSW = np.float(values.split(',')[3]) lonNE = np.float(values.split(',')[4]) latNE = np.float(values.split(',')[5]) objfile = NetCDFFile(ncfile, 'r') lonobj = objfile.variables[lonn] latobj = objfile.variables[latn] lonv = lonobj[:] latv = latobj[:] if varn == 'all': varns = objfile.variables elif varn.find(',') != -1: varns = varn.split(',') else: varns = [varn] for vn in varns: if not searchInlist(objfile.variables, vn): print errormsg print ' ' + fname + ': variable name "' + vn + '" is not in file "' + \ ncfile + '" !!!!!' quit(-1) varobj = objfile.variables[vn] Ndimslon = len(lonobj.shape) Ndimslat = len(latobj.shape) Ndimsvar = len(varobj.shape) # Looking for coincidence of dimensions samedim = [] for idv in range(Ndimsvar): for idl in range(Ndimslon): if varobj.dimensions[idv] == lonobj.dimensions[idl]: if not searchInlist(samedim,varobj.dimensions[idv]): samedim.append(varobj.dimensions[idv]) break for idl in range(Ndimslat): if varobj.dimensions[idv] == latobj.dimensions[idl]: if not searchInlist(samedim,varobj.dimensions[idv]): samedim.append(varobj.dimensions[idv]) break Ndimshare = len(samedim) print 'variable and lon/lat matrices share ', Ndimshare,' dimensions: ',samedim samedimslonlat = True for idL in range(Ndimslon): for idl in range(Ndimslat): if lonobj.dimensions[idl] != latobj.dimensions[idl]: samedimslonlat = False break if not samedimslonlat: break # Creation of the lon/lat matrices to search if Ndimshare == 1: lonmat = lonobj[:] latmat = latobj[:] elif Ndimshare == 2: if samedimslonlat: lonmat = lonobj[:] latmat = latobj[:] else: if Ndimslon != 1 and Ndimslat != 1: print errormsg print ' ' + fname + ': I do not know how to keep going!' print ' lon ', Ndimslon,' and lat ',Ndimslat, \ ' matrices do not share the same dimensions, and do not ' + \ 'have shape 1' quit(-1) else: lonmat = np.zeros((latobj.shape[0], lonobj.shape[0]), dtype=np.float) latmat = np.zeros((latobj.shape[0], lonobj.shape[0]), dtype=np.float) for i in range(lonobj.shape[0]): latmat[:,i] = latobj[:] for j in range(latobj.shape[0]): lonmat[j,:] = lonobj[:] elif Ndimshare == 3: if samedimslonlat: lonmat = lonobj[0,:,:] latmat = latobj[0,:,:] else: print errormsg print ' ' + fname + ': dimension sharing of lon/lat not ready!' quit(-1) # Searching for inside points iind = 0 if Ndimshare == 1: inside = {} for iL in range(lonobj.shape[0]): if lonobj[iL] >= lonSW and lonobj[iL] <= lonNE and latobj[iL] >= latSW\ and latobj[iL] <= latNE: inside[iind] = iL iind = iind + 1 elif Ndimshare == 2: newidx = [] newidy = [] if (len(lonobj.shape) == 3): inside = np.zeros((lonobj.shape[1], lonobj.shape[2]), dtype=bool) else: inside = np.zeros((lonobj.shape), dtype=bool) for iL in range(lonobj.shape[1]): for il in range(lonobj.shape[0]): if lonobj[il,iL] >= lonSW and lonobj[il,iL] <= lonNE and \ latobj[il,iL] >= latSW and latobj[il,iL] <= latNE: if not searchInlist(newidx, iL): newidx.append(iL) if not searchInlist(newidy, il): newidy.append(il) inside[il, iL] = True iind = iind + 1 elif Ndimshare == 3: newidx = [] newidy = [] inside = np.zeros((lonobj.shape[1], lonobj.shape[2]), dtype=bool) for iL in range(lonobj.shape[2]): for il in range(lonobj.shape[1]): if lonobj[0,il,iL] >= lonSW and lonobj[0,il,iL] <= lonNE and \ latobj[0,il,iL] >= latSW and latobj[0,il,iL] <= latNE: if not searchInlist(newidx, iL): newidx.append(iL) if not searchInlist(newidy, il): newidy.append(il) inside[il, iL] = True iind = iind + 1 Ninpts = len(inside) newdx = len(newidx) newdy = len(newidy) print ' ' + fname + ': ', Ninpts, 'pts found in the box:', lonSW, ',', \ latSW, 'x', lonNE, ',', latNE # Creation of cropped matrices inlon = np.zeros((Ninpts), dtype=np.float) inlat = np.zeros((Ninpts), dtype=np.float) invar = np.zeros((varobj.shape[0],Ninpts), dtype=np.float) # Creation of the netCDF file ## if vn in varns[0]: objofile = NetCDFFile(ofile, 'w') if Ndimshare == 1: # Dimensions newdim = objofile.createDimension('boxpt', Ninpts) newdim = objofile.createDimension('time', None) # var dimensions newvar = objofile.createVariable('lon', 'f8', ('boxpt')) newattr = basicvardef(newvar, 'lon', 'longitude', 'degrees west_east') newvar[:] = lonobj[inside[iin]] newvar = objofile.createVariable('lat', 'f8', ('boxpt')) newattr = basicvardef(newvar, 'lat', 'latitude', 'degrees north_south') newvar[:] = latobj[inside[iin]] newvar = objofile.createVariable('time', 'f8', ('time')) if objfile.variables.has_key('time'): otime = objfile.variables['time'] timevals = otime[:] if searchInlist(otime.ncattrs(),'units'): tunits = otime.getncattr['units'] else: tunits = 'unkown' else: timevals = np.arange(varobj.shape[0]) tunits = 'unkown' newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = timevals # variable newvar = objofile.createVariable(varn, 'f4', ('time', 'boxpt')) newvar[:] = varobj[invar] else: # Dimensions newdim = objofile.createDimension('x', newdx) newdim = objofile.createDimension('y', newdy) newdim = objofile.createDimension('time', None) # var dimensions newvar = objofile.createVariable('lon', 'f8', ('y', 'x')) if Ndimshare == 2: Vals = lonobj[:] else: Vals = lonobj[0,:,:] for it in range(varobj.shape[0]): newvar[:] = Vals[inside].reshape(newdy,newdx) newattr = basicvardef(newvar, 'lon', 'longitude', 'degrees west_east') newvar = objofile.createVariable('lat', 'f8', ('y', 'x')) newattr = basicvardef(newvar, 'lat', 'latitude', 'degrees north_south') if Ndimshare == 2: Vals = latobj[:] else: Vals = latobj[0,:,:] for it in range(varobj.shape[0]): newvar[:] = Vals[inside].reshape(newdy,newdx) newvar = objofile.createVariable('time', 'f8', ('time')) if objfile.variables.has_key('time'): otime = objfile.variables['time'] timevals = otime[:] if searchInlist(otime.ncattrs(),'units'): tunits = otime.getncattr['units'] else: tunits = 'unkown' else: timevals = np.arange(varobj.shape[0]) tunits = 'unkown' newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = timevals # variable newvar = objofile.createVariable(varn, 'f4', ('time', 'y', 'x')) for it in range(varobj.shape[0]): valsvar = varobj[it,:,:] newvar[it,:,:] = valsvar[inside] if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = varn if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = varn if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = 'unkown' newattr = basicvardef(newvar, vsname, vlname, vunits) # global attributes objofile.setncattr('author', 'L. Fita') objofile.setncattr('institution', 'Laboratire de Meteorologie Dynamique') objofile.setncattr('university', 'Pierre Marie Curie - Jussieu') objofile.setncattr('center', 'Centre National de Recherches Scientifiques') objofile.setncattr('city', 'Paris') objofile.setncattr('country', 'France') objofile.setncattr('script', 'nc_var_tools.py') objofile.setncattr('function', 'sellonlatbox') objofile.setncattr('version', '1.0') objfile.close() objofile.sync() objofile.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return #sellonlatbox('h', '/san0/lflmd/DATA/SAFRAN/ForcPRCP_france_SAFRAN_8Km_1hour_1996080100_1997073123_V1_01.nc', 'RR') #sellonlatbox('lon,lat,2.,42.5,5.,45.', '/home/lluis/DATA/SAFRAN/safran_RR_1996.nc', 'RR') #sellonlatbox('lon,lat,2.,42.5,5.,45.', '/san0/lflmd/DATA/SAFRAN/ForcPRCP_france_SAFRAN_8Km_1hour_1996080100_1997073123_V1_01.nc', 'product') #sellonlatbox('XLONG,XLAT,2.,42.5,5.,45.', '/san0/lflmd/etudes/WRF_LMDZ/WL_HyMeX/Cevennes96/control/wrfout/wrfout_d01_1996-09-17_00:00:00', 'RAINC') def compute_deaccum(values, ncfile, varname): """ Function to compute deaccum: deaccumulation of a variable (VAR[t+1]-VAR[t]) values= [timedimname] name of the time dimension ncfile= netCDF file to use varname= variable to deaccumulate """ fname='compute_deaccum' ofile = 'deaccumulated_' + varname + '.nc' if values == 'h': print fname + '_____________________________________________________________' print compute_deacccum.__doc__ quit() objnc = NetCDFFile(ncfile, 'r') if not objnc.variables.has_key(varname): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varname + '" !!!!' quit(-1) var1obj = objnc.variables[varname] vardims = var1obj.dimensions Ndimvar = len(vardims) vdim = [] deacdims = [] # Searcing for time dimension for idd in range(Ndimvar): vdim.append(slice(0,var1obj.shape[idd])) deacdims.append(var1obj.shape[idd]) if vardims[idd] == values: dimtid = idd dimt = var1obj.shape[dimtid] print 'Number of time-steps: ',dimt,' id dimt: ',dimtid deacdims[dimtid] = dimt varvals = np.zeros(tuple(deacdims), dtype=np.float) # Slicing for the deaccumulation for it in range(dimt): vals1 = list(vdim) vals2 = list(vdim) vals1[dimtid] = it if it > 0: vals2[dimtid] = it - 1 else: vals2[dimtid] = it var1 = var1obj[tuple(vals1)] var2 = var1obj[tuple(vals2)] varvals[tuple(vals1)] = var1-var2 # Creation of the new file objofile = NetCDFFile(ofile, 'w') # Dimensions fdims = objnc.dimensions for dimn in fdims: dimobj = objnc.dimensions[dimn] if dimobj.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(dimobj) newdim = objofile.createDimension(dimn, dimsize) # Variables fvars = objnc.variables for varn in fvars: varobj = objnc.variables[varn] if varn == varname: newvarn = 'deaccum' + varn else: newvarn = varn newvar = objofile.createVariable(newvarn, varobj.dtype, varobj.dimensions) for attrn in varobj.ncattrs(): attrval = varobj.getncattr(attrn) newattr = newvar.setncattr(attrn, attrval) if varn == varname: newvar[:] = varvals else: newvar[:] = varobj[:] # Global attributes for attrn in objnc.ncattrs(): attrval = objnc.getncattr(attrn) newattr = objofile.setncattr(attrn, attrval) objnc.close() objofile.sync() objofile.close() print ' ' + fname + ' success written of "' + ofile + '" !!!!!' return #compute_deaccum('time', 'control/sellonlatbox_PRECIP_dayaccum.nc', 'PRECIP') def compute_opersvarsfiles(values, varname): """ Function to compute opersvarfiles: operation of variables from different files (OPER1.FILE1_VAR1 OPER2.FILE2_VAR2), operations are going to be secuentially made values= [dimvons]@[operfilevars] [dimnvons]: [dimo1]|[dimo2]|[...|[dimoN]] '|' seaprated list of names with the variables which contain the values of the dimensions of the compute variable operfilevars: [oper1]|[file1]|[varN],[...,[operK]|[fileM]|[varN]] ',' separated list of triplets of [operation], [file], [variable name to use] * [oper]: operations: add,sub,mul,div,pot varname= name to the final variable """ # import numpy.ma as ma fname='compute_opersvarsfiles' if values == 'h': print fname + '_____________________________________________________________' print compute_opersvarsfiles.__doc__ quit() dimovars = values.split('@')[0].split('|') filevars = values.split('@')[1].split(',') ofile = 'opersvarsfiles_' + varname + '.nc' Nfilevars = len(filevars) operation = 'operation: ' newunits = '-' opervars = [] for ifv in range(Nfilevars): opern = filevars[ifv].split('|')[0] filen = filevars[ifv].split('|')[1] varn = filevars[ifv].split('|')[2] opervars.append(varn) if not os.path.isfile(filen): print errormsg print ' ' + fname + ' file: "' + filen + '" does not exist !!' quit(-1) objnc = NetCDFFile(filen, 'r') if not objnc.variables.has_key(varn): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varn + '" !!!!' quit(-1) varobj = objnc.variables[varn] varvals = varobj[:] # Creation of the operated variable if ifv == 0: vardims = varobj.dimensions vartype = varobj.dtype if opern == 'add' or opern == 'sub': newvarv = np.zeros(tuple(varobj.shape), dtype=np.float) elif opern == 'mul' or opern == 'div' or opern == 'pot': newvarv = np.ones(tuple(varobj.shape), dtype=np.float) else: print errormsg print ' ' + fname + ': operation "' + opern + '" is not ready !!!!' quit(-1) if searchInlist(varobj.ncattrs(), '_FillValue'): NOvalue = varobj.getncattr('_FillValue') print warnmsg print ' ' + fname + ': variable with a missing value:',NOvalue if opern == 'add': if searchInlist(varobj.ncattrs(), '_FillValue'): prevals = varvals.filled(0.) newvarv = newvarv + prevals else: newvarv = newvarv + varvals operation = operation + ' +' + varn newunits = variables_values(varname)[5] elif opern == 'sub': if searchInlist(varobj.ncattrs(), '_FillValue'): prevals = varvals.filled(0.) newvarv = newvarv - prevals else: newvarv = newvarv - varvals operation = operation + ' -' + varn newunits = variables_values(varname)[5] elif opern == 'mul': if searchInlist(varobj.ncattrs(), '_FillValue'): prevals = varvals.filled(1.) newvarv = newvarv * prevals else: newvarv = newvarv * varvals operation = operation + ' *' + varn newunits = variables_values(varname)[5] elif opern == 'div': if searchInlist(varobj.ncattrs(), '_FillValue'): prevals = varvals.filled(1.) newvarv = newvarv / prevals else: newvarv = newvarv / varvals operation = operation + ' /' + varn newunits = variables_values(varname)[5] elif opern == 'pot': if searchInlist(varobj.ncattrs(), '_FillValue'): prevals = varvals.filled(1.) newvarv = newvarv ** prevals else: newvarv = newvarv ** varvals operation = operation + ' **' + varn newunits = variables_values(varname)[5] # Creation of the new file objofile = NetCDFFile(ofile, 'w') # Dimensions for dimn in vardims: dimobj = objnc.dimensions[dimn] if dimobj.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(dimobj) newdim = objofile.createDimension(dimn, dimsize) # Variables for dimensions for varn in dimovars: varobj = objnc.variables[varn] newvar = objofile.createVariable(varn, varobj.dtype, varobj.dimensions) for attrn in varobj.ncattrs(): attrval = varobj.getncattr(attrn) newattr = newvar.setncattr(attrn, attrval) newvar[:] = varobj[:] newvar = objofile.createVariable(varname, vartype, vardims) newvar[:] = newvarv newattr = basicvardef(newvar, varname, operation, newunits) # Global attributes for attrn in objnc.ncattrs(): attrval = objnc.getncattr(attrn) newattr = objofile.setncattr(attrn, attrval) objnc.close() objofile.sync() objofile.close() print ' ' + fname + ' success written of "' + ofile + '" !!!!!' return #compute_opersvarsfiles('add|control/sellonlatbox_RAINC.nc|RAINC,add|control/sellonlatbox_RAINNC.nc|RAINNC', 'PRECIP') def compute_opervartimes(values, ncfile, varname): """ Function to compute opervartimes: operation of variable for a given sub-set of time-steps (OPER1.VAR[IT,ET]), operations are going to be secuentially made values= [oper]|[timen]|[intT] dimension [oper]: operations: add,sub,mul,div,pot [timen]: name of the dimension time [intT]: number time-steps to accumulate ncfile= netCDF file to use varname= variable to use """ fname='compute_opervartimes' ofile = 'opervartimes_' + varname + '.nc' if values == 'h': print fname + '_____________________________________________________________' print opervartimes.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ' file: "' + ncfile + '" does not exist !!' quit(-1) objnc = NetCDFFile(ncfile, 'r') if not objnc.variables.has_key(varname): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varname + '" !!!!' quit(-1) opern = values.split('|')[0] timen = values.split('|')[1] intT = int(values.split('|')[2]) varobj = objnc.variables[varname] vardims = varobj.dimensions vartype = varobj.dtype Ndims = len(vardims) newdimns = vardims newdims = np.zeros((Ndims), dtype=int) newslice = [] newunits = '-' idim = 0 for dimn in vardims: if dimn == timen: dimtid = idim newslice.append(0) origdimt = varobj.shape[dimtid] newdims[idim] = len(np.arange(origdimt)[intT/2::intT]) else: newdims[idim] = varobj.shape[idim] newslice.append(slice(0,varobj.shape[idim])) idim = idim + 1 print 'dimtid: ',dimtid if opern == 'add' or opern == 'sub': newvarv = np.zeros(tuple(newdims), dtype=np.float) elif opern == 'mul' or opern == 'div' or opern == 'pot': newvarv = np.ones(tuple(newdims), dtype=np.float) else: print errormsg print ' ' + fname + ': operation "' + opern + '" is not ready !!!!' quit(-1) # Creation of the accumulated operated variable idint = 0 itt = 0 nint = 0 for it in range(1,origdimt): tslice = list(newslice) tslice[dimtid] = it tnewvar = list(newslice) tnewvar[dimtid] = nint if opern == 'add': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] + varobj[tuple(tslice)] if searchInlist(varobj.ncattrs(), 'units') and newunits != '-': newunits = varobj.getncattr('units') elif opern == 'sub': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] - varobj[tuple(tslice)] if searchInlist(varobj.ncattrs(), 'units') and newunits != '-': newunits = varobj.getncattr('units') elif opern == 'mul': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] * varobj[tuple(tslice)] elif opern == 'div': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] / varobj[tuple(tslice)] elif opern == 'pot': newvarv[tuple(tnewvar)] = newvarv[tuple(tnewvar)] **varobj[tuple(tslice)] it = it + 1 itt = itt + 1 if itt == intT: nint = nint + 1 itt = 0 # Creation of the new file objofile = NetCDFFile(ofile, 'w') # Dimensions for dimn in newdimns: dimobj = objnc.dimensions[dimn] if dimobj.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(dimobj) if dimn == timen: if np.mod(varobj.shape[dimtid],intT) != 0: dimsize = varobj.shape[dimtid]/intT + 1 newdim = objofile.createDimension(dimn, dimsize) newdim = objofile.createDimension('bnds', 2) # Variables fvars = objnc.variables for varn in fvars: if varn != varname: varobj = objnc.variables[varn] if varn != timen: if not searchInlist(varobj.dimensions,timen): newvar = objofile.createVariable(varn, varobj.dtype, \ varobj.dimensions) for attrn in varobj.ncattrs(): attrval = varobj.getncattr(attrn) newattr = newvar.setncattr(attrn, attrval) newvar[:] = varobj[:] else: newvar = objofile.createVariable(varn, varobj.dtype, ('time')) basicvardef(newvar, 'time', 'time steps','-') times = np.arange(origdimt)[intT/2::intT] newvar[:] = times newdimt = newvar.shape[0] newattr = newvar.setncattr('bounds','time_bnds') newvar = objofile.createVariable('time_bnds', 'i4', ('time', 'bnds')) basicvardef(newvar, 'time_bnds', 'time steps bnds of accumulation','-') for it in range(newdimt): newvar[it,0] = (it - 1)*intT newvar[it,1] = (it)*intT-1 newvar = objofile.createVariable('ac' + varname, vartype, vardims) newvar[:] = newvarv newattr = basicvardef(newvar, 'ac' + varname, 'accumulated ' + opern + \ ' values' , newunits) # Global attributes for attrn in objnc.ncattrs(): attrval = objnc.getncattr(attrn) newattr = objofile.setncattr(attrn, attrval) objnc.close() objofile.sync() objofile.close() print ' ' + fname + ' success written of "' + ofile + '" !!!!!' return #compute_opervartimes('add|time|24', 'obs/PRCP.nc', 'product') # def compute_opervaralltime(values, ncfile, varname): """ Function to compute opervaralltime: operation of variable successible allover the time-steps (OPER1.VAR), operations are going to be secuentially made values= [oper]|[timen] dimension [oper]: operations: add,sub,mul,div,pot [timen]: name of the dimension time ncfile= netCDF file to use varname= variable to use """ fname='compute_opervarlltime' ofile = 'opervaralltime_' + varname + '.nc' if values == 'h': print fname + '_____________________________________________________________' print compute_opervaralltime.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ' file: "' + ncfile + '" does not exist !!' quit(-1) objnc = NetCDFFile(ncfile, 'r') if not objnc.variables.has_key(varname): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varname + '" !!!!' quit(-1) opern = values.split('|')[0] timen = values.split('|')[1] varobj = objnc.variables[varname] vardims = varobj.dimensions vartype = varobj.dtype Ndims = len(vardims) newdimns = vardims newdims = np.zeros((Ndims), dtype=int) newslice = [] prevslice = [] newunits = '-' idim = 0 for dimn in vardims: if dimn == timen: dimtid = idim newslice.append(0) origdimt = varobj.shape[dimtid] newdims[idim] = origdimt else: newdims[idim] = varobj.shape[idim] newslice.append(slice(0,varobj.shape[idim])) idim = idim + 1 print 'dimtid: ',dimtid if opern == 'add' or opern == 'sub': newvarv = np.zeros(tuple(newdims), dtype=np.float) elif opern == 'mul' or opern == 'div' or opern == 'pot': newvarv = np.ones(tuple(newdims), dtype=np.float) else: print errormsg print ' ' + fname + ': operation "' + opern + '" is not ready !!!!' quit(-1) # Creation of the accumulated operated variable idint = 0 for it in range(1,origdimt): tslice = list(newslice) tslice[dimtid] = it prevslice = list(newslice) prevslice[dimtid] = it - 1 if opern == 'add': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] + varobj[tuple(tslice)] if searchInlist(varobj.ncattrs(), 'units') and newunits != '-': newunits = varobj.getncattr('units') elif opern == 'sub': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] - varobj[tuple(tslice)] if searchInlist(varobj.ncattrs(), 'units') and newunits != '-': newunits = varobj.getncattr('units') elif opern == 'mul': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] * varobj[tuple(tslice)] elif opern == 'div': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] / varobj[tuple(tslice)] elif opern == 'pot': newvarv[tuple(tslice)] = newvarv[tuple(prevslice)] **varobj[tuple(tslice)] it = it + 1 # Creation of the new file objofile = NetCDFFile(ofile, 'w') # Dimensions for dimn in newdimns: dimobj = objnc.dimensions[dimn] if dimobj.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(dimobj) newdim = objofile.createDimension(dimn, dimsize) newdim = objofile.createDimension('bnds', 2) # Variables fvars = objnc.variables for varn in fvars: if varn != varname: varobj = objnc.variables[varn] if varn != timen: if not searchInlist(varobj.dimensions,timen): newvar = objofile.createVariable(varn, varobj.dtype, \ varobj.dimensions) for attrn in varobj.ncattrs(): attrval = varobj.getncattr(attrn) newattr = newvar.setncattr(attrn, attrval) newvar[:] = varobj[:] else: newvar = objofile.createVariable(varn, varobj.dtype, ('time')) basicvardef(newvar, 'time', 'time steps','-') timevals = varobj[:] newvar[:] = timevals newdimt = newvar.shape[0] newattr = newvar.setncattr('bounds','time_bnds') newvar = objofile.createVariable('time_bnds', 'i4', ('bnds')) basicvardef(newvar, 'time_bnds', 'time steps bnds of accumulation','-') newvar[0] = timevals[0] newvar[1] = timevals[origdimt-1] newvar = objofile.createVariable('ac' + varname, vartype, vardims) newvar[:] = newvarv newattr = basicvardef(newvar, 'ac' + varname, 'accumulated ' + opern + \ ' values' , newunits) # Global attributes for attrn in objnc.ncattrs(): attrval = objnc.getncattr(attrn) newattr = objofile.setncattr(attrn, attrval) objnc.close() objofile.sync() objofile.close() print ' ' + fname + ' success written of "' + ofile + '" !!!!!' return #compute_opervaralltime('add|time', 'obs/PRCP.nc', 'product') def retype(val, vtype): """ Function to transform a value to a given type retype(val, vtype) [val]= value [vtype]= type to transform >>> retype(0, type(True)) False """ fname = 'retype' if val == 'h': print fname + '_____________________________________________________________' print retype.__doc__ quit() if vtype == type(int(1)): newval = int(val) elif vtype == type(float(1)): newval = float(val) # elif vtype == type(float32(1)): # newval = float32(val) elif vtype == type(np.int(1)): typeinf = np.iinfo(np.int) if (abs(np.float64(val)) > typeinf.max): print warnmsg print ' ' + fname + ': value to transform ', val,' overpasses type:', \ vtype,' limits!!' print ' Changing value to kind largest allowed value:', typeinf.max newval = abs(np.float64(val))/np.float64(val) * np.int(typeinf.max) else: newval = np.int(val) elif vtype == type(np.int16(1)): typeinf = np.iinfo(np.int16) if (abs(np.float64(val)) > typeinf.max): print warnmsg print ' ' + fname + ': value to transform ', val,' overpasses type:', \ vtype,' limits!!' print ' Changing value to kind largest allowed value:', typeinf.max newval = abs(np.float64(val))/np.float64(val) * np.int16(typeinf.max) else: newval = np.int16(val) elif vtype == type(np.int32(1)): typeinf = np.iinfo(np.int32) if (np.abs(np.float64(val)) > typeinf.max): print warnmsg print ' ' + fname + ': value to transform ', val,' overpasses type:', \ vtype,' limits!!' print ' Changing value to kind largest allowed value:', typeinf.max newval = abs(np.float64(val))/np.float64(val) * np.int32(typeinf.max) else: newval = np.int32(np.float64(val)) elif vtype == type(np.int64(1)): newval = np.int64(val) elif vtype == type(np.float(1)): newval = np.float(val) # elif vtype == type(np.float16(1)): # newval = np.float16(val) elif vtype == type(np.float32(1)): newval = np.float32(val) elif vtype == type(np.float64(1)): newval = np.float64(val) elif vtype == type(True): if val == 0: newval = False else: newval = True elif vtype == '|S1': newval = str(val) else: print errormsg print ' ' + fname + ': variable type "', vtype, '" not ready!!!' quit(-1) return newval def setvar_asciivalues(values, ncfile, varname): """ Function to set de values of a variable with an ASCII file (common Fortran-like format) values= asciifile: ASCII file with a Fortran-like structure [dim1]_1 [[dim2]_1 ... [dim2]_M] ... [dim1]_N NOTE: for character values, use '!' for space ncfile= netCDF file to use varname= variable to use """ fname='setvar_asciivalues' if values == 'h': print fname + '_____________________________________________________________' print setvar_asciivalues.__doc__ quit() if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ' file: "' + ncfile + '" does not exist !!' quit(-1) if not os.path.isfile(values): print errormsg print ' ' + fname + ' ASCII file: "' + values + '" does not exist !!' quit(-1) objnc = NetCDFFile(ncfile, 'a') if not objnc.variables.has_key(varname): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + varname + '" !!!!' quit(-1) objvar = objnc.variables[varname] vardims = objvar.shape vartype = objvar.dtype Ndims = len(vardims) objasciif = open(values, 'r') values = np.zeros((objvar.shape), dtype=vartype) it = 0 if Ndims == 1: valfinal = np.zeros((1), dtype=vartype) for line in objasciif: val = (line.replace('\n','').replace(' ','').replace('\t','')) if vartype != '|S1': valfinal[0] = retype(val, vartype) if it <= values.shape[0]-1: values[it] = retype(valfinal[0], vartype) it = it +1 else: for i1 in range(len(val)): values[it] = val.replace('!',' ') elif Ndims == 2: iline=0 for line in objasciif: vals = line.replace('\n','').replace('\t','').split(' ') if vartype != '|S1': if len(vals) != values.shape[1]: print errormsg print ' ' + fname + ': given: ',len(vals),' but variable ' + \ 'reaquires: ',values.shape[1],'!!!' exit(-1) for i1 in range(len(vals)): values[iline,i1] = retype(vals[i1], vartype) else: for i1 in range(len(vals)): values[iline,i1] = vals[i1].replace('!',' ') iline=iline+1 else: print errormsg print ' ' + fname + ': number of dimensions',Ndims,'not ready !!!!' quit(-1) objasciif.close() objvar[:] = values objnc.sync() objnc.close() #setvar_asciivalues('obstimes.inf', 'obs/re_project_accumulated-1h_PRCP.nc', 'time') class stats_space2D(object): """spatial statistics for a 2D file: vals= variable ro check (assuming var[t,dy,dx]) self.minv[t]: spatial minimum at each time-step self.maxv[t]: spatial maximum at each time-step self.meanv[t]: spatial mean at each time-step self.mean2v[t]: spatial quadratic mean at each time-step self.stdv[t]: spatial standard deviation at each time-step self.anomv[t]: spatial anomaly from the whole mean at each time-step """ def __init__(self, vals): from scipy import stats as sts fname = 'stats_space2D' if vals1 == 'h': print fname + '_____________________________________________________________' print stats_space2Dvars.__doc__ quit() if vals1 is None: self.minv = None self.maxv = None self.meanv = None self.mean2v = None self.stdv = None else: dimt = vals.shape[0] stats=np.zeros((dimt,5), dtype=np.float) absmean = np.mean(vals) for it in range(dimt): stats[it,0]=np.min(vals[it,:,:]) stats[it,1]=np.max(vals[it,:,:]) stats[it,2]=np.mean(vals[it,:,:]) stats[it,3]=np.mean(vals[it,:,:]*vals[it,:,:]) stats[it,4]=absmean - stats[it,2] stats = np.where(stats > 0.1*fillValue, fillValue, stats) stats = np.where(stats is nan, fillValue, stats) stats = np.where(stats is np.inf, fillValue, stats) stats = np.where(stats is None, fillValue, stats) self.minv=stats[:,0] self.maxv=stats[:,1] self.meanv=stats[:,2] self.mean2v=stats[:,3] self.stdv=np.sqrt(stats[:,3]-stats[:,2]*stats[:,2]) self.anomv=stats[:,4] class stats_space2Dvars(object): """spatial statistics beween 2 2D files: valsA = variable 1 (assuming var[t,dy,dx]) valsB = variable 2 (assuming var[t,dy,dx]) self.min[t,var], self.max[t,var], self.mean[t,var], self.mean2[t,var], self.std[t,var] [var] = var1+var2[v1Av2], var1-var2[v1Sv2], var1/var2[v1Dv2], var1*var2[v1Pv2] self.mae=mean[t,abs(var1-var2)] self.correlation=correlation[var1,var2] (and p-value) self.bias=[t,mean(var1)-mean(var2)] self.meancorrelation=correlation[mean(var1),mean(var2)] (and p-value) """ def __init__(self, vals1, vals2): from scipy import stats as sts fname = 'stats_space2Dvars' if vals1 == 'h': print fname + '_____________________________________________________________' print stats_space2Dvars.__doc__ quit() if vals1 is None: self.minv1Av2 = None self.maxv1Av2 = None self.meanv1Av2 = None self.mean2v1Av2 = None self.stdv1Av2 = None self.minv1Sv2 = None self.maxv1Sv2 = None self.meanv1Sv2 = None self.mean2v1Sv2 = None self.stdv1Sv2 = None self.minv1Dv2 = None self.maxv1Dv2 = None self.meanv1Dv2 = None self.mean2v1Dv2 = None self.stdv1Dv2 = None self.minv1Pv2 = None self.maxv1Pv2 = None self.meanv1Pv2 = None self.mean2v1Pv2 = None self.stdv1Pv2 = None self.mae = None self.corr = None else: dimt = vals1.shape[0] stats=np.zeros((dimt,26), dtype=np.float) meanvals1 = np.zeros((dimt), dtype=np.float) meanvals2 = np.zeros((dimt), dtype=np.float) for it in range(dimt): v1 = vals1[it,:,:].flatten() v2 = vals2[it,:,:].flatten() # add vs = v1 + v2 stats[it,0] = np.min(vs) stats[it,1] = np.max(vs) stats[it,2] = np.mean(vs) stats[it,3] = np.mean(vs*vs) stats[it,4] = np.sqrt(stats[it,3] - stats[it,2]*stats[it,2]) # sub stats[it,20] = np.mean(np.abs(v1-v2)) vs = v1 - v2 stats[it,5] = np.min(vs) stats[it,6] = np.max(vs) stats[it,7] = np.mean(vs) stats[it,8] = np.mean(vs*vs) stats[it,9] = np.sqrt(stats[it,8] - stats[it,7]*stats[it,7]) # mul vs = v1 * v2 stats[it,10] = np.min(vs) stats[it,11] = np.max(vs) stats[it,12] = np.mean(vs) stats[it,13] = np.mean(vs*vs) stats[it,14] = np.sqrt(stats[it,13] - stats[it,12]*stats[it,12]) # div vs = v1 / v2 stats[it,15] = np.min(vs) stats[it,16] = np.max(vs) stats[it,17] = np.mean(vs) stats[it,18] = np.mean(vs*vs) stats[it,19] = np.sqrt(stats[it,18] - stats[it,17]*stats[it,17]) # corr stats[it,21], stats[it,22] = mask_pearsonr(v1, v2) # Mean values meanvals1[it] = np.mean(v1) meanvals2[it] = np.mean(v2) stats[it,23] = meanvals1[it] - meanvals2[it] stats = np.where(stats > 0.1*fillValue, fillValue, stats) stats = np.where(stats is np.nan, fillValue, stats) stats = np.where(stats is np.inf, fillValue, stats) stats = np.where(stats is None, fillValue, stats) self.minv1Av2 = stats[:,0] self.maxv1Av2 = stats[:,1] self.meanv1Av2 = stats[:,2] self.meanv21Av2 = stats[:,3] self.stdv1Av2 = stats[:,4] self.minv1Sv2 = stats[:,5] self.maxv1Sv2 = stats[:,6] self.meanv1Sv2 = stats[:,7] self.meanv21Sv2 = stats[:,8] self.stdv1Sv2 = stats[:,9] self.minv1Pv2 = stats[:,10] self.maxv1Pv2 = stats[:,11] self.meanv1Pv2 = stats[:,12] self.meanv21Pv2 = stats[:,13] self.stdv1Pv2 = stats[:,14] self.minv1Dv2 = stats[:,15] self.maxv1Dv2 = stats[:,16] self.meanv1Dv2 = stats[:,17] self.meanv21Dv2 = stats[:,18] self.stdv1Dv2 = stats[:,19] self.mae = stats[:,20] self.corr = stats[:,21] self.p_value = stats[:,22] self.bias = stats[:,23] self.meancorr, self.meanp_value = sts.pearsonr(meanvals1, meanvals2) class stats_time2D(object): """temporal statistics for a 2D file: vals= variable ro check (assuming var[t,dy,dx]) self.minv[t]: temporal minimum at each grid point self.maxv[t]: temporal maximum at each grid point self.meanv[t]: temporal mean at each grid point self.mean2v[t]: temporal quadratic mean at each grid point self.stdv[t]: temporal standard deviation at each grid point self.anomv[t]: temporal anomaly from the whole mean at each grid point """ def __init__(self, vals): from scipy import stats as sts fname = 'stats_time2D' if vals1 == 'h': print fname + '_____________________________________________________________' print stats_time2Dvars.__doc__ quit() if vals1 is None: self.minv = None self.maxv = None self.meanv = None self.mean2v = None self.stdv = None else: dimx = vals.shape[2] dimy = vals.shape[1] stats=np.zeros((dimy,dimx,5), dtype=np.float) absmean = np.mean(vals,axis=0) stats[:,:,0]=np.min(vals, axis=0) stats[:,:,1]=np.max(vals, axis=0) stats[:,:,2]=np.mean(vals, axis=0) stats[:,:,3]=np.mean(vals*vals, axis=0) stats[:,:,4]=absmean - stats[:,:,2] stats = np.where(stats > 0.1*fillValue, fillValue, stats) stats = np.where(stats is np.nan, fillValue, stats) stats = np.where(stats is np.inf, fillValue, stats) stats = np.where(stats is None, fillValue, stats) self.minv=stats[:,:,0] self.maxv=stats[:,:,1] self.meanv=stats[:,:,2] self.mean2v=stats[:,:,3] self.stdv=np.sqrt(stats[:,:,3]-stats[:,:,2]*stats[:,:,2]) self.anomv=stats[:,:,4] class stats_time2Dvars(object): """temporal statistics beween 2 2D files: valsA = variable 1 (assuming var[t,dy,dx]) valsB = variable 2 (assuming var[t,dy,dx]) self.min[dy,dx,var], self.max[dy,dx,var], self.mean[dy,dx,var], self.mean2[dy,dx,var], self.std[dy,dx,var] [var] = var1+var2[v1Av2], var1-var2[v1Sv2], var1/var2[v1Dv2], var1*var2[v1Pv2] self.mae=mean[dy,dx,abs(var1-var2)] self.correlation=correlation[var1,var2] (and p-value) self.bias=[dy,dx,mean(var1)-mean(var2)] self.meancorrelation=correlation[mean(var1),mean(var2)] (and p-value) """ def __init__(self, vals1, vals2): from scipy import stats as sts fname = 'stats_time2Dvars' if vals1 == 'h': print fname + '_____________________________________________________________' print stats_time2Dvars.__doc__ quit() if vals1 is None: self.minv1Av2 = None self.maxv1Av2 = None self.meanv1Av2 = None self.mean2v1Av2 = None self.stdv1Av2 = None self.minv1Sv2 = None self.maxv1Sv2 = None self.meanv1Sv2 = None self.mean2v1Sv2 = None self.stdv1Sv2 = None self.minv1Dv2 = None self.maxv1Dv2 = None self.meanv1Dv2 = None self.mean2v1Dv2 = None self.stdv1Dv2 = None self.minv1Pv2 = None self.maxv1Pv2 = None self.meanv1Pv2 = None self.mean2v1Pv2 = None self.stdv1Pv2 = None self.mae = None self.corr = None else: dimx = vals1.shape[1] dimy = vals1.shape[1] stats=np.zeros((dimy,dimx,24), dtype=np.float) meanvals1 = np.zeros((dimy,dimx), dtype=np.float) meanvals2 = np.zeros((dimy,dimx), dtype=np.float) # add vs = vals1 + vals2 stats[:,:,0] = np.min(vs,axis=0) stats[:,:,1] = np.max(vs,axis=0) stats[:,:,2] = np.mean(vs,axis=0) stats[:,:,3] = np.mean(vs*vs,axis=0) stats[:,:,4] = np.sqrt(stats[:,:,3] - stats[:,:,2]*stats[:,:,2]) # sub stats[:,:,20] = np.mean(np.abs(vals1-vals2), axis=0) vs = vals1 - vals2 stats[:,:,5] = np.min(vs,axis=0) stats[:,:,6] = np.max(vs,axis=0) stats[:,:,7] = np.mean(vs,axis=0) stats[:,:,8] = np.mean(vs*vs,axis=0) stats[:,:,9] = np.sqrt(stats[:,:,8] - stats[:,:,7]*stats[:,:,7]) # mul vs = vals1 * vals2 stats[:,:,10] = np.min(vs,axis=0) stats[:,:,11] = np.max(vs,axis=0) stats[:,:,12] = np.mean(vs,axis=0) stats[:,:,13] = np.mean(vs*vs,axis=0) stats[:,:,14] = np.sqrt(stats[:,:,13] - stats[:,:,12]*stats[:,:,12]) # div vs = vals1 / vals2 stats[:,:,15] = np.min(vs,axis=0) stats[:,:,16] = np.max(vs,axis=0) stats[:,:,17] = np.mean(vs,axis=0) stats[:,:,18] = np.mean(vs*vs,axis=0) stats[:,:,19] = np.sqrt(stats[:,:,18] - stats[:,:,17]*stats[:,:,17]) # Mean values meanvals1[:,:] = np.mean(vals1,axis=0) meanvals2[:,:] = np.mean(vals2,axis=0) stats[:,:,23] = meanvals1[:,:] - meanvals2[:,:] # corr self.meancorr, self.meanp_value = sts.pearsonr(meanvals1.flatten(), meanvals2.flatten()) for j in range(dimy): for i in range(dimx): stats[j,i,21], stats[j,i,22] = sts.pearsonr(vals1[:,j,i], vals2[:,j,i]) stats = np.where(stats > 0.1*fillValue, fillValue, stats) stats = np.where(stats is np.nan, fillValue, stats) stats = np.where(stats is np.inf, fillValue, stats) stats = np.where(stats is None, fillValue, stats) self.minv1Av2 = stats[:,:,0] self.maxv1Av2 = stats[:,:,1] self.meanv1Av2 = stats[:,:,2] self.meanv21Av2 = stats[:,:,3] self.stdv1Av2 = stats[:,:,4] self.minv1Sv2 = stats[:,:,5] self.maxv1Sv2 = stats[:,:,6] self.meanv1Sv2 = stats[:,:,7] self.meanv21Sv2 = stats[:,:,8] self.stdv1Sv2 = stats[:,:,9] self.minv1Pv2 = stats[:,:,10] self.maxv1Pv2 = stats[:,:,11] self.meanv1Pv2 = stats[:,:,12] self.meanv21Pv2 = stats[:,:,13] self.stdv1Pv2 = stats[:,:,14] self.minv1Dv2 = stats[:,:,15] self.maxv1Dv2 = stats[:,:,16] self.meanv1Dv2 = stats[:,:,17] self.meanv21Dv2 = stats[:,:,18] self.stdv1Dv2 = stats[:,:,19] self.mae = stats[:,:,20] self.corr = stats[:,:,21] self.p_value = stats[:,:,22] self.bias = stats[:,:,23] #vals1 = np.arange(27).reshape(3,3,3)*1. #vals2 = np.arange(1,28).reshape(3,3,3)*1. #printing_class(stats_time2Dvars(vals1,vals2)) def statcompare_files(values): """ Python script to statistically compare two different files values= [file1]:[var1],[file2][var2] [file1/2]: files to compare any diferent would be [file1]-[file2] [var1/2]: variable in file1/2 to compare """ fname = 'statcompare_files' ofile='statcompare_files.nc' if values == 'h': print fname + '_____________________________________________________________' print statcompare_files.__doc__ quit() file1=values.split(',')[0].split(':')[0] var1=values.split(',')[0].split(':')[1] file2=values.split(',')[1].split(':')[0] var2=values.split(',')[1].split(':')[1] if not os.path.isfile(file1): print errormsg print ' ' + fname + ' file: "' + file1 + '" does not exist !!' quit(-1) objnc1 = NetCDFFile(file1, 'r') if not objnc1.variables.has_key(var1): print errormsg print ' ' + fname + ': netCDF file "' + file1 + \ '" does not have variable "' + var1 + '" !!!!' quit(-1) if not os.path.isfile(file2): print errormsg print ' ' + fname + ' file: "' + file2 + '" does not exist !!' quit(-1) objnc2 = NetCDFFile(file2, 'r') if not objnc2.variables.has_key(var2): print errormsg print ' ' + fname + ': netCDF file "' + file2 + \ '" does not have variable "' + var2 + '" !!!!' quit(-1) objvar1 = objnc1.variables[var1] objvar2 = objnc2.variables[var2] Ndim1 = len(objvar1.shape) Ndim2 = len(objvar2.shape) if Ndim1 != Ndim2: print errormsg print ' ' + fname + ' variable: "' + var1 + '" from file "' + file1 + \ '" does not have the same size:' , Ndim1, 'as variable: "' + var2 + \ '" from file "' + file2 + '": ',Ndim2 quit(-1) dims1 = objvar1.shape dims2 = objvar2.shape for iid in range(Ndim1): if dims1[iid] != dims2[iid]: print errormsg print ' ' + fname + ' shape:',iid,'is different between variables!!!' print ' dim_var1:', dims1[iid], 'dim_var2:', dims2[iid] print ' var1 shape:', dims1 print ' var2 shape:', dims2 quit(-1) varvals1 = objvar1[:] varvals2 = objvar2[:] if Ndim1 == 3: timecompare = stats_time2Dvars(varvals1,varvals2) spacecompare = stats_space2Dvars(varvals1,varvals2) dimt = varvals1.shape[0] dimy = varvals1.shape[1] dimx = varvals1.shape[2] # print 'bias std.dev. ___________________________________' # for it in range(dimt): # print it,':',spacecompare.bias[it],spacecompare.stdv1Sv2[it] # print 'global spatial correlation (% significance)______' # for it in range(dimt): # print it, ':', spacecompare.corr[it], '(', \ # (1.-spacecompare.p_value[it])*100.,')' # print 'spatial means temporal correlation:',spacecompare.meancorr,'(', \ # (1.-spacecompare.meanp_value)*100.,' % significance)' # print 'temporal means temporal correlation:',timecompare.meancorr,'(', \ # (1.-timecompare.meanp_value)*100.,' % significance)' # print 'temporal bias map________________________________' # print timecompare.meanv1Sv2 # print 'temporal correlation map_________________________' # print timeecompare.corr else: print errormsg print ' ' + fname + ' numbe of dimensions:' + Ndim1 + ' not ready!!!' quit(-1) # Creation of results file ## objofile = NetCDFFile(ofile, 'w') # Dimensions for dimn in objnc1.dimensions: dimobj = objnc1.dimensions[dimn] if dimobj.isunlimited(): print ' ' + fname + ': Unlimited dimension ' dimsize = None else: dimsize = len(dimobj) newdim = objofile.createDimension(dimn, dimsize) # Variables fvars = objnc1.variables for varn in fvars: if varn != var1: varobj = objnc1.variables[varn] if varn != var1: newvar = objofile.createVariable(varn, varobj.dtype, \ varobj.dimensions) for attrn in varobj.ncattrs(): attrval = varobj.getncattr(attrn) newattr = newvar.setncattr(attrn, attrval) newvar[:] = varobj[:] # Statistical variables vartype = objvar1.dtype varunits = objvar2.getncattr('units') print 'vartype=',vartype # temporal bias newvar = objofile.createVariable('t_bias_'+var1+'_'+var2, vartype, ('y', 'x'), \ fill_value = fillValue) newvar[:] = timecompare.bias newattr = basicvardef(newvar, 't_bias_'+var1+'_'+var2, 'temporal bias between ' +\ var1 + '&' + var2, varunits) # temporal Standard deviation newvar = objofile.createVariable('t_stddev_'+var1+'_'+var2, vartype, ('y', 'x'), \ fill_value = fillValue) newvar[:] = timecompare.stdv1Sv2 newattr = basicvardef(newvar, 't_stddev_'+var1+'_'+var2, \ 'temporal standard deviation between '+ var1 + '&' + var2, varunits) # space bias newvar = objofile.createVariable('s_bias_'+var1+'_'+var2, vartype, ('time'), \ fill_value = fillValue) newvar[:] = spacecompare.bias newattr = basicvardef(newvar, 's_bias_'+var1+'_'+var2, 'spatial bias between ' + \ var1 + '&' + var2, varunits) # space Standard deviation newvar = objofile.createVariable('s_stddev_'+var1+'_'+var2, vartype, ('time'), \ fill_value = fillValue) newvar[:] = spacecompare.stdv1Sv2 newattr = basicvardef(newvar, 's_stddev_'+var1+'_'+var2, \ 'spatial standard deviation between '+ var1 + '&' + var2, varunits) # t Correlation map newvar = objofile.createVariable('t_corr_'+var1+'_'+var2, vartype, ('y','x'), \ fill_value = fillValue) newvar[:] = timecompare.corr newattr = basicvardef(newvar, 't_corr_'+var1+'_'+var2, \ 'temporal correlation between ' + var1 + '&' + var2, '-') newattr = newvar.setncattr('projection', 'lon lat') # t p-value map newvar = objofile.createVariable('t_p_value_'+var1+'_'+var2, vartype, ('y','x'), \ fill_value = fillValue) newvar[:] = timecompare.p_value newattr = basicvardef(newvar, 't_p_value_'+var1+'_'+var2, \ 'temporal p_value between ' + var1 + '&' + var2, '-') newattr = newvar.setncattr('projection', 'lon lat') # s Correlation map newvar = objofile.createVariable('s_corr_'+var1+'_'+var2, vartype, ('time'), \ fill_value = fillValue) newvar[:] = spacecompare.corr newattr = basicvardef(newvar, 's_corr_'+var1+'_'+var2, \ 'spatial correlation between ' + var1 + '&' + var2, '-') # s p-value map newvar = objofile.createVariable('s_p_value_'+var1+'_'+var2, vartype, ('time'), \ fill_value = fillValue) newvar[:] = spacecompare.p_value newattr = basicvardef(newvar, 's_p_value_'+var1+'_'+var2, \ 'spatial p_value between ' + var1 + '&' + var2, '-') # print 'spatial means temporal correlation:',spacecompare.meancorr,'(', \ # (1.-spacecompare.meanp_value)*100.,' % significance)' # t mean Correlation newvar = objofile.createVariable('t_mean_corr_'+var1+'_'+var2, 'f4') newvar[:] = timecompare.meancorr newattr = basicvardef(newvar, 't_mean_corr_'+var1+'_'+var2, \ 'time mean values correlation between ' + var1 + '&' + var2, '-') # t mean p-value newvar = objofile.createVariable('t_mean_p_value_'+var1+'_'+var2, 'f4') newvar[:] = timecompare.meanp_value newattr = basicvardef(newvar, 't_mean_p_value_'+var1+'_'+var2, \ 'time mean p_value between ' + var1 + '&' + var2, '-') # s mean Correlation newvar = objofile.createVariable('s_mean_corr_'+var1+'_'+var2, 'f4') newvar[:] = spacecompare.meancorr newattr = basicvardef(newvar, 's_mean_corr_'+var1+'_'+var2, \ 'space mean values correlation between ' + var1 + '&' + var2, '-') # s mean p-value newvar = objofile.createVariable('s_mean_p_value_'+var1+'_'+var2, 'f4') newvar[:] = spacecompare.meanp_value newattr = basicvardef(newvar, 's_mean_p_value_'+var1+'_'+var2, \ 'space mean p_value between ' + var1 + '&' + var2, '-') # Global attributes for attrn in objnc1.ncattrs(): attrval = objnc1.getncattr(attrn) newattr = objofile.setncattr(attrn, attrval) newattr = objofile.setncattr('statisitcs', 'variables retrieved from files ' + \ file1 + ' & ' + file2) objnc1.close() objnc2.close() objofile.sync() objofile.close() varsfillvalue = ['t_bias_', 't_stddev_', 's_bias_', 's_stddev_', 't_corr_', \ 't_p_value', 's_corr_', 's_p_value'] # for varn in varsfillvalue: # print varn # fill = varaddattrk('_FillValue|'+ str(fillValue) + '|R32', ofile, \ # varn+var1+'_'+var2) print ' ' + fname + ': successfull generation of file "' + ofile + '" !!!!!' return #statcompare_files('control/sellonlatbox_wss_17-20.nc:wss,obs/re_project_Vu_17-20.nc:wss') def sellonlatlevbox(values, ncfile, varn): """ Function to select a lotlan box and a given level from a data-set values= [lonName],[latName],[lonSW],[latSW],[lonNE],[latNE],[levi],[levf] [lonName]: name of the variable with the longitudes [latName]: name of the variable with the latitudes [levName]: name of the variable with the value of the levels [lonSW],[latSW]: longitude and latitude of the SW vertex of the box [lonNE],[latNE]: longitude and latitude of the NE vertex of the box [levI],[levF]: range of levels to retrieve ncfile= netCDF file varn= name of the variable """ fname = 'sellonlatlevbox' ofile = 'sellonlatlevbox_' + varn + '.nc' if values == 'h': print fname + '_____________________________________________________________' print sellonlatlevbox.__doc__ quit() lonn = values.split(',')[0] latn = values.split(',')[1] levn = values.split(',')[2] lonSW = np.float(values.split(',')[3]) latSW = np.float(values.split(',')[4]) lonNE = np.float(values.split(',')[5]) latNE = np.float(values.split(',')[6]) levI = int(values.split(',')[7]) levF = int(values.split(',')[8]) objfile = NetCDFFile(ncfile, 'r') lonobj = objfile.variables[lonn] latobj = objfile.variables[latn] levobj = objfile.variables[levn] lonv = lonobj[:] latv = latobj[:] levv = levobj[:] if not searchInlist(objfile.variables, varn): print errormsg print ' ' + fname + ': variable name "' + varn + '" is not in file "' + \ ncfile + '" !!!!!' quit(-1) varobj = objfile.variables[varn] Ndimslon = len(lonobj.shape) Ndimslat = len(latobj.shape) Ndimslev = len(levobj.shape) Ndimsvar = len(varobj.shape) # Looking for coincidence of dimensions samedim = [] for idv in range(Ndimsvar): for idl in range(Ndimslon): if varobj.dimensions[idv] == lonobj.dimensions[idl]: if not searchInlist(samedim,varobj.dimensions[idv]): samedim.append(varobj.dimensions[idv]) break for idl in range(Ndimslat): if varobj.dimensions[idv] == latobj.dimensions[idl]: if not searchInlist(samedim,varobj.dimensions[idv]): samedim.append(varobj.dimensions[idv]) break Ndimshare = len(samedim) print 'variable and lon/lat matrices share ', Ndimshare,' dimensions: ',samedim samedimslonlat = True for idL in range(Ndimslon): for idl in range(Ndimslat): if lonobj.dimensions[idl] != latobj.dimensions[idl]: samedimslonlat = False break if not samedimslonlat: break # Creation of the lon/lat matrices to search if Ndimshare == 1: lonmat = lonobj[:] latmat = latobj[:] elif Ndimshare == 2: if samedimslonlat: lonmat = lonobj[:] latmat = latobj[:] else: if Ndimslon != 1 and Ndimslat != 1: print errormsg print ' ' + fname + ': I do not know how to keep going!' print ' lon ', Ndimslon,' and lat ',Ndimslat, \ ' matrices do not share the same dimensions, and do not have shape 1' quit(-1) else: lonmat = np.zeros((latobj.shape[0], lonobj.shape[0]), dtype=np.float) latmat = np.zeros((latobj.shape[0], lonobj.shape[0]), dtype=np.float) for i in range(lonobj.shape[0]): latmat[:,i] = latobj[:] for j in range(latobj.shape[0]): lonmat[j,:] = lonobj[:] elif Ndimshare == 3: if samedimslonlat: lonmat = lonobj[0,:,:] latmat = latobj[0,:,:] else: print errormsg print ' ' + fname + ': dimension sharing of lon/lat not ready!' quit(-1) # Searching for inside points iind = 0 if Ndimshare == 1: inside = {} for iL in range(lonobj.shape[0]): if lonobj[iL] >= lonSW and lonobj[iL] <= lonNE and latobj[iL] >= latSW \ and latobj[iL] <= latNE: inside[iind] = iL iind = iind + 1 elif Ndimshare == 2: newidx = [] newidy = [] newidz = [] if (len(lonobj.shape) == 3): inside3D = np.zeros((varobj.shape[1], lonobj.shape[1], lonobj.shape[2]), dtype=bool) inside2D = np.zeros((lonobj.shape[1], lonobj.shape[2]), dtype=bool) insideZ = np.zeros((varobj.shape[1]), dtype=bool) else: inside3D = np.zeros((varobj.shape[1], lonobj.shape), dtype=bool) inside2D = np.zeros((lonobj.shape), dtype=bool) insideZ = np.zeros((varobj.shape[1]), dtype=bool) for iL in range(lonobj.shape[1]): for il in range(lonobj.shape[0]): if lonobj[il,iL] >= lonSW and lonobj[il,iL] <= lonNE and \ latobj[il,iL] >= latSW and latobj[il,iL] <= latNE: inside2D[il, iL] = True if not searchInlist(newidx, iL): newidx.append(iL) if not searchInlist(newidy, il): newidy.append(il) for iz in range(Ndimslev): if iz >= levI and iz <= levF: if not searchInlist(newidz, iz): newidz.append(iz) inside3D[iz,il, iL] = True insideZ[iz] = True iind = iind + 1 elif Ndimshare == 3: newidx = [] newidy = [] newidz = [] inside3D = np.zeros((varobj.shape[1], lonobj.shape[1], lonobj.shape[2]), dtype=bool) inside2D = np.zeros((lonobj.shape[1], lonobj.shape[2]), dtype=bool) insideZ = np.zeros((varobj.shape[1]), dtype=bool) for iL in range(lonobj.shape[2]): for il in range(lonobj.shape[1]): if lonobj[0,il,iL] >= lonSW and lonobj[0,il,iL] <= lonNE and \ latobj[0,il,iL] >= latSW and latobj[0,il,iL] <= latNE: inside2D[il, iL] = True if not searchInlist(newidx, iL): newidx.append(iL) if not searchInlist(newidy, il): newidy.append(il) for iz in range(Ndimslev): if iz >= levI and iz <= levF: if not searchInlist(newidz, iz): newidz.append(iz) inside3D[iz, il, iL] = True insideZ[iz] = True iind = iind + 1 Ninpts3D = len(inside3D) Ninpts2D = len(inside2D) NinptsZ = len(insideZ) newdx = len(newidx) newdy = len(newidy) newdz = len(newidz) print ' ' + fname + ': ',Ninpts3D,' pts found in the box: ',lonSW,',',latSW,',', \ levI,' x ', lonNE,',',latNE,',',levF print ' ' + fname + ': dimensions output matrix:', newdz, ',', newdy, ',', newdx # Creation of cropped matrices Ninlev = levF - levI + 1 inlon = np.zeros((Ninpts2D), dtype=np.float) inlat = np.zeros((Ninpts2D), dtype=np.float) inlev = np.zeros((NinptsZ), dtype=np.float) invar = np.zeros((varobj.shape[0],Ninpts3D), dtype=np.float) # Creation of the netCDF file ## objofile = NetCDFFile(ofile, 'w') if Ndimshare == 1: # Dimensions newdim = objofile.createDimension('boxpt', Ninpts3D) newdim = objofile.createDimension('time', None) # var dimensions newvar = objofile.createVariable('lon', 'f8', ('boxpt')) newattr = basicvardef(newvar, 'lon', 'longitude', 'degrees west_east') newvar[:] = lonobj[inside2D[iin]] newvar = objofile.createVariable('lat', 'f8', ('boxpt')) newattr = basicvardef(newvar, 'lat', 'latitude', 'degrees north_south') newvar[:] = latobj[inside2D[iin]] newvar = objofile.createVariable('lev', 'f8', ('NinptsZ')) newattr = basicvardef(newvar, 'lev', 'level', '-') newvar[:] = levobj[insideZ[iin]] newvar = objofile.createVariable('time', 'f8', ('time')) if objfile.variables.has_key('time'): otime = objfile.variables['time'] timevals = otime[:] if searchInlist(otime.ncattrs(),'units'): tunits = otime.getncattr['units'] else: tunits = 'unkown' else: timevals = np.arange(varobj.shape[0]) tunits = 'unkown' newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = timevals # variable newvar = objofile.createVariable(varn, 'f4', ('time', 'boxpt')) newvar[:] = varobj[invar] else: # Dimensions newdim = objofile.createDimension('x', newdx) newdim = objofile.createDimension('y', newdy) newdim = objofile.createDimension('z', newdz) newdim = objofile.createDimension('time', None) # var dimensions newvar = objofile.createVariable('lon', 'f8', ('y', 'x')) if Ndimshare == 2: Vals = lonobj[:] else: Vals = lonobj[0,:,:] for it in range(varobj.shape[0]): newvar[:] = Vals[inside2D].reshape(newdy,newdx) newattr = basicvardef(newvar, 'lon', 'longitude', 'degrees west_east') newvar = objofile.createVariable('lat', 'f8', ('y', 'x')) newattr = basicvardef(newvar, 'lat', 'latitude', 'degrees north_south') if Ndimshare == 2: Vals = latobj[:] else: Vals = latobj[0,:,:] for it in range(varobj.shape[0]): newvar[:] = Vals[inside2D].reshape(newdy,newdx) newvar = objofile.createVariable('lev', 'f8', ('z')) newattr = basicvardef(newvar, 'lev', 'level', '-') if len(levobj.shape) == 2: newvar[:] = levobj[0,insideZ] else: newvar[:] = levobj[insideZ] newvar = objofile.createVariable('time', 'f8', ('time')) if objfile.variables.has_key('time'): otime = objfile.variables['time'] timevals = otime[:] if searchInlist(otime.ncattrs(),'units'): tunits = otime.getncattr['units'] else: tunits = 'unkown' else: timevals = np.arange(varobj.shape[0]) tunits = 'unkown' newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = timevals # variable newvar = objofile.createVariable(varn, 'f4', ('time', 'z', 'y', 'x')) for it in range(varobj.shape[0]): valsvar = varobj[it,:,:,:] newvar[it,:,:,:] = valsvar[inside3D] if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = varn if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = varn if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = 'unkown' newattr = basicvardef(newvar, vsname, vlname, vunits) # global attributes objofile.setncattr('author', 'L. Fita') objofile.setncattr('institution', 'Laboratire de Meteorologie Dynamique') objofile.setncattr('university', 'Pierre Marie Curie - Jussieu') objofile.setncattr('center', 'Centre National de Recherches Scientifiques') objofile.setncattr('city', 'Paris') objofile.setncattr('country', 'France') objofile.setncattr('script', 'nc_var_tools.py') objofile.setncattr('function', 'sellonlatlevbox') objofile.setncattr('version', '1.0') objfile.close() objofile.sync() objofile.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return #sellonlatlevbox('XLONG,XLAT,ZNU,-12.4,25.35,32.4,52.65,0,0', 'control/wrfout_d01_2001-11-09_00:00:00', 'P') def file_nlines(filen,char): """ Function to provide the number of lines of a file filen= name of the file char= character as no line >>> file_nlines('trajectory.dat','#') 49 """ fname = 'file_nlines' if not os.path.isfile(filen): print errormsg print ' ' + fname + ' file: "' + filen + '" does not exist !!' quit(-1) fo = open(filen,'r') nlines=0 for line in fo: if line[0:1] != char: nlines = nlines + 1 fo.close() return nlines def datetimeStr_conversion(StringDT,typeSi,typeSo): """ Function to transform a string date to an another date object StringDT= string with the date and time typeSi= type of datetime string input typeSo= type of datetime string output [typeSi/o] 'cfTime': [time],[units]; ]time in CF-convention format [units] = [tunits] since [refdate] 'matYmdHMS': numerical vector with [[YYYY], [MM], [DD], [HH], [MI], [SS]] 'YmdHMS': [YYYY][MM][DD][HH][MI][SS] format 'Y-m-d_H:M:S': [YYYY]-[MM]-[DD]_[HH]:[MI]:[SS] format 'Y-m-d H:M:S': [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] format 'Y/m/d H-M-S': [YYYY]/[MM]/[DD] [HH]-[MI]-[SS] format 'WRFdatetime': [Y], [Y], [Y], [Y], '-', [M], [M], '-', [D], [D], '_', [H], [H], ':', [M], [M], ':', [S], [S] >>> datetimeStr_conversion('1976-02-17_08:32:05','Y-m-d_H:M:S','matYmdHMS') [1976 2 17 8 32 5] >>> datetimeStr_conversion(str(137880)+',minutes since 1979-12-01_00:00:00','cfTime','Y/m/d H-M-S') 1980/03/05 18-00-00 """ import datetime as dt fname = 'datetimeStr_conversion' if StringDT[0:1] == 'h': print fname + '_____________________________________________________________' print datetimeStr_conversion.__doc__ quit() if typeSi == 'cfTime': timeval = np.float(StringDT.split(',')[0]) tunits = StringDT.split(',')[1].split(' ')[0] Srefdate = StringDT.split(',')[1].split(' ')[2] # Does reference date contain a time value [YYYY]-[MM]-[DD] [HH]:[MI]:[SS] ## yrref=Srefdate[0:4] monref=Srefdate[5:7] dayref=Srefdate[8:10] trefT = Srefdate.find(':') if not trefT == -1: # print ' ' + fname + ': refdate with time!' horref=Srefdate[11:13] minref=Srefdate[14:16] secref=Srefdate[17:19] refdate = datetimeStr_datetime( yrref + '-' + monref + '-' + dayref + \ '_' + horref + ':' + minref + ':' + secref) else: refdate = datetimeStr_datetime( yrref + '-' + monref + '-' + dayref + \ + '_00:00:00') if tunits == 'weeks': newdate = refdate + dt.timedelta(weeks=float(timeval)) elif tunits == 'days': newdate = refdate + dt.timedelta(days=float(timeval)) elif tunits == 'hours': newdate = refdate + dt.timedelta(hours=float(timeval)) elif tunits == 'minutes': newdate = refdate + dt.timedelta(minutes=float(timeval)) elif tunits == 'seconds': newdate = refdate + dt.timedelta(seconds=float(timeval)) elif tunits == 'milliseconds': newdate = refdate + dt.timedelta(milliseconds=float(timeval)) else: print errormsg print ' timeref_datetime: time units "' + tunits + '" not ready!!!!' quit(-1) yr = newdate.year mo = newdate.month da = newdate.day ho = newdate.hour mi = newdate.minute se = newdate.second elif typeSi == 'matYmdHMS': yr = StringDT[0] mo = StringDT[1] da = StringDT[2] ho = StringDT[3] mi = StringDT[4] se = StringDT[5] elif typeSi == 'YmdHMS': yr = int(StringDT[0:4]) mo = int(StringDT[4:6]) da = int(StringDT[6:8]) ho = int(StringDT[8:10]) mi = int(StringDT[10:12]) se = int(StringDT[12:14]) elif typeSi == 'Y-m-d_H:M:S': dateDT = StringDT.split('_') dateD = dateDT[0].split('-') timeT = dateDT[1].split(':') yr = int(dateD[0]) mo = int(dateD[1]) da = int(dateD[2]) ho = int(timeT[0]) mi = int(timeT[1]) se = int(timeT[2]) elif typeSi == 'Y-m-d H:M:S': dateDT = StringDT.split(' ') dateD = dateDT[0].split('-') timeT = dateDT[1].split(':') yr = int(dateD[0]) mo = int(dateD[1]) da = int(dateD[2]) ho = int(timeT[0]) mi = int(timeT[1]) se = int(timeT[2]) elif typeSi == 'Y/m/d H-M-S': dateDT = StringDT.split(' ') dateD = dateDT[0].split('/') timeT = dateDT[1].split('-') yr = int(dateD[0]) mo = int(dateD[1]) da = int(dateD[2]) ho = int(timeT[0]) mi = int(timeT[1]) se = int(timeT[2]) elif typeSi == 'WRFdatetime': yr = int(StringDT[0])*1000 + int(StringDT[1])*100 + int(StringDT[2])*10 + \ int(StringDT[3]) mo = int(StringDT[5])*10 + int(StringDT[6]) da = int(StringDT[8])*10 + int(StringDT[9]) ho = int(StringDT[11])*10 + int(StringDT[12]) mi = int(StringDT[14])*10 + int(StringDT[15]) se = int(StringDT[17])*10 + int(StringDT[18]) else: print errormsg print ' ' + fname + ': type of String input date "' + typeSi + \ '" not ready !!!!' quit(-1) if typeSo == 'matYmdHMS': dateYmdHMS = np.zeros((6), dtype=int) dateYmdHMS[0] = yr dateYmdHMS[1] = mo dateYmdHMS[2] = da dateYmdHMS[3] = ho dateYmdHMS[4] = mi dateYmdHMS[5] = se elif typeSo == 'YmdHMS': dateYmdHMS = str(yr).zfill(4) + str(mo).zfill(2) + str(da).zfill(2) + \ str(ho).zfill(2) + str(mi).zfill(2) + str(se).zfill(2) elif typeSo == 'Y-m-d_H:M:S': dateYmdHMS = str(yr).zfill(4) + '-' + str(mo).zfill(2) + '-' + \ str(da).zfill(2) + '_' + str(ho).zfill(2) + ':' + str(mi).zfill(2) + ':' + \ str(se).zfill(2) elif typeSo == 'Y-m-d H:M:S': dateYmdHMS = str(yr).zfill(4) + '-' + str(mo).zfill(2) + '-' + \ str(da).zfill(2) + ' ' + str(ho).zfill(2) + ':' + str(mi).zfill(2) + ':' + \ str(se).zfill(2) elif typeSo == 'Y/m/d H-M-S': dateYmdHMS = str(yr).zfill(4) + '/' + str(mo).zfill(2) + '/' + \ str(da).zfill(2) + ' ' + str(ho).zfill(2) + '-' + str(mi).zfill(2) + '-' + \ str(se).zfill(2) elif typeSo == 'WRFdatetime': dateYmdHMS = [] yM = yr/1000 yC = (yr-yM*1000)/100 yD = (yr-yM*1000-yC*100)/10 yU = yr-yM*1000-yC*100-yD*10 mD = mo/10 mU = mo-mD*10 dD = da/10 dU = da-dD*10 hD = ho/10 hU = ho-hD*10 miD = mi/10 miU = mi-miD*10 sD = se/10 sU = se-sD*10 dateYmdHMS.append(str(yM)) dateYmdHMS.append(str(yC)) dateYmdHMS.append(str(yD)) dateYmdHMS.append(str(yU)) dateYmdHMS.append('-') dateYmdHMS.append(str(mD)) dateYmdHMS.append(str(mU)) dateYmdHMS.append('-') dateYmdHMS.append(str(dD)) dateYmdHMS.append(str(dU)) dateYmdHMS.append('_') dateYmdHMS.append(str(hD)) dateYmdHMS.append(str(hU)) dateYmdHMS.append(':') dateYmdHMS.append(str(miD)) dateYmdHMS.append(str(miU)) dateYmdHMS.append(':') dateYmdHMS.append(str(sD)) dateYmdHMS.append(str(sU)) else: print errormsg print ' ' + fname + ': type of output date "' + typeSo + '" not ready !!!!' quit(-1) return dateYmdHMS def table_tex(tablevals, colnames, rownames, of): """ Function to write into a LaTeX tabukar from a table of values tablevals = (ncol nrow) of values colnames = list with ncol labels for the columns rownames = list with nrow labels for the rows of= object ASCII file to write the table """ errormsg = 'ERROR -- error -- ERROR -- error' fname = 'table_tex' Ncol = tablevals.shape[0] Nrow = tablevals.shape[1] if Ncol + 1 != len(colnames): print errormsg print ' ' + fname + ': wrong number of column names!!' print ' data has:', Ncol, 'and you provided:', len(colnames) print ' remember to provide one more for the label of the rows!' print ' you provide:',colnames quit(-1) if Nrow != len(rownames): print errormsg print ' ' + fname + ': wrong number of row names!!' print ' data has:', Nrow, 'and you provided:', len(rownames) print ' you provide:',rownames quit(-1) colcs = '' colns = '' for icol in colnames: colcs = colcs + 'c' if icol == colnames[0]: colns = ' {\\bfseries{' + icol + '}}' else: colns = colns + ' & {\\bfseries{' + icol + '}}' of.write('\n') of.write("%Automatically written file from function '" + fname + "'\n") of.write('\\begin{tabular}{l'+colcs+'}\n') of.write(colns + ' \\\\ \\hline\n') ir = 0 for irow in rownames: rowns = '{\\bfseries{' + irow + '}}' for ic in range(Ncol): rowns = rowns + ' & ' + str(tablevals[ic,ir]) rowns = rowns + ' \\\\' of.write(rowns + '\n') ir = ir + 1 of.write('\\end{tabular}\n') return def radius_dist(dx,dy,ptx,pty): """ Function to generate a matrix with the distance at a given point radius_dist(dx,dy,ptx,pty) [dx/y]: dimension of the matrix [ptx/y]: grid point coordinates of the point >>> radius_dist(3,5,2,2) [[ 1.41421356 1. 1.41421356 2.23606798 3.16227766] [ 1. 0. 1. 2. 3. ] [ 1.41421356 1. 1.41421356 2.23606798 3.16227766]] """ fname = 'radius_dist' if ptx < 0 or ptx > dx-1 or pty < 0 or pty > dy-1: print errormsg print ' ' + fname + ': wrong point coordinates:',dx,',',dy,'for matrix;',dx \ ,'x',dy quit(-1) xdist = np.zeros((dx,dy), dtype=np.float) ydist = np.zeros((dx,dy), dtype=np.float) dist = np.zeros((dx,dy), dtype=np.float) for ix in range(dx): xdist[ix,:] = np.float(ix-ptx) for iy in range(dy): ydist[:,iy] = np.float(iy-pty) dist = np.sqrt(xdist*xdist + ydist*ydist) return dist def lonlat2D(lon,lat): """ Function to return lon, lat 2D matrices from any lon,lat matrix lon= matrix with longitude values lat= matrix with latitude values """ fname = 'lonlat2D' if len(lon.shape) != len(lat.shape): print errormsg print ' ' + fname + ': longitude values with shape:', lon.shape, \ 'is different that latitude values with shape:', lat.shape, '(dif. size) !!' quit(-1) if len(lon.shape) == 3: lonvv = lon[0,:,:] latvv = lat[0,:,:] elif len(lon.shape) == 2: lonvv = lon[:] latvv = lat[:] elif len(lon.shape) == 1: lonlatv = np.meshgrid(lon[:],lat[:]) lonvv = lonlatv[0] latvv = lonlatv[1] return lonvv, latvv def compute_tevolboxtraj(values, ncfile, varn): """ Function to compute tevolboxtraj: temporal evolution at a given point along a box following a trajectory values= [trajfile]@[Tbeg],[lonname],[latname],[zname],[timename],[timekind],[boxsize],[circler] [trajfile]: ASCII file with the trajectory ('#' not readed) [time] [xpoint] [ypoint] [Tbeg]: equivalent first time-step of the trajectory within the netCDF file [lonname],[latname],[zname],[timename]: longitude, latitude, z and time variables names [timekind]: kind of time 'cf': cf-compilant 'wrf': WRF kind [boxsize]: size in grid points of the box (square centered, better even number!) [circler]: radius in grid points of a centerd circle (as 2*circler+1) ncfile= netCDF file to use varn= ',' list of variables' name ('all', for all variables) EMERGENCY version, assuming 3D [time,lat,lon] variable ! """ import numpy.ma as ma import subprocess as sub fname='compute_tevolboxtraj' if values == 'h': print fname + '_____________________________________________________________' print compute_tevolboxtraj.__doc__ quit() arguments = '[trajfile]@[Tbeg],[lonname],[latname],[zname],[timename],' + \ '[timekind],[boxsize],[circler]]' check_arguments(fname,values,arguments,',') trajfile = values.split(',')[0].split('@')[0] Tbeg = int(values.split(',')[0].split('@')[1]) lonn = values.split(',')[1] latn = values.split(',')[2] zn = values.split(',')[3] timn = values.split(',')[4] timekind = values.split(',')[5] boxs = int(values.split(',')[6]) circler = int(values.split(',')[7]) if np.mod(boxs,2) != 1: print errormsg print ' ' + fname + ': box size:', boxs, 'is not even !!!' quit(-1) box2 = int(boxs/2) if not os.path.isfile(ncfile): print errormsg print ' ' + fname + " netCDF file: '" + ncfile + "' does not exist !!" quit(-1) if not os.path.isfile(trajfile): print errormsg print ' ' + fname + " trajectory file: '" + trajfile + "' does not exist !!" quit(-1) # circle values # radius = np.float(rboxs.split('#')[0]) # ddist = np.float(rboxs.split('#')[1]) # if radius < ddist: # print errormsg # print ' ' + fname + ': radius of the circle:',radius,' is smaller than ' + \ # 'the distance between grid points !!' # quit(-1) # else: # Nrad = int(radius/ddist) Nrad = int(circler) objfile = NetCDFFile(ncfile, 'r') lonobj = objfile.variables[lonn] latobj = objfile.variables[latn] timobj = objfile.variables[timn] if varn.find(',') != -1: varns = varn.split(',') else: if varn == 'all': varns = objfile.variables else: varns = [varn] lonv, latv = lonlat2D(lonobj[:],latobj[:]) dimx = lonv.shape[1] dimy = lonv.shape[0] timv = timobj[:] # Selecting accordingly a trajectory ## Ttraj = file_nlines(trajfile,'#') if timekind == 'wrf': dimt = objfile.variables[timn].shape[0] else: dimt = objfile.variables[timn].shape[0] if Tbeg + Ttraj > dimt: print errormsg print ' ' + fname + ': trajectory has ', Ttraj, ' time steps and starts ' + \ ' at',Tbeg,' and data',dimt, '" !!!!!' quit(-1) print ' ' + fname + ': Number of time-steps in trajectory file: ',Ttraj, 'dimt',dimt trajobj = open(trajfile,'r') # Trajectory values/slices ## iline = 0 it = 0 xrangeslice = [] yrangeslice = [] xrangeslice2D = [] yrangeslice2D = [] cxrangeslice = [] cyrangeslice = [] cxrangeslice2D = [] cyrangeslice2D = [] cslicev = [] cslice2D = [] cslicevnoT = [] gtrajvals = np.zeros((Ttraj,3), dtype=int) trajvals = np.zeros((Ttraj,3), dtype=np.float) circdist = np.zeros((Ttraj, dimy, dimx), dtype=np.float) it = 0 iline = 0 for line in trajobj: ## Skipping first line ## if not iline == 0: ## it = iline - 1 ## it = iline # Slicing brings to reduce 1 time-step.... ??? if line[0:1] != '#': gtrajvals[it,0] = Tbeg + iline - 1 gtrajvals[it,1] = int(line.split(' ')[1]) gtrajvals[it,2] = int(line.split(' ')[2]) # print it,'t:',gtrajvals[it,0],'x y:', gtrajvals[it,1], gtrajvals[it,2] if timekind == 'wrf': gdate = datetimeStr_conversion(timv[gtrajvals[it,0]],'WRFdatetime', \ 'matYmdHMS') trajvals[it,0] = realdatetime1_CFcompilant(gdate, '19491201000000', \ 'hours') tunits = 'hours since 1949-12-01 00:00:00' elif timekind == 'cf': trajvals[it,0] = timv[gtrajvals[it,0]] tunits = timobj.getncattr('units') else: print errormsg print ' ' + fname + ' time kind: "' + timekind + '" not ready !!' quit(-1) trajvals[it,1] = lonv[gtrajvals[it,2],gtrajvals[it,1]] trajvals[it,2] = latv[gtrajvals[it,2],gtrajvals[it,1]] # print iline, it,'time:',trajvals[it,0],'lon lat:', trajvals[it,1], \ # trajvals[it,2] # Assuming time as the first dimension in a fortran like way (t=0 as 1) # trajcut[0] = tstept - 1 # Assuming time as the first dimension in a C/python like way (t=0) # box values if gtrajvals[it,2]-box2 < 0 or gtrajvals[it,2]+box2 + 1 > dimy + 1 \ or gtrajvals[it,1]-box2 < 0 or gtrajvals[it,1]+box2 + 1 > dimx + 1: varvalst = np.ones((boxs, boxs), dtype=np.float)*fillValue if gtrajvals[it,2]-box2 < 0: yinit = 0 yinit2D = box2 - gtrajvals[it,2] else: yinit = gtrajvals[it,2]-box2 yinit2D = 0 if gtrajvals[it,2]+box2 + 1 > dimy: yend = dimy - 1 yend2D = dimy - 1 - gtrajvals[it,2] + box2 else: yend = gtrajvals[it,2]+box2 + 1 yend2D = boxs if gtrajvals[it,1]-box2 < 0: xinit = 0 xinit2D = box2 - gtrajvals[it,1] else: xinit = gtrajvals[it,1]-box2 xinit2D = 0 if gtrajvals[it,1]+box2 + 1 > dimx: xend = dimx - 1 xend2D = dimx - 1 - gtrajvals[it,1] - box2 else: xend = gtrajvals[it,1]+box2 + 1 xend2D = boxs yrangeslice.append([yinit, yend]) xrangeslice.append([xinit, xend]) yrangeslice2D.append([yinit2D, yend2D]) xrangeslice2D.append([xinit2D, xend2D]) else: yrangeslice.append([gtrajvals[it,2]-box2, gtrajvals[it,2]+box2 + 1]) xrangeslice.append([gtrajvals[it,1]-box2, gtrajvals[it,1]+box2 + 1]) yrangeslice2D.append([gtrajvals[it,2]-box2, gtrajvals[it,2]+box2 + 1]) xrangeslice2D.append([gtrajvals[it,1]-box2, gtrajvals[it,1]+box2 + 1]) # circle values circdist[it,:,:] = radius_dist(dimy,dimx,gtrajvals[it,2],gtrajvals[it,1]) if gtrajvals[it,2]-Nrad < 0 or gtrajvals[it,2]+Nrad + 1 > dimy + 1 \ or gtrajvals[it,1]-Nrad < 0 or gtrajvals[it,1]+Nrad + 1 > dimx + 1: if gtrajvals[it,2]-Nrad < 0: yinit = 0 yinit2D = Nrad - gtrajvals[it,2] else: yinit = gtrajvals[it,2]-Nrad yinit2D = 0 if gtrajvals[it,2]+Nrad + 1 > dimy: yend = dimy - 1 yend2D = dimy - 1 - gtrajvals[it,2] + Nrad else: yend = gtrajvals[it,2]+Nrad + 1 yend2D = 2*Nrad+1 if gtrajvals[it,1]-Nrad < 0: xinit = 0 xinit2D = Nrad - gtrajvals[it,1] else: xinit = gtrajvals[it,1]-Nrad xinit2D = 0 if gtrajvals[it,1]+Nrad + 1 > dimx: xend = dimx - 1 xend2D = dimx - 1 - gtrajvals[it,1] - Nrad else: xend = gtrajvals[it,1]+Nrad + 1 xend2D = 2*Nrad+1 cyrangeslice.append([yinit, yend]) cxrangeslice.append([xinit, xend]) cyrangeslice2D.append([yinit2D, yend2D]) cxrangeslice2D.append([xinit2D, xend2D]) else: cyrangeslice.append([gtrajvals[it,2]-Nrad, gtrajvals[it,2]+Nrad + 1]) cxrangeslice.append([gtrajvals[it,1]-Nrad, gtrajvals[it,1]+Nrad + 1]) cyrangeslice2D.append([gtrajvals[it,2]-Nrad, gtrajvals[it,2]+Nrad+1]) cxrangeslice2D.append([gtrajvals[it,1]-Nrad, gtrajvals[it,1]+Nrad+1]) it = it + 1 iline = iline + 1 trajobj.close() # Creation of the netCDF file ## if varn.find(',') != -1: varnS = 'multi-var' else: varnS = varn.replace(',','-') ofile = 'tevolboxtraj_' + varnS + '.nc' objofile = NetCDFFile(ofile, 'w') # Dimensions newdim = objofile.createDimension('x', boxs) newdim = objofile.createDimension('y', boxs) newdim = objofile.createDimension('xr', Nrad*2+1) newdim = objofile.createDimension('yr', Nrad*2+1) newdim = objofile.createDimension('time', None) stsn = ['min', 'max', 'mean', 'mean2', 'stdev', 'ac'] vstlname = ['minimum value within', 'maximum value within', 'mean value within', \ 'squared mean value within', 'standard deviation value within', \ 'accumulated value within'] Nsts = len(stsn) statnames = [] cstatnames = [] for i in range(Nsts): statnames.append(stsn[i] + 'box') cstatnames.append(stsn[i] + 'circle') # Getting values ## ivar = 0 for vn in varns: vnst = variables_values(vn)[0] if not searchInlist(objfile.variables, vn): print errormsg print ' ' + fname + ": variable name '" + vn + "' is not in file " + \ ncfile + '" !!!!!' quit(-1) varobj = objfile.variables[vn] Nvardims = len(varobj.shape) print ' ' + fname + ": getting '" + vn + "' ... .. ." slicev = [] slice2D = [] slicevnoT = [] cslicev = [] cslice2D = [] cslicevnoT = [] if Nvardims == 4: # Too optimistic, but at this stage... if not objofile.dimensions.has_key('z'): varzobj = objfile.variables[zn] if len(varzobj.shape) == 1: dimz = varzobj.shape zvals = varzobj[:] elif len(varzobj.shape) == 2: dimz = varzobj.shape[1] zvals = varzobj[0,:] elif len(varzobj.shape) == 3: dimz = varzobj.shape[2] zvals = varzobj[0,0,:] objofile.createDimension('z', dimz) newvar = objofile.createVariable(zn, 'f4', ('z'),fill_value=fillValue) if searchInlist(varzobj.ncattrs(),'standard_name'): vsname = varzobj.getncattr('standard_name') else: vsname = variables_values(zn)[1] if searchInlist(varzobj.ncattrs(),'long_name'): vlname = varzobj.getncattr('long_name') else: vlname = variables_values(zn)[4].replace('|',' ') if searchInlist(varzobj.ncattrs(),'units'): vunits = varzobj.getncattr('units') else: vunits = variables_values(zn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newvar[:] = zvals varvals = np.ones(tuple([Ttraj,dimz,boxs,boxs]), dtype=np.float) lonvals = np.ones(tuple([Ttraj,dimz,boxs,boxs]), dtype=np.float) latvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) rvarvals = np.ones(tuple([Ttraj,dimz,Nrad*2+1,Nrad*2+1]), dtype=np.float) rlonvals = np.ones(tuple([Ttraj,dimz,Nrad*2+1,Nrad*2+1]), dtype=np.float) rlatvals = np.ones(tuple([Ttraj,dimz,Nrad*2+1,Nrad*2+1]), dtype=np.float) # One dimension plus for the values at the center of the trajectory statvarvals = np.ones(tuple([Ttraj,dimz,Nsts+1]), dtype=np.float) rstatvarvals = np.ones(tuple([Ttraj,dimz,Nsts+1]), dtype=np.float) for it in range(Ttraj): it0 = Tbeg + it slicev = [] slice2D = [] slicevnoT = [] cslicev = [] cslice2D = [] cslice2Dhor = [] cslicevnoT = [] cslicevnoThor = [] slicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-box2 < 0 or gtrajvals[it,2]+box2 + 1 > dimy + 1 \ or gtrajvals[it,1]-box2 < 0 or gtrajvals[it,1]+box2 + 1 > dimx +1: # box values slicev.append(slice(0,dimz)) slicev.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicev.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slicevnoT.append(slice(0,dimz)) slicevnoT.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicevnoT.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slice2D.append(slice(0,dimz)) slice2D.append(slice(0,yrangeslice[it][1]-yrangeslice[it][0])) slice2D.append(slice(0,xrangeslice[it][1]-xrangeslice[it][0])) rvarvalst = np.ones((dimz, Nrad*2+1, Nrad*2+1),dtype=np.float)* \ fillValue varvalst[tuple(slice2D)] = varobj[tuple(slicev)] varvals[it,:,:,:] = varvalst # box stats values maskedvals = ma.masked_values (varvalst, fillValue) maskedvals2 = maskedvals*maskedvals for iz in range(dimz): statvarvals[it,iz,0] = varvalst[iz,box2,box2] statvarvals[it,iz,1] = np.min(varvalst[iz,:,:]) statvarvals[it,iz,2] = np.max(varvalst[iz,:,:]) statvarvals[it,iz,3] = np.mean(varvalst[iz,:,:]) statvarvals[it,iz,4] = maskedvals2[iz,:,:].mean() statvarvals[it,iz,5] = np.sqrt(statvarvals[it,iz,4] - \ statvarvals[it,iz,3]*statvarvals[it,iz,3]) statvarvals[it,iz,6] = np.sum(varvalst[iz,:,:]) else: slicev.append(slice(0,dimz)) slicev.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2+1)) slicev.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2+1)) slicevnoT.append(slice(0,dimz)) slicevnoT.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+ \ box2+1)) slicevnoT.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+ \ box2+1)) slice2D.append(slice(0,dimz)) slice2D.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2] + \ box2 + 1)) slice2D.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1] + \ box2 + 1)) varvalst = varobj[tuple(slicev)] # box values varvals[it,:,:,:] = varvalst # print 'values at time t______' # print varvalst # box stats values for iz in range(dimz): statvarvals[it,:,0] = varvalst[:,box2,box2] statvarvals[it,iz,1] = np.min(varvalst[iz,:,:]) statvarvals[it,iz,2] = np.max(varvalst[iz,:,:]) statvarvals[it,iz,3] = np.mean(varvalst[iz,:,:]) statvarvals[it,iz,4] = np.mean(varvalst[iz,:,:]* \ varvalst[iz,:,:]) statvarvals[it,iz,5] = np.sqrt(statvarvals[it,iz,4] - \ statvarvals[it,iz,3]*statvarvals[it,iz,3]) statvarvals[it,iz,6] = np.sum(varvalst[iz,:,:]) # Circle values cslicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-Nrad < 0 or gtrajvals[it,2]+Nrad + 1 >= dimy \ or gtrajvals[it,1]-Nrad < 0 or gtrajvals[it,1]+Nrad + 1 >= dimx: maxx = np.min([cxrangeslice[it][1], dimx]) maxy = np.min([cyrangeslice[it][1], dimy]) cslicev.append(slice(0,dimz)) cslicev.append(slice(cyrangeslice[it][0], maxy)) cslicev.append(slice(cxrangeslice[it][0], maxx)) cslicevnoT.append(slice(0,dimz)) cslicevnoT.append(slice(cyrangeslice[it][0], cyrangeslice[it][1])) cslicevnoT.append(slice(cxrangeslice[it][0], cxrangeslice[it][1])) cslice2D.append(slice(0,dimz)) cslice2D.append(slice(0,maxy-cyrangeslice[it][0])) cslice2D.append(slice(0,maxx-cxrangeslice[it][0])) cslice2Dhor.append(slice(0, maxy - cyrangeslice[it][0])) cslice2Dhor.append(slice(0, maxx -cxrangeslice[it][0])) rvarvalst = np.ones((dimz,Nrad*2+1,Nrad*2+1),dtype=np.float)* \ fillValue rvarvalst[tuple(cslice2D)] = varobj[tuple(cslicev)] for iz in range(dimz): tslice = [slice(it,it)]+cslice2Dhor zslice = [slice(iz,iz)]+cslice2Dhor rvarvalst[tuple(zslice)] = np.where(circdist[tuple(tslice)] > \ np.float(Nrad), fillValue, rvarvalst[tuple(zslice)]) rvarvals[it,:,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) maskedvals2 = maskedvals*maskedvals for iz in range(dimz): rstatvarvals[it,iz,0] = varvalst[iz,box2,box2] rstatvarvals[it,iz,1] = np.min(varvalst[iz,:,:]) rstatvarvals[it,iz,2] = np.max(varvalst[iz,:,:]) rstatvarvals[it,iz,3] = np.mean(varvalst[iz,:,:]) rstatvarvals[it,iz,4] = maskedvals2[iz,:,:].mean() rstatvarvals[it,iz,5] = np.sqrt(rstatvarvals[it,iz,4] - \ rstatvarvals[it,iz,3]*rstatvarvals[it,iz,3]) rstatvarvals[it,iz,6] = np.sum(varvalst[iz,:,:]) else: cslicev.append(slice(0,dimz)) cslicev.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslicev.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) cslicevnoT.append(slice(0,dimz)) cslicevnoT.append(slice(gtrajvals[it,2]-Nrad, gtrajvals[it,2]+ \ Nrad+1)) cslicevnoT.append(slice(gtrajvals[it,1]-Nrad, gtrajvals[it,1]+ \ Nrad+1)) cslicevnoThor.append(slice(gtrajvals[it,2]-Nrad, \ gtrajvals[it,2] + Nrad+1)) cslicevnoThor.append(slice(gtrajvals[it,1]-Nrad, \ gtrajvals[it,1] + Nrad+1)) cslice2D.append(slice(0,dimz)) cslice2D.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2] + \ Nrad+1)) cslice2D.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1] + \ Nrad+1)) rvarvalst = varobj[tuple(cslicev)] # circle values for iz in range(dimz): tslice = [it]+cslicevnoThor rvarvalst[iz,:,:] = np.where(circdist[tuple(tslice)] > \ np.float(Nrad), fillValue, rvarvalst[iz,:,:]) rvarvals[it,:,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) maskedvals2 = maskedvals*maskedvals for iz in range(dimz): rstatvarvals[it,iz,0] = rvarvalst[iz,Nrad,Nrad] rstatvarvals[it,iz,1] = maskedvals[iz,:,:].min() rstatvarvals[it,iz,2] = maskedvals[iz,:,:].max() rstatvarvals[it,iz,3] = maskedvals[iz,:,:].mean() rstatvarvals[it,iz,4] = maskedvals2[iz,:,:].mean() rstatvarvals[it,iz,5] = np.sqrt(rstatvarvals[it,iz,4] - \ rstatvarvals[it,iz,3]*rstatvarvals[it,iz,3]) rstatvarvals[it,iz,6] = maskedvals[iz,:,:].sum() # print 'statistics:',rstatvarvals[it,:] # variable box values newvar = objofile.createVariable(vnst + 'box', 'f4', ('time','z','y','x'),\ fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = varvals # center of the trajectory newvar = objofile.createVariable('trj_' + vnst, 'f', ('time','z'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'trj_' + vsname, 'value along the ' + \ 'trajectory of '+ vn, vunits) newvar[:] = statvarvals[:,:,0] # variable box statistics ist = 0 for statn in statnames: newvar = objofile.createVariable(statn + '_' + vnst,'f',('time','z'),\ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the box ('+str(boxs)+'x'+str(boxs)+') of ' + vnst, vunits) newvar[:] = statvarvals[:,:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 # variable circle values newvar = objofile.createVariable(vnst + 'circle','f4',('time','z','yr', \ 'xr'), fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = rvarvals # variable circle statistics ist = 0 for statn in cstatnames: newvar = objofile.createVariable(statn + '_' + vnst,'f',('time','z'),\ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the circle of radius ('+ str(circler)+') of ' + vnst, vunits) newvar[:] = rstatvarvals[:,:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 elif Nvardims == 3: # Too optimistic, but at this stage... dimt = varobj.shape[0] varvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) lonvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) latvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) rvarvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) rlonvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) rlatvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) # One dimension plus for the values at the center of the trajectory statvarvals = np.ones(tuple([Ttraj,Nsts+1]), dtype=np.float) rstatvarvals = np.ones(tuple([Ttraj,Nsts+1]), dtype=np.float) for it in range(Ttraj): it0 = Tbeg + it slicev = [] slice2D = [] slicevnoT = [] cslicev = [] cslice2D = [] cslicevnoT = [] cslicevC = [] slicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-box2 < 0 or gtrajvals[it,2]+box2 + 1 > dimy + 1 \ or gtrajvals[it,1]-box2 < 0 or gtrajvals[it,1]+box2 + 1 > dimx + 1: # box values slicev.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicev.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slicevnoT.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicevnoT.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slice2D.append(slice(0,yrangeslice[it][1]-yrangeslice[it][0])) slice2D.append(slice(0,xrangeslice[it][1]-xrangeslice[it][0])) rvarvalst = np.ones((Nrad*2+1, Nrad*2+1),dtype=np.float)*fillValue varvalst[tuple(slice2D)] = varobj[tuple(slicev)] varvals[it,:,:] = varvalst # box stats values maskedvals = ma.masked_values (varvalst, fillValue) statvarvals[it,0] = varvalst[box2,box2] statvarvals[it,1] = maskedvals.min() statvarvals[it,2] = maskedvals.max() statvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals statvarvals[it,4] = maskedvals2.mean() statvarvals[it,5] = np.sqrt(statvarvals[it,4] - \ statvarvals[it,3]*statvarvals[it,3]) statvarvals[it,6] = maskedvals.sum() varvalst[tuple(slice2D)] = lonv[tuple(slicevnoT)] lonvals[it,:,:] = varvalst varvalst[tuple(slice2D)] = latv[tuple(slicevnoT)] latvals[it,:,:] = varvalst else: slicev.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2+1)) slicev.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2+1)) slicevnoT.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+ \ box2+1)) slicevnoT.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+ \ box2+1)) slice2D.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2 +\ 1)) slice2D.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2 +\ 1)) varvalst = varobj[tuple(slicev)] # box values varvals[it,:,:] = varvalst # print 'values at time t______' # print varvalst # box stats values statvarvals[it,0] = varvalst[box2,box2] statvarvals[it,1] = np.min(varvalst) statvarvals[it,2] = np.max(varvalst) statvarvals[it,3] = np.mean(varvalst) statvarvals[it,4] = np.mean(varvalst*varvalst) statvarvals[it,5] = np.sqrt(statvarvals[it,4] - \ statvarvals[it,3]*statvarvals[it,3]) statvarvals[it,6] = np.sum(varvalst) varvalst = lonv[tuple(slicevnoT)] lonvals[it,:,:] = varvalst varvalst = latv[tuple(slicevnoT)] latvals[it,:,:] = varvalst # Circle values cslicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-Nrad < 0 or gtrajvals[it,2]+Nrad + 1 >= dimy \ or gtrajvals[it,1]-Nrad < 0 or gtrajvals[it,1]+Nrad + 1 >= dimx: maxx = np.min([cxrangeslice[it][1], dimx]) maxy = np.min([cyrangeslice[it][1], dimy]) cslicev.append(slice(cyrangeslice[it][0],maxy)) cslicev.append(slice(cxrangeslice[it][0],maxx)) cslicevnoT.append(slice(cyrangeslice[it][0],cyrangeslice[it][1])) cslicevnoT.append(slice(cxrangeslice[it][0],cxrangeslice[it][1])) cslice2D.append(slice(0,maxy - cyrangeslice[it][0])) cslice2D.append(slice(0,maxx - cxrangeslice[it][0])) rvarvalst = np.ones((Nrad*2+1,Nrad*2+1),dtype=np.float)*fillValue rvarvalst[tuple(cslice2D)] = varobj[tuple(cslicev)] rvarvalst[tuple(cslice2D)] = np.where(circdist[tuple(cslicev)] >\ np.float(Nrad), fillValue, rvarvalst[tuple(cslice2D)]) rvarvals[it,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) rstatvarvals[it,0] = rvarvalst[Nrad,Nrad] rstatvarvals[it,1] = maskedvals.min() rstatvarvals[it,2] = maskedvals.max() rstatvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals rstatvarvals[it,4] = maskedvals2.mean() rstatvarvals[it,5] = np.sqrt(rstatvarvals[it,4] - \ rstatvarvals[it,3]*rstatvarvals[it,3]) rstatvarvals[it,6] = maskedvals2.sum() else: cslicev.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslicev.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) cslicevnoT.append(slice(gtrajvals[it,2]-Nrad, gtrajvals[it,2]+ \ Nrad+1)) cslicevnoT.append(slice(gtrajvals[it,1]-Nrad, gtrajvals[it,1]+ \ Nrad+1)) cslice2D.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslice2D.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) cslicevC.append(it) cslicevC.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslicevC.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) rvarvalst = varobj[tuple(cslicev)] cdist = circdist[tuple(cslicevC)] # circle values rvarvalst = np.where(cdist > np.float(Nrad),fillValue,rvarvalst) rvarvals[it,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) rstatvarvals[it,0] = rvarvalst[Nrad,Nrad] rstatvarvals[it,1] = maskedvals.min() rstatvarvals[it,2] = maskedvals.max() rstatvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals rstatvarvals[it,4] = maskedvals2.mean() rstatvarvals[it,5] = np.sqrt(rstatvarvals[it,4] - \ rstatvarvals[it,3]*rstatvarvals[it,3]) rstatvarvals[it,6] = maskedvals.sum() # print 'statistics:',rstatvarvals[it,:] # variable box values newvar = objofile.createVariable(vnst + 'box', 'f4', ('time', 'y', 'x'), \ fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = varvals # center of the trajectory newvar = objofile.createVariable('trj_' + vnst, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'trj_' + vsname, 'value along the ' + \ 'trajectory of '+ vnst, vunits) newvar[:] = statvarvals[:,0] # variable box statistics ist = 0 for statn in statnames: newvar = objofile.createVariable(statn + '_' + vnst, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the box ('+str(boxs)+'x'+str(boxs)+') of ' + vnst, vunits) newvar[:] = statvarvals[:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 # variable circle values newvar = objofile.createVariable(vnst + 'circle','f4',('time','yr','xr'),\ fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = rvarvals # variable circle statistics ist = 0 for statn in cstatnames: newvar = objofile.createVariable(statn + '_' + vnst, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the circle of radius ('+ str(circler)+') of ' + vnst, vunits) newvar[:] = rstatvarvals[:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 else: # Other variables print warnmsg print ' ' + fname + ": variable '" + vn + "' shape:",varobj.shape,' not'\ + ' ready!!' print ' skipping variable' if len(varns) == 1: objofile.close() sub.call(['rm', ofile]) print ' uniq variable! removing file and finishing program' quit() if not objofile.variables.has_key('trlon') and Nvardims == 3: # var dimensions newvar = objofile.createVariable('trlon', 'f8', ('time')) newattr = basicvardef(newvar,'trlon','trajectory longitude', \ 'degrees west_east') newvar[:] = trajvals[:,1] newvar = objofile.createVariable('trlat', 'f8', ('time')) newattr = basicvardef(newvar,'trlat','trajectory latitude', \ 'degrees north_south') newvar[:] = trajvals[:,2] newvar = objofile.createVariable('time', 'f8', ('time')) newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = trajvals[:,0] newvar = objofile.createVariable('lon', 'f8', ('time', 'y', 'x'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'longitude', 'longitude', \ 'degrees west_east') newvar[:] = lonvals newvar = objofile.createVariable('lat', 'f8', ('time', 'y', 'x'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'latitude', 'latitude', \ 'degrees north_south') newvar[:] = latvals ivar = ivar + 1 # global attributes objofile.setncattr('author', 'L. Fita') objofile.setncattr('institution', 'Laboratire de Meteorologie Dynamique') objofile.setncattr('university', 'Pierre Marie Curie - Jussieu') objofile.setncattr('center', 'Centre National de Recherches Scientifiques') objofile.setncattr('city', 'Paris') objofile.setncattr('country', 'France') objofile.setncattr('script', 'nc_var_tools.py') objofile.setncattr('function', 'compute_tevolboxtraj') objofile.setncattr('version', '1.0') objofile.setncattr('data_file',ncfile) objfile.close() objofile.sync() objofile.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return #compute_tevolboxtraj('h', 'wrfout*', 'PSFC') #compute_tevolboxtraj('control/trajectory.dat@0,XLONG,XLAT,ZNU,Times,wrf,3,3', \ # '../../superstorm/control/wrfout/wrfout_d01_2001-11-09_00:00:00', 'PSFC') #compute_tevolboxtraj('control/trajectory.dat@0,lon,lat,time,cf,5,5', \ # 'control/wss.nc', 'wss') # in camelot, /bdd/PCER/workspace/lfita/etudes/FF/50sims #compute_tevolboxtraj('001/trajectory.dat@0,XLONG,XLAT,ZNU,Times,wrf,3,3', \ # '001/full_concatenated.nc', 'PSFC') def interpolate_locs(locs,coords,kinterp): """ Function to provide interpolate locations on a given axis interpolate_locs(locs,axis,kinterp) locs= locations to interpolate coords= axis values with the reference of coordinates kinterp: kind of interpolation 'lin': linear >>> coordinates = np.arange((10), dtype=np.float) >>> values = np.array([-1.2, 2.4, 5.6, 7.8, 12.0]) >>> interpolate_locs(values,coordinates,'lin') [ -1.2 2.4 5.6 7.8 13. ] >>> coordinates[0] = 0.5 >>> coordinates[2] = 2.5 >>> interpolate_locs(values,coordinates,'lin') [ -3.4 1.93333333 5.6 7.8 13. ] """ fname = 'interpolate_locs' if locs == 'h': print fname + '_____________________________________________________________' print interpolate_locs.__doc__ quit() Nlocs = locs.shape[0] Ncoords = coords.shape[0] dcoords = coords[Ncoords-1] - coords[0] intlocs = np.zeros((Nlocs), dtype=np.float) minc = np.min(coords) maxc = np.max(coords) for iloc in range(Nlocs): for icor in range(Ncoords-1): if locs[iloc] < minc and dcoords > 0.: a = 0. b = 1. / (coords[1] - coords[0]) c = coords[0] elif locs[iloc] > maxc and dcoords > 0.: a = (Ncoords-1)*1. b = 1. / (coords[Ncoords-1] - coords[Ncoords-2]) c = coords[Ncoords-2] elif locs[iloc] < minc and dcoords < 0.: a = (Ncoords-1)*1. b = 1. / (coords[Ncoords-1] - coords[Ncoords-2]) c = coords[Ncoords-2] elif locs[iloc] > maxc and dcoords < 0.: a = 0. b = 1. / (coords[1] - coords[0]) c = coords[0] elif locs[iloc] >= coords[icor] and locs[iloc] < coords[icor+1] and dcoords > 0.: a = icor*1. b = 1. / (coords[icor+1] - coords[icor]) c = coords[icor] print coords[icor], locs[iloc], coords[icor+1], ':', icor, '->', a, b elif locs[iloc] <= coords[icor] and locs[iloc] > coords[icor+1] and dcoords < 0.: a = icor*1. b = 1. / (coords[icor+1] - coords[icor]) c = coords[icor] if kinterp == 'lin': intlocs[iloc] = a + (locs[iloc] - c)*b else: print errormsg print ' ' + fname + ": interpolation kind '" + kinterp + "' not ready !!!!!" quit(-1) return intlocs def vertical_interpolation2D(varb,vart,zorigvals,znewval,kinterp): """ Function to vertically integrate a 3D variable vertical_interpolation2D(varb,vart,zorigvals,znewval) varb= values at the base of the interval of interpolation (2D field) vart= values at the top of the interval of interpolation (2D field) zorigvals= pair of original values (2, 2D field) znewval= new value to interpolate Possible cases: zorigvals[0,:,:] <= znewval < zorigvals[1,:,:] znewval <= zorigvals[0,:,:] < zorigvals[1,:,:] zorigvals[0,:,:] < zorigvals[1,:,:] <= znewval kinterp: kind of interpolation 'lin': linear >>> dx=5 >>> dy=7 >>> vals1 = np.ones((dy,dx), dtype=np.float) >>> vals2 = np.ones((dy,dx), dtype=np.float)*2. >>> zbase = np.zeros((2,dy,dx), dtype=np.float) >>> zbase[0,:,:] = 0.5 >>> zbase[1,:,:] = 1. >>> vertical_interpolation2D(vals1,vals2, zbase, newz,'lin') [[ 1.5 1.5 1.5 1.5 1.5] [ 1.5 1.5 1.5 1.5 1.5] [ 1.5 1.5 1.5 1.5 1.5] [ 1.5 1.5 1.5 1.5 1.5] [ 1.5 1.5 1.5 1.5 1.5] [ 1.5 1.5 1.5 1.5 1.5] [ 1.5 1.5 1.5 1.5 1.5]] """ fname = 'vertical_interpolation2D' if varb == 'h': print fname + '_____________________________________________________________' print vertical_interpolation2D.__doc__ quit() newvar = np.zeros((varb.shape), dtype=np.float) if kinterp == 'lin': ## print ' ' + fname + ' Vertical linear interpolation at',znewval # Standard linear interpolation (y = a + b*incz) # a = zorig[0], b = (vart-varb)/(zorig[1]-zorig[0])), incz = znewval - zorig[0] a = varb b = np.where(zorigvals[1,:,:] == zorigvals[0,:,:], 0., \ (vart - varb)/(zorigvals[1,:,:] - zorigvals[0,:,:])) incz = np.ones((varb.shape), dtype=np.float)*znewval - zorigvals[0,:,:] newvar = a + b*incz # Too code for not be used... but maybe? # dx=varb.shape[1] # dy=varb.shape[0] # for j in range(dy): # for i in range(dx): # if zorigvals[1,j,i] == zorigvals[0,j,i]: print 'equals!', \ # zorigvals[1,j,i], zorigvals[0,j,i],':',newvar[j,i],'@',vart[j,i] # if zorigvals[0,j,i] != zorigvals[0,j,i]: print '0 Nan', \ # zorigvals[0,j,i],':',newvar[j,i],'@',vart[j,i] # if zorigvals[1,j,i] != zorigvals[1,j,i]: print '1 Nan', \ # zorigvals[1,j,i],':',newvar[j,i],'@',vart[j,i] # if zorigvals[0,j,i] is None: print '0 None', zorigvals[0,j,i],':', \ # newvar[j,i],'@',vart[j,i] # if zorigvals[1,j,i] is None: print '1 None', zorigvals[1,j,i],':', \ # newvar[j,i],'@',vart[j,i] else: print errormsg print ' ' + fname + ": interpolation kind '" + kinterp + "' not ready !!!!!" quit(-1) return newvar #dx=5 #dy=7 #vals1 = np.ones((dy,dx), dtype=np.float) #vals2 = np.ones((dy,dx), dtype=np.float)*2. #zbase = np.zeros((2,dy,dx), dtype=np.float) #zbase[0,:,:] = 0.5 #for i in range(dx): # for j in range(dy): # zbase[1,j,i] = np.sqrt((j-dy/2.)**2. + (i-dx/2.)**2.) / \ # np.sqrt((dy/2.)**2.+(dy/2.)**2.) + 1. #zbase[1,:,:] = 1. #newz = 0.75 #print vertical_interpolation2D(vals1,vals2, zbase, newz,'lin') def vertical_interpolation(varb,vart,zorigvals,znewval,kinterp): """ Function to vertically integrate a 1D variable vertical_interpolation(varb,vart,zorigvals,znewval) varb= values at the base of the interval of interpolation vart= values at the top of the interval of interpolation zorigvals= pair of original values (2) znewval= new value to interpolate Possible cases: zorigvals[0,:] <= znewval < zorigvals[1,:] znewval <= zorigvals[0,:] < zorigvals[1,:] zorigvals[0,:] < zorigvals[1,:] <= znewval kinterp: kind of interpolation 'lin': linear >>> dx=5 >>> vals1 = np.ones((dx), dtype=np.float) >>> vals2 = np.ones((dx), dtype=np.float)*2. >>> zbase = np.zeros((2,dx), dtype=np.float) >>> zbase[0,:] = 0.5 >>> for i in range(dx): >>> zbase[1,i] = i + 1. >>> newz = 0.75 >>> vertical_interpolation2D(vals1,vals2, zbase, newz,'lin') [ 1.5 1.16666667 1.1 1.07142857 1.05555556] """ fname = 'vertical_interpolation' if varb == 'h': print fname + '_____________________________________________________________' print vertical_interpolation.__doc__ quit() newvar = np.zeros((varb.shape), dtype=np.float) if kinterp == 'lin': ## print ' ' + fname + ' Vertical linear interpolation at',znewval # Standard linear interpolation (y = a + b*incz) # a = zorig[0], b = (vart-varb)/(zorig[1]-zorig[0])), incz = znewval - zorig[0] a = varb b = np.where(zorigvals[1,:] == zorigvals[0,:], 0., \ (vart - varb)/(zorigvals[1,:] - zorigvals[0,:])) incz = np.ones((varb.shape), dtype=np.float)*znewval - zorigvals[0,:] newvar = a + b*incz else: print errormsg print ' ' + fname + ": interpolation kind '" + kinterp + "' not ready !!!!!" quit(-1) return newvar def interpolate_3D(zorigvs, varv, zintvs, kint): """ Function to interpolate a 3D variable interpolate_3D(zintvs, varv, zorigvals) zorigvs= original 3D z values varv= 3D variable to interpolate zintvs= new series of z values to interpolate kint= kind of interpolation 'lin': linear """ fname = 'interpolate_3D' if zorigvs == 'h': print fname + '_____________________________________________________________' print interpolate_3D.__doc__ quit() Ninv = len(zintvs) dimv = varv.shape # Sense of the original z-variable Lzorig = zorigvs.shape[0] incz = zorigvs[Lzorig-1,0,0] - zorigvs[0,0,0] varin = np.zeros((Ninv,dimv[1],dimv[2]), dtype=np.float) for ilev in range(Ninv): # print ' vertical interpolate level: ',zintvs[ilev] valinf = np.zeros((dimv[1], dimv[2]), dtype=np.float) valsup = np.zeros((dimv[1], dimv[2]), dtype=np.float) zorigv = np.zeros((2,dimv[1], dimv[2]), dtype=np.float) for j in range(valinf.shape[0]): for i in range(valinf.shape[1]): if np.min(zorigvs[:,j,i]) > zintvs[ilev] and incz > 0.: valinf[j,i] = varv[0,j,i] valsup[j,i] = varv[1,j,i] zorigv[0,j,i] = zorigvs[0,j,i] zorigv[1,j,i] = zorigvs[1,j,i] elif np.max(zorigvs[:,j,i]) < zintvs[ilev] and incz < 0.: valinf[j,i] = varv[0,j,i] valsup[j,i] = varv[1,j,i] zorigv[0,j,i] = zorigvs[0,j,i] zorigv[1,j,i] = zorigvs[1,j,i] elif np.max(zorigvs[:,j,i]) < zintvs[ilev] and incz > 0.: valinf[j,i] = varv[Lzorig-2,j,i] valsup[j,i] = varv[Lzorig-1,j,i] zorigv[0,j,i] = zorigvs[Lzorig-2,j,i] zorigv[1,j,i] = zorigvs[Lzorig-1,j,i] elif np.min(zorigvs[:,j,i]) > zintvs[ilev] and incz < 0.: valinf[j,i] = varv[Lzorig-2,j,i] valsup[j,i] = varv[Lzorig-1,j,i] zorigv[0,j,i] = zorigvs[Lzorig-2,j,i] zorigv[1,j,i] = zorigvs[Lzorig-1,j,i] # print 'top: ',i,j,':',zorigv[0,j,i], zintvs[ilev], zorigv[1,j,i] else: for z in range(Lzorig-1): if (zorigvs[z,j,i]-zintvs[ilev])*(zorigvs[z+1,j,i]- \ zintvs[ilev]) <= 0.: valinf[j,i] = varv[z,j,i] valsup[j,i] = varv[z+1,j,i] zorigv[0,j,i] = zorigvs[z,j,i] zorigv[1,j,i] = zorigvs[z+1,j,i] break # print 'zorigvs: ',zorigvs[:,0,0] # print ' valinf:', valinf[0,0] # print ' valsup:', valsup[0,0] # print ' zorigv 0:',zorigv[0,0,0] # print ' zorigv 1:',zorigv[1,0,0] varin[ilev,:,:] = vertical_interpolation2D(valinf,valsup,zorigv, \ zintvs[ilev], kint) # print ' varin:',varin[ilev,0,0] # quit() # quit() return varin def interpolate_2D(zorigvs0, varv0, zdim, zintvs, kint): """ Function to interpolate a 2D variable interpolate_2D(zintvs, varv, zorigvals) zorigvs= original 2D z values varv= 2D variable to interpolate zdim= number of the dimension with the z-values zintvs= new series of z values to interpolate kint= kind of interpolation 'lin': linear """ fname = 'interpolate_2D' if zorigvs0 == 'h': print fname + '_____________________________________________________________' print interpolate_2D.__doc__ quit() Ninv = len(zintvs) if zdim == 0: varv = varv0 zorigvs = zorigvs0 else: varv = np.transpose(varv0) zorigvs = np.transpose(zorigvs0) dimv = varv.shape # Sense of the original z-variable Lzorig = zorigvs.shape[0] incz = zorigvs[Lzorig-1,0] - zorigvs[0,0] varin = np.zeros((Ninv,dimv[1]), dtype=np.float) for ilev in range(Ninv): valinf = np.zeros((dimv[1]), dtype=np.float) valsup = np.zeros((dimv[1]), dtype=np.float) zorigv = np.zeros((2,dimv[1]), dtype=np.float) for i in range(valinf.shape[0]): if np.min(zorigvs[:,i]) > zintvs[ilev] and incz > 0.: valinf[i] = varv[0,i] valsup[i] = varv[1,i] zorigv[0,i] = zorigvs[0,i] zorigv[1,i] = zorigvs[1,i] elif np.max(zorigvs[:,i]) < zintvs[ilev] and incz < 0.: valinf[i] = varv[0,i] valsup[i] = varv[1,i] zorigv[0,i] = zorigvs[0,i] zorigv[1,i] = zorigvs[1,i] elif np.max(zorigvs[:,i]) < zintvs[ilev] and incz > 0.: valinf[i] = varv[Lzorig-2,i] valsup[i] = varv[Lzorig-1,i] zorigv[0,i] = zorigvs[Lzorig-2,i] zorigv[1,i] = zorigvs[Lzorig-1,i] elif np.min(zorigvs[:,i]) > zintvs[ilev] and incz < 0.: valinf[i] = varv[Lzorig-2,i] valsup[i] = varv[Lzorig-1,i] zorigv[0,i] = zorigvs[Lzorig-2,i] zorigv[1,i] = zorigvs[Lzorig-1,i] else: for z in range(Lzorig-1): if (zorigvs[z,i]-zintvs[ilev])*(zorigvs[z+1,i]- \ zintvs[ilev]) <= 0.: valinf[i] = varv[z,i] valsup[i] = varv[z+1,i] zorigv[0,i] = zorigvs[z,i] zorigv[1,i] = zorigvs[z+1,i] break varin[ilev,:] = vertical_interpolation(valinf,valsup,zorigv, \ zintvs[ilev], kint) if zdim == 0: varin0 = varin else: varin0 = np.transpose(varin) return varin0 def list_toVec(listv, kind): """ Function to transform from a list of values to a vector list_toVec(listv, kind) listv= list with the values kind= kind of values >>> list_toVec(['1', 2, '3', 4, 5, '6.9', 7, 9], 'npfloat') [ 1. 2. 3. 4. 5. 6.9 7. 9. ] """ fname = 'list_toVec' if listv == 'h': print fname + '_____________________________________________________________' print list_toVec.__doc__ quit() Nvals = len(listv) if kind == 'npfloat': vecv = np.zeros((Nvals), dtype=np.float) for iv in range(Nvals): vecv[iv] = np.float(listv[iv]) else: print errormsg print ' ' + fname + ': type of value "' + kind + '" not ready!!' quit(-1) return vecv def running_mean(values, run): """ Function to compute a running mean of a series of values running_mean(vals, int) [vals]: series of values [run]: interval to use to compute the running mean NOTE: resultant size would be the same but with None except at [run/2,size(vals)-run/2] >>> running_mean(np.arange(100),10) [None None None None None 4.5 5.5 6.5 7.5 8.5 9.5 10.5 11.5 12.5 13.5 14.5 15.5 16.5 17.5 18.5 19.5 20.5 21.5 22.5 23.5 24.5 25.5 26.5 27.5 28.5 29.5 30.5 31.5 32.5 33.5 34.5 35.5 36.5 37.5 38.5 39.5 40.5 41.5 42.5 43.5 44.5 45.5 46.5 47.5 48.5 49.5 50.5 51.5 52.5 53.5 54.5 55.5 56.5 57.5 58.5 59.5 60.5 61.5 62.5 63.5 64.5 65.5 66.5 67.5 68.5 69.5 70.5 71.5 72.5 73.5 74.5 75.5 76.5 77.5 78.5 79.5 80.5 81.5 82.5 83.5 84.5 85.5 86.5 87.5 88.5 89.5 90.5 91.5 92.5 93.5 None None None None None] """ fname = 'running_mean' if values == 'h': print fname + '_____________________________________________________________' print running_mean.__doc__ quit() runmean = np.ones((len(values)), dtype=np.float)*fillValue for ip in range(run/2,len(values)-run/2): runmean[ip] = np.mean(values[ip-run/2:ip+run/2]) runmean = np.where(runmean == fillValue, None, runmean) return runmean def slice_variable(varobj, dimslice): """ Function to return a slice of a given variable according to values to its dimensions slice_variable(varobj, dimslice) varobj= object wit the variable dimslice= [[dimname1]:[value1]|[[dimname2]:[value2], ...] pairs of dimension [value]: * [integer]: which value of the dimension * -1: all along the dimension * -9: last value of the dimension * [beg]@[end] slice from [beg] to [end] """ fname = 'slice_variable' if varobj == 'h': print fname + '_____________________________________________________________' print slice_variable.__doc__ quit() vardims = varobj.dimensions Ndimvar = len(vardims) Ndimcut = len(dimslice.split('|')) if Ndimcut == 0: Ndimcut = 1 dimcut = list(dimslice) dimsl = dimslice.split('|') varvalsdim = [] dimnslice = [] for idd in range(Ndimvar): found = False for idc in range(Ndimcut): dimcutn = dimsl[idc].split(':')[0] dimcutv = dimsl[idc].split(':')[1] if vardims[idd] == dimcutn: posfrac = dimcutv.find('@') if posfrac != -1: inifrac = int(dimcutv.split('@')[0]) endfrac = int(dimcutv.split('@')[1]) varvalsdim.append(slice(inifrac,endfrac)) if endfrac > varobj.shape[idd]: print errormsg print ' ' + fname + ": value for dimension '" + \ dimcutn + "' is too big (", endfrac, '>', \ varobj.shape[idd],'[dimension length] ) !!' quit(-1) dimnslice.append(vardims[idd]) else: if int(dimcutv) == -1: varvalsdim.append(slice(0,varobj.shape[idd])) dimnslice.append(vardims[idd]) elif int(dimcutv) == -9: varvalsdim.append(int(varobj.shape[idd])-1) else: if int(dimcutv) >= varobj.shape[idd]: print errormsg print ' ' + fname + ": value for dimension '" + \ dimcutn + "' is too big (", int(dimcutv), '>', \ varobj.shape[idd],'[dimension length] ) !!' quit(-1) varvalsdim.append(int(dimcutv)) found = True break if not found and not searchInlist(dimnslice,vardims[idd]): varvalsdim.append(slice(0,varobj.shape[idd])) dimnslice.append(vardims[idd]) print 'orig shape:',varobj[:].shape,'sliced:',varobj[tuple(varvalsdim)].shape varvalues = np.squeeze(varobj[tuple(varvalsdim)]) print 'varvalues shape:',varvalues.shape return varvalues, dimnslice def operation_alongdims(ovar,dimvals,dimsoper,opkind): """ Function to operate along different dimensions of a variable operation_alongdims(ovar,dimvals,opkind) ovar= netCDF variable object dimvals= [dimname1]:[val1]|[dimdname2]:[val2]|[...[dimnameN]:[valN]] [value]: * [integer]: which value of the dimension * -1: all along the dimension * [beg]:[end] slice from [beg] to [end] dimsoper= [dimname1]:[dimname2]:[...[dimnameN]] names of the dimensions along which operation has to be done opkind= operation to perform along the dimensions with a range: max, mean, mean2, min, sum """ fname = 'operation_alongdims' if ovar == 'h': print fname + '_____________________________________________________________' print operation_alongdims.__doc__ quit() vardims = ovar.dimensions Nvardims = len(vardims) cutdims = dimvals.split('|') Ncutdims = len(cutdims) if Nvardims != Ncutdims: print warnmsg print ' ' + fname + ': cutting for',Ncutdims,'but variable has:',Nvardims, \ '!!' # quit(-1) cutdimvals = '' for idv in range(Nvardims): found = False Lcutdimvals = len(cutdimvals) for idc in range(Ncutdims): if vardims[idv] == cutdims[idc].split(':')[0]: if Lcutdimvals > 0: cutdimvals = cutdimvals + '|' + cutdims[idc] else: cutdimvals = cutdims[idc] found = True break if not found: # print errormsg # print ' ' + fname + ": cutting by '",cutdims[idc].split(':')[0], \ # "' but variable has not it !!!" # print ' variable dims:',vardims print warnmsg print ' ' + fname + ": variable has dimension '",vardims[idv], \ "' but is not cut taking all length !!!" if Lcutdimvals > 0: cutdimvals = cutdimvals + '|' + vardims[idv] + ':-1' else: cutdimvals = vardims[idv] + ':-1' # quit(-1) slicedvar, dimslice = slice_variable(ovar, cutdimvals) # operation dimnsoper = dimsoper.split(':') if opkind == 'mean2': origvar = slicedvar*slicedvar else: origvar = slicedvar.copy() dimsnewvar = [] Ndimslice = len(dimslice) Ndimsoper = len(dimnsoper) noper=0 for ids in range(Ndimslice): found = False for ido in range(Ndimsoper): if dimslice[ids] == dimnsoper[ido]: if opkind == 'max': varoper = np.max(origvar, axis=ids-noper) elif opkind == 'mean': varoper = np.mean(origvar, axis=ids-noper) elif opkind == 'mean2': varoper = np.mean(origvar, axis=ids-noper) elif opkind == 'min': varoper = np.min(origvar, axis=ids-noper) elif opkind == 'sum': varoper = np.sum(origvar, axis=ids-noper) else: print errormsg print ' ' + fname + ': operation "' + opkind + '" not ready!!' quit(-1) del(origvar) origvar = varoper.copy() noper = noper + 1 found = True break if not found: dimsnewvar.append(dimslice[ids]) return origvar, dimsnewvar def var_3desc(ovar): """ Function to provide std_name, long_name and units from an object variable ovar= object variable """ fname = 'var_desc' varattrs = ovar.ncattrs() if searchInlist(varattrs,'std_name'): stdn = ovar.getncattr('std_name') else: vvalues = variables_values(ovar._name) stdn = vvalues[1] if searchInlist(varattrs,'long_name'): lonn = ovar.getncattr('long_name') else: lonn = vvalues[4].replace('|',' ') if searchInlist(varattrs,'units'): un = ovar.getncattr('units') else: un = vvalues[5] return stdn, lonn, un def file_oper_alongdims(values, ncfile, varn): """ Function to operate a file along different dimensions of a variable file_oper_alongdims(values, ncfile, varn) values= [dimvals],[dimsoper],[opkind],[dimvn] [dimvals]; [dimname1]:[val1]|[dimdname2]:[val2]|[...[dimnameN]:[valN]] [value]; * [integer]: which value of the dimension * -1: all along the dimension * [beg]:[end] slice from [beg] to [end] [dimsoper]; [dimname1]:[dimname2]:[...[dimnameN]] names of the dimensions along which operation has to be done [opkind]; operation to perform along the dimensions with a range: max, mean, mean2, min, sum [dimvn]; [varname1]:[varname2]:[...[varnameM]] variables with the values of the dimensions of the resultant operation ncfile= netCDF file varn= ',' list of variable names ('all' for all) """ fname = 'file_oper_alongdims' if values == 'h': print fname + '_____________________________________________________________' print file_oper_alongdims.__doc__ quit() dimvals = values.split(',')[0] dimsoper = values.split(',')[1] operkind = values.split(',')[2] dimvn = values.split(',')[3] dimsoperS = numVector_String(dimsoper.split(':'),', ') objnc = NetCDFFile(ncfile, 'r') if varn.find(',') != -1: varns = varn.split(',') else: if varn == 'all': varns = objnc.variables else: varns = [varn] ofile = 'file_oper_alongdims_' + operkind + '.nc' if dimsoper.find(':') != -1: dimstouse = dimsoper.split(':') else: dimstouse = [dimsoper] # Creation of output file ## objnewnc = NetCDFFile(ofile, 'w') print ' ' + fname + ": '" + operkind + "' with:",dimsoper,'_______' for vn in varns: print "'" + vn + "' ... .. ." if not objnc.variables.has_key(vn): print errormsg print ' ' + fname + ': netCDF file "' + ncfile + \ '" does not have variable "' + vn + '" !!!!' quit(-1) # Variable objvar = objnc.variables[vn] allfound = True for idimt in dimstouse: if not searchInlist(objvar.dimensions, idimt): allfound = False break if not allfound: print warnmsg print ' ' + fname + ": variable '" + vn + "' (", objvar.dimensions, \ ') does not have all the equired dimensions:', dimstouse, '!!' print ' skipping it' else: newvarv, newvardimns=operation_alongdims(objvar,dimvals,dimsoper,operkind) # Dimensions for idim in range(len(newvardimns)): objdim = objnc.dimensions[newvardimns[idim]] if not objnewnc.dimensions.has_key(newvardimns[idim]): if objdim.isunlimited(): objnewnc.createDimension(newvardimns[idim], None) else: objnewnc.createDimension(newvardimns[idim], len(objdim)) # Variables with dimension values vardims = dimvn.split(':') for ivdim in range(len(vardims)): if not objnewnc.variables.has_key(vardims[ivdim]): objvdim = objnc.variables[vardims[ivdim]] vdimtype = objvdim.dtype newdimvarv, newdimvardimns=operation_alongdims(objvdim,dimvals,dimsoper,operkind) # Old method introducing new dimension if needed ## dimsvdim = objvdim.dimensions ## dimvslice = [] ## dimvdims = [] ## for idmvd in range(len(dimsvdim)): ## found = False ## for idimv in range(len(newvardimns)): ## if dimsvdim[idmvd] == newvardimns[idimv]: ## dimvslice.append(slice(0,objvdim.shape[idmvd])) ## dimvdims.append(dimsvdim[idmvd]) ## found = True ## break ## if not found: ## print warnmsg ## print ' ' + fname + ": dimension '" + dimsvdim[idmvd] + \ ## "' not found !!" ## print " adding it for variable dimension: '" + \ ## vardims[ivdim] + "'" ## if not objnewnc.dimensions.has_key(dimsvdim[idmvd]): ## dsize = len(objnc.dimensions[dimsvdim[idmvd]]) ## objnewnc.createDimension(dimsvdim[idmvd], dsize) ## dimvslice.append(slice(len(objnc.dimensions[dimsvdim[idmvd]]))) ## dimvdims.append(dimsvdim[idmvd]) newvar = objnewnc.createVariable(vardims[ivdim], vdimtype, \ tuple(newdimvardimns)) newvar[:] = newdimvarv dimvattrs = objvdim.ncattrs() for attrn in dimvattrs: attrv = objvdim.getncattr(attrn) newattr = set_attribute(newvar,attrn,attrv) newattr = set_attribute(newvar, 'operation:', operkind + \ ' along ' + dimsoperS) # new variable oldvarattr = objvar.ncattrs() varname = variables_values(vn)[0] if searchInlist(oldvarattr, 'standard_name'): stdname = objvar.getncattr('standard_name') else: stdname = variables_values(vn)[1] if searchInlist(oldvarattr, 'long_name'): lname = objvar.getncattr('long_name') else: lname = variables_values(vn)[4].replace('|',' ') if searchInlist(oldvarattr, 'units'): uname = objvar.getncattr('units') else: uname = variables_values(vn)[5] newvar = objnewnc.createVariable(varname + operkind, 'f4', tuple(newvardimns)) newattr = basicvardef(newvar, stdname + operkind, lname + ' '+ operkind +\ ' along ' + dimsoperS, uname) newvar[:] = newvarv for idim in range(len(objvar.shape)): dimn = dimvals.split('|')[idim].split(':')[0] dimv = '{:d}'.format(len(objnc.dimensions[dimn])) if idim == 0: origdimsS = dimn + '(' + dimv + ')' else: origdimsS = origdimsS + ' ' + dimn + '(' + dimv + ')' newattr = set_attribute(newvar, 'orig_dimensions', origdimsS) # global attributes objnewnc.setncattr('author', 'L. Fita') objnewnc.setncattr('institution', 'Laboratire de Meteorologie Dynamique') objnewnc.setncattr('university', 'Pierre Marie Curie - Jussieu') objnewnc.setncattr('center', 'Centre National de Recherches Scientifiques') objnewnc.setncattr('city', 'Paris') objnewnc.setncattr('country', 'France') objnewnc.setncattr('script', 'nc_var_tools.py') objnewnc.setncattr('function', 'file_oper_alongdims') objnewnc.setncattr('version', '1.0') objnewnc.setncattr('data_file', ncfile) gorigattrs = objnc.ncattrs() for attr in gorigattrs: attrv = objnc.getncattr(attr) atvar = set_attribute(objnewnc, attr, attrv) objnc.close() objnewnc.sync() objnewnc.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return #file_oper_alongdims('Time:-1|bottom_top:-1|south_north:-1|west_east:-1,bottom_top,sum,Times:XLAT:XLONG', # '/home/lluis/PY/wrfout_d01_2001-11-11_00:00:00.tests', 'QVAPOR') #file_oper_alongdims('time:-1|z:-1|x:-1|y:-1,time:y,mean,pressure:lat', '/home/lluis/etudes/WRF_LMDZ/WaquaL/WRF_LMDZ/AR40/vertical_interpolation_WRFp.nc', 'WRFt') def WRF_CFtime_creation(values, ncfile, varn): """ Function to add a CF-convention time unit in a WRF file WRF_CFtime_creation(values, ncfile, varn) [values]= [refdate],[units] [refdate]: reference date to compute the times in [YYYY][MM][DD][HH][MI][SS] format [units]: time units to use: 'days','hours','minutes','seconds' [ncfile]= WRF file to add the CF-convention time variable [varn]= name of the variable time to add """ fname='WRF_CFtime_creation' if values == 'h': print fname + '_____________________________________________________________' print WRF_CFtime_creation.__doc__ quit() refdate=values.split(',')[0] tunitsval=values.split(',')[1] wrfnc = NetCDFFile(ncfile, 'a') if wrfnc.variables.has_key(varn): print warnmsg print ' ' + fname + ": WRF file '" + ncfile + "' already has variable '" + \ varn + "' !!!" print ' overwritting values' timeobj = wrfnc.variables['Times'] timewrfv = timeobj[:] yrref=refdate[0:4] monref=refdate[4:6] dayref=refdate[6:8] horref=refdate[8:10] minref=refdate[10:12] secref=refdate[12:14] refdateS = yrref + '-' + monref + '-' + dayref + ' ' + horref + ':' + minref + \ ':' + secref dt = timeobj.shape[0] cftimes = np.zeros((dt), dtype=np.float) for it in range(dt): wrfdates = datetimeStr_conversion(timewrfv[it,:],'WRFdatetime', 'matYmdHMS') cftimes[it] = realdatetime1_CFcompilant(wrfdates, refdate, tunitsval) tunits = tunitsval + ' since ' + refdateS if wrfnc.variables.has_key(varn): newvar = wrfnc.variables[varn] else: newvar = wrfnc.createVariable('time','f4',('Time')) newattr = basicvardef(newvar, 'time','time',tunits) newvar[:] = cftimes newattr = set_attribute(newvar, 'calendar', 'standard') wrfnc.sync() wrfnc.close() return #WRF_CFtime_creation('19491201000000,hours', 'wrfout_d01_1995-01-15_00:00:00', 'time') def WRF_CFxtime_creation(values, ncfile, varn): """ Function to add a CF-convention time unit in a WRF file using variable 'XTIME' WRF_CFxtime_creation(values, ncfile, varn) [values]= [refdate],[units] [refdate]: reference date to compute the times in [YYYY][MM][DD][HH][MI][SS] format [units]: time units to use: 'days','hours','minutes','seconds' [ncfile]= WRF file to add the CF-convention time variable [varn]= name of the variable time to add """ fname='WRF_CFxtime_creation' if values == 'h': print fname + '_____________________________________________________________' print WRF_CFxtime_creation.__doc__ quit() refdate=values.split(',')[0] tunitsval=values.split(',')[1] wrfnc = NetCDFFile(ncfile, 'a') if not wrfnc.variables.has_key('XTIME'): print errormsg print ' ' + fname + ": WRF file '" + ncfile + "' does not have variable '" +\ "XTIME' !!" quit(-1) if wrfnc.variables.has_key(varn): print warnmsg print ' ' + fname + ": WRF file '" + ncfile + "' already has variable '" + \ varn + "' !!!" print ' over-writting values' if not searchInlist(wrfnc.ncattrs(), 'SIMULATION_START_DATE'): print errormsg print ' ' + fname + ": WRF file '" + ncfile + "' does not have attribute '"+\ "'SIMULATION_START_DATE' !!!" quit(-1) startdate = wrfnc.SIMULATION_START_DATE timeobj = wrfnc.variables['XTIME'] timewrfv = timeobj[:] yrref=refdate[0:4] monref=refdate[4:6] dayref=refdate[6:8] horref=refdate[8:10] minref=refdate[10:12] secref=refdate[12:14] refdateS = yrref + '-' + monref + '-' + dayref + '_' + horref + ':' + minref + \ ':' + secref dt = timeobj.shape[0] cftimes = np.zeros((dt), dtype=np.float) for it in range(dt): wrfdates = datetimeStr_conversion(str(timewrfv[it]) +',minutes since ' + \ startdate, 'cfTime', 'matYmdHMS') cftimes[it] = realdatetime1_CFcompilant(wrfdates, refdate, tunitsval) tunits = tunitsval + ' since ' + refdateS if wrfnc.variables.has_key('time'): newvar = wrfnc.variables['time'] else: newvar = wrfnc.createVariable('time','f4',('Time')) newattr = basicvardef(newvar, 'time','time',tunits) newvar[:] = cftimes newattr = set_attribute(newvar, 'calendar', 'standard') wrfnc.sync() wrfnc.close() return def insert_variable(onc, varn, varv, vardims, varvdims, onewnc): """ Function to insert a variable in an existing file insert_variable(onc, varn, varv, vardims, onewnc) [onc]= object of the original file [varn]= name of the variable [varv]= values of the variable [vardims]= dimension names of the variable [varvdims]= name of the variables with the values of the dimensions [onewnc]= object of the new file """ fname = 'insert_variable' if onc == 'h': print fname + '_____________________________________________________________' print insert_variable.__doc__ quit() ininewncdims = onewnc.dimensions ininewncvars = onewnc.variables # Creation of dimensions for idim in vardims: if not ininewncdims.has_key(idim): if not onc.dimensions.has_key(idim): print errormsg print ' ' + fname + ': initial file does not have dimension: "' + \ idim + '"' quit(-1) else: refdimv = onc.dimensions[idim] if refdimv.isunlimited(): dimsize = None else: dimsize = len(refdimv) onewnc.createDimension(idim, dimsize) # Variables with dimension values for ivdim in varvdims: if not ininewncvars.has_key(ivdim): if not onc.variables.has_key(ivdim): print errormsg print ' ' + fname + ': initial file does not have variable: "' + \ ivdim + '"' quit(-1) else: refvdimv = onc.variables[ivdim] refvdimvals = refvdimv[:] for dmn in refvdimv.dimensions: if not onewnc.dimensions.has_key(dmn): print 'dmn:',dmn if onc.dimensions[dmn].isunlimited(): newdim = onewnc.createDimension(dmn, None ) else: newdim = onewnc.createDimension(dmn,len(onc.dimensions[dmn])) newvar = onewnc.createVariable(ivdim, refvdimv.dtype, \ (refvdimv.dimensions)) varattrs = refvdimv.ncattrs() newvar[:] = refvdimvals for attrn in varattrs: attrv = refvdimv.getncattr(attrn) newattr = set_attribute(newvar, attrn, attrv) # Variable varvalues = variables_values(varn) if onewnc.variables.has_key(varvalues[0]): print warnmsg print ' ' + fname + ": file already has variable '" + varvalues[0] + "' !!" print ' re-writting values' newvar = onewnc.variables[varvalues[0]] else: newvar = onewnc.createVariable(varvalues[0], 'f4', (vardims)) newattr = basicvardef(newvar, varvalues[1], varvalues[4].replace('|',' '), \ varvalues[5]) print ' ' + fname + ': shape; newvar',newvar.shape,'varv:',varv.shape newvar[:] = varv return def var_dim_dimv(dimvar, dimns, dimvns): """ Function to bring back which variables with the values should be use for a given variable var_dim_dimv(ovar, dimns, dimvns) [dimvar]= list of the dimensions of the variable [dimns]= names of the list of available dimensions [dimvns]= correspondant name of variables with the values for the dimensions >>> dimsvar = ['t', 'y', 'x'] >>> dimnames = ['t', 'z', 'l', 'y' , 'x'] >>> dimvarnames = ['time', 'pressure', 'soil_levels', 'lat' ,' lon'] >>> var_dim_dimv(dimsvar, dimnames, dimvarnames) ['time', 'lat', ' lon'] """ fname = 'var_dim_dimv' if dimvar == 'h': print fname + '_____________________________________________________________' print var_dim_dimv.__doc__ quit() dimvvar = list(dimvns) Ndimvar = len(dimvar) Ndimns = len(dimns) for indimv in range(Ndimns): for indima in range(Ndimvar): found = False if dimns[indimv] == dimvar[indima]: found = True break if not found: dimvvar.remove(dimvns[indimv]) return dimvvar def WRF_CFlonlat_creation(values, ncfile, varn): """ Function to add a CF-convention longitude/latitude variables in a WRF file WRF_CFlon_creation(values, ncfile, varn) [values]= [lonname],[latname] names of the variables to add (standard names 'longitude', 'latitude') [ncfile]= WRF file to add the CF-convention time variable [varn]= [lonname],[latname] name of the variables to use """ fname='WRF_CFlonlat_creation' if values == 'h': print fname + '_____________________________________________________________' print WRF_CFlonlat_creation.__doc__ quit() cflonn = values.split(',')[0] cflatn = values.split(',')[1] wlonn = varn.split(',')[0] wlatn = varn.split(',')[1] wrfnc = NetCDFFile(ncfile, 'a') if not searchInlist(wrfnc.variables.keys(),wlonn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not lon variable '" + \ wlonn + "' !!!" quit(-1) if not searchInlist(wrfnc.variables.keys(),wlatn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not lat variable '" + \ wlatn + "' !!!" quit(-1) if not searchInlist(wrfnc.dimensions.keys(),'west_east'): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not 'west_east' dimension !!!" quit(-1) if not searchInlist(wrfnc.dimensions.keys(),'south_north'): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not 'south_north' dimension !!!" quit(-1) lonobj = wrfnc.variables[wlonn] lonv = lonobj[:] latobj = wrfnc.variables[wlatn] latv = latobj[:] if len(lonv.shape) == 3: cflons = np.zeros((lonv.shape[1],lonv.shape[2]), dtype=np.float) cflons = np.where(lonv[0,:,:] < 0., lonv[0,:,:] + 360., lonv[0,:,:]) cflats = np.zeros((latv.shape[1],latv.shape[2]), dtype=np.float) cflats = latv[0,:,:] elif len(lonv.shape) == 2: cflons = np.zeros((lonv.shape), dtype=np.float) cflons = np.where(lonv < 0., lonv + 360., lonv) cflats = latv elif len(lonv.shape) == 1: cflons = np.zeros((lonv.shape), dtype=np.float) cflons = np.where(lonv < 0., lonv + 360., lonv) cflats = latv else: print errormsg print ' ' + fname + ": original longitude variable '" + varn + "' shape:" + \ lonv.shape + "not ready !!!" quit(-1) if len(lonv.shape) == 3: for ix in range(lonv.shape[2]): print ix, lonv[0,10,ix], cflons[10,ix] elif len(lonv.shape) == 2: for ix in range(lonv.shape[1]): print ix, lonv[10,ix], cflons[10,ix] if wrfnc.variables.has_key(cflonn): print warnmsg print ' ' + fname + ": file '" + ncfile + "' already has variable '" + \ cflonn + "' !!!" print ' over-writting values' newvar = wrfnc.variables[cflonn] else: newvar = wrfnc.createVariable(cflonn,'f4',('south_north','west_east')) newattr = basicvardef(newvar, 'longitude','longitude','degrees East') newvar[:] = cflons if wrfnc.variables.has_key(cflatn): print warnmsg print ' ' + fname + ": file '" + ncfile + "' already has variable '" + \ cflatn + "' !!!" print ' over-writting values' newvar = wrfnc.variables[cflatn] else: newvar = wrfnc.createVariable(cflatn,'f4',('south_north','west_east')) newattr = basicvardef(newvar, 'latitude','latitude','degrees North') newvar[:] = cflats wrfnc.sync() wrfnc.close() return #WRF_CFlon_creation('longitude', 'wrfout_d01_1980-03-01_00:00:00_Time_B0-E48-I1.nc', 'XLONG') def dimVar_creation(values, ncfile): """ Function to add a 1D variable with the size of a given dimension in a file dimVar_creation(values, ncfile) [values]= [dimname] name of the dimesion to use [ncfile]= file to add the variable """ fname='dimVar_creation' if values == 'h': print fname + '_____________________________________________________________' print dimVar_creation.__doc__ quit() ncobj = NetCDFFile(ncfile, 'a') varn = 'varDIM' + values if not searchInlist(ncobj.dimensions.keys(),values): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not '" + values + \ "' dimension !!" quit(-1) dimsize = len(ncobj.dimensions[values]) if ncobj.variables.has_key(varn): print warnmsg print ' ' + fname + ": file '" + ncfile + "' already has variable '" + \ varn + "' !!!" print ' over-writting values' newvar = ncobj.variables[varn] else: newvar = ncobj.createVariable(varn,'f4',(values)) newattr = basicvardef(newvar, varn,"variable from '" +values+ "' dimension" ,'-') newvar[:] = np.arange(dimsize)*1. ncobj.sync() ncobj.close() print fname + ": successfull creation of variable '" + varn + "' !!" return #dimVar_creation('bottom_top', 'wrfout_d01_1979-12-01_00:00:00_south_north_B3-E3-I1_west_east_B26-E26-I1.nc') def files_folder(folder,headfile): """ Function to retrieve a list of files from a folder [fold] and head [headfile] >>> files_folder('/media/data2/etudes/WRF_LMDZ/WL_HyMeX_HighRes/medic950116/', 'wrfout_d03') ['wrfout_d03_1995-01-13_02:00:00', 'wrfout_d03_1995-01-13_04:00:00', 'wrfout_d03_1995-01-13_06:00:00', 'wrfout_d03_1995-01-13_06:15:00', 'wrfout_d03_1995-01-13_08:15:00', 'wrfout_d03_1995-01-13_10:15:00', 'wrfout_d03_1995-01-13_12:15:00', 'wrfout_d03_1995-01-13_14:15:00', 'wrfout_d03_1995-01-13_16:15:00', (...) 'wrfout_d03_1995-01-17_18:15:00', 'wrfout_d03_1995-01-17_20:15:00', 'wrfout_d03_1995-01-17_22:15:00'] """ import subprocess as sub fname = 'files_folder' if folder == 'h': print fname + '_____________________________________________________________' print files_folder.__doc__ quit() # ins = folder + "/" + headfile + "*" ins = folder + "/" print 'ins:',ins files = sub.Popen(["/bin/ls","-1",ins], stdout=sub.PIPE) fileslist = files.communicate() listfiles0 = str(fileslist).split('\\n') # Filtering output Lheader=len(headfile) listfiles = [] for ifile in listfiles0: if ifile[0:Lheader] == headfile: listfiles.append(ifile) return listfiles def netcdf_fold_concatenation(values, ncfile, varn): """ Function to concatenate netCDF files in a given folder for a given set of variables netcdf_fold_concateation(values, ncfile, varn) [values]= [fold],[dimname] folder with the location of the netCDF files [fold]: folder with the location of the netCDF files [dimname]: dimension along which files should be concatenated [ncfile]= header of the name of the files to concatenate [ncfile]* [varn]= ',' separated list of variables to concatenate [var1],[var2],[...[varN]] or 'all' for all variables """ import subprocess as sub fname='netcdf_fold_concatenate' ofile = 'netcdf_fold_concatenated.nc' if values == 'h': print fname + '_____________________________________________________________' print netcdf_fold_concatenation.__doc__ quit() fold = values.split(',')[0] condim = values.split(',')[1] confiles = files_folder(fold,ncfile) Nfiles = len(confiles) print ' ' + fname +': concatenating:',Nfiles,'files' print confiles # Opening all files ncobjs = [] for filen in confiles: print 'charging: ',filen ncobjs.append(NetCDFFile(fold + '/' + filen, 'r')) # Looking for the new length of the concatenation dimension if not ncobjs[0].dimensions.has_key(condim): print errormsg print ' ' + fname + ": files do not have dimensions '" + condim + "' !!!" quit(-1) totcondim = 0 for ifile in range(Nfiles): ncobj = ncobjs[ifile] totcondim = totcondim + len(ncobjs[ifile].dimensions[condim]) print "total concatenated dimension '" + condim + "': ", totcondim # concatenated file creation ## newnc = NetCDFFile(ofile, 'w') # dimension creation ## if ncobjs[0].dimensions[condim].isunlimited(): newnc.createDimension(condim, None) else: newnc.createDimension(condim, totcondim) # Fake variable for the dimension print 'condim:',condim dims = [] dims.append(condim) newvar = newnc.createVariable('int' + condim, 'i4', tuple(dims)) newvar[:] = range(totcondim) # Looping variables ## if varn == 'all': desvars = ncobjs[0].variables else: desvars = varn.split(',') for dvar in desvars: if not ncobjs[0].variables.has_key(dvar): print errormsg print ' ' + fname + ": files do not have variable '" + dvar + "' !!!" quit(-1) for dvar in desvars: print ' ' + fname + ": concatenating '" + dvar + "'..." objvar = ncobjs[0].variables[dvar] # creation of dimensions vardims = objvar.dimensions for ndim in vardims: if not newnc.dimensions.has_key(ndim): vardimo = ncobjs[0].dimensions[ndim] if vardimo.isunlimited(): newnc.createDimension(ndim, None) else: newnc.createDimension(ndim, len(vardimo)) # creation of variable kvar = objvar.dtype newvar = newnc.createVariable(dvar, kvar, objvar.dimensions) varattrs = objvar.ncattrs() for atr in varattrs: attrv = objvar.getncattr(atr) newattr = set_attribute(newvar, atr, attrv) # variable values # vartotv = np.zeros((newvar.shape), dtype=kvar) objvar = ncobjs[0].variables[dvar] if searchInlist(list(objvar.dimensions),condim): begslicetot = 0 for ifile in range(Nfiles): slicevartot = [] if ncobjs[ifile].variables.has_key(dvar): objvar = ncobjs[ifile].variables[dvar] for dimn in objvar.dimensions: ldimfile = len(ncobjs[ifile].dimensions[dimn]) if dimn == condim: slicevartot.append(slice(begslicetot,begslicetot+ldimfile)) begslicetot = begslicetot + ldimfile else: slicevartot.append(slice(0,ldimfile)) newvar[tuple(slicevartot)] = objvar[:] newnc.sync() else: print errormsg print ' ' + fname + ": file '" + fold + '/' + confiles[ifile] + \ "' does not have variable '" + dvar + "' !!" quit(-1) else: newvar[:] = objvar[:] newnc.sync() # newvar[:] = vartotv gattrs = ncobjs[0].ncattrs() for attr in gattrs: attrv = ncobjs[0].getncattr(attr) newattr = set_attribute(newnc, attr, attrv) for ifile in range(Nfiles): ncobjs[ifile].close() newnc.sync() newnc.close() print "Successfull creation of concatenated file '" + ofile + "' !!!" return #netcdf_fold_concatenation('/media/data2/etudes/WRF_LMDZ/WL_HyMeX_HighRes/medic950116,Time', 'wrfout_d03', 'Times,XLONG,XLAT,T2,TH2,PSFC,U10,V10,MAPFAC_M') def netcdf_concatenation(ncfile): """ Function to concatenate netCDF files for a given set of variables netcdf_concateation(values, ncfile, varn) [ncfile]= [filename1]@[varn1]|[filename1]@[varn1] '|' list of file names and variables to concatenate [varn]= ',' separated list of variables to concatenate [var1],[var2],[...[varN]] or 'all' for all variables """ fname='netcdf_concatenate' ofile = 'netcdf_concatenated.nc' if ncfile == 'h': print fname + '_____________________________________________________________' print netcdf_concatenation.__doc__ quit() confiles = ncfile.split('|') Nfiles = len(confiles) print ' ' + fname +': concatenating:',Nfiles,'files' print confiles # Opening all files filenames = [] ncobjs = [] for files in confiles: filen = files.split('@')[0] filenames.append(filen) if not os.path.isfile(filen): print errormsg print ' ' + fname + ": file '" + filen + "' does no exist!!" quit(-1) print 'charging: ',filen ncobjs.append(NetCDFFile(filen, 'r')) # concatenated file creation ## newnc = NetCDFFile(ofile, 'w') # Looping variables ## ifile = 0 for files in confiles: filen = files.split('@')[0] desvar = files.split('@')[1] if desvar == 'all': desvars = ncobjs[ifile].variables else: desvars = desvar.split(',') for dvar in desvars: if not ncobjs[ifile].variables.has_key(dvar): print errormsg print ' ' + fname + ": files do not have variable '" +dvar+ "' !!!" quit(-1) for dvar in desvars: if not newnc.variables.has_key(dvar): print ' ' + fname + ": concatenating '" + dvar + "' from '" + filen+\ "'..." objvar = ncobjs[ifile].variables[dvar] # creation of dimensions vardims = objvar.dimensions for ndim in vardims: if not newnc.dimensions.has_key(ndim): vardimo = ncobjs[ifile].dimensions[ndim] if vardimo.isunlimited(): newnc.createDimension(ndim, None) else: newnc.createDimension(ndim, len(vardimo)) # creation of variable kvar = objvar.dtype newvar = newnc.createVariable(dvar, kvar, objvar.dimensions) varattrs = objvar.ncattrs() for atr in varattrs: attrv = objvar.getncattr(atr) newattr = set_attribute(newvar, atr, attrv) # variable values # vartotv = np.zeros((newvar.shape), dtype=kvar) objvar = ncobjs[ifile].variables[dvar] newvar[:] = objvar[:] newnc.sync() # newvar[:] = vartotv ifile = ifile + 1 gattrs = ncobjs[0].ncattrs() for attr in gattrs: attrv = ncobjs[0].getncattr(attr) newattr = set_attribute(newnc, attr, attrv) filesnvec = numVector_String(filenames,', ') newattr = set_attribute(newnc, 'concatenated', filesnvec) for ifile in range(Nfiles): ncobjs[ifile].close() newnc.sync() newnc.close() print "Successfull creation of concatenated file '" + ofile + "' !!!" return #netcdf_concatenation('mean_pluc.nc@all|mean_dtcon.nc@all') def count_cond(var, val, con): """ Function to count values of a variable which attain a condition [var]= variable [val]= value [con]= statistics/condition 'eq': equal than [val] 'neq': not equal than [val] 'lt': less than [val] 'le': less and equal than [val] 'gt': greater than [val] 'ge': greater and equal than [val] 'in': inside the range [val[0]] <= [var] <= [val[1]] (where [val]=[val1, val2] 'out': inside the range [val[0]] > [var]; [val[1]] < [var] (where [val]=[val1, val2] >>> vals = np.arange(100) >>> count_cond(vals,50,'eq') 1 >>> count_cond(vals,50,'neq') 99 >>> count_cond(vals,50,'lt') 50 >>> count_cond(vals,50,'le') 51 >>> count_cond(vals,50,'gt') 49 >>> count_cond(vals,50,'ge') 50 >>> count_cond(vals,[25,75],'in') 51 >>> count_cond(vals,[25,75],'out') 49 """ fname = 'count_cond' cons = ['eq', 'neq', 'lt', 'le', 'gt', 'ge', 'in', 'out'] if con == 'eq': Nvals = np.sum(var == val) elif con == 'neq': Nvals = np.sum(var != val) elif con == 'lt': Nvals = np.sum(var < val) elif con == 'le': Nvals = np.sum(var <= val) elif con == 'gt': Nvals = np.sum(var > val) elif con == 'ge': Nvals = np.sum(var >= val) elif con == 'in': if len(val) != 2: print errormsg print ' ' + fname + ": for condition '" + con + "' we must have two " + \ " values!!" print ' we have:', val quit(-1) Nvals = np.sum((var >= val[0])*(var <= val[1])) elif con == 'out': if len(val) != 2: print errormsg print ' ' + fname + ": for condition '" + con + "' we must have two " + \ " values!!" print ' we have:', val quit(-1) Nvals = np.sum((var < val[0])+(var > val[1])) else: print errormsg print ' ' + fname + ": condition '" + con + "' not ready!!" print ' only ready:', cons quit(-1) return Nvals def field_stats(values, ncfile, varn): """ Function to retrieve statistics from a field field_stats(values, ncfile, varn) [values]= [stats],[fillVals] [stats]: kind of statistics 'full': all statistics given variable [fillVals]: ':' list of _fillValues ('None' for any) [countVals]: ':' list of Values@cond to use to count conditions ('None' for any) [ncfile]= name of the netCDF file to use [varn]= variable name to use ('all' for all variables) """ import numpy.ma as ma fname='field_stats' if values == 'h': print fname + '_____________________________________________________________' print field_stats.__doc__ quit() arguments = '[stats],[fillVals],[coutnVals]' check_arguments(fname,values,arguments,',') stats=values.split(',')[0] fillVals=values.split(',')[1] countVals=values.split(',')[2] # Fill Values if fillVals == 'None': fillV = None else: if fillVals.find(':') != -1: fillV = fillVals.split(':') NfillV = len(fillV) else: fillV = [fillVals] NfillV = 1 # Count Values if countVals == 'None': countV = None else: if countVals.find(':') != -1: countV = countVals.split(':') NcountV = len(countV) else: countV = [countVals] NcountV = 1 ncobj = NetCDFFile(ncfile, 'r') if varn == 'all': varstats = list(ncobj.variables) else: varstats = [varn] print ' ' + fname + " file: '" + ncfile + "'..." print ' ' + fname + '______ ______ _____ ____ ___ __ _' for vn in varstats: if not ncobj.variables.has_key(vn): print errormsg print ' ' + fname + ": file do not have variable '" + vn + "' !!" quit(-1) objfield = ncobj.variables[vn] field = objfield[:] dtype = objfield.dtype counts = [] if countVals is not None: for icV in range(NcountV): cV = countV[icV].split('@')[0] cC = countV[icV].split('@')[1] countval = retype(cV, dtype) counts.append([count_cond(field, countval, cC), cC, countval]) if fillVals is not None: for ifV in range(NfillV): fillval = retype(fillV[ifV], dtype) field = ma.masked_equal(field, fillval) print ' ' + vn + '... .. .' if stats == 'full': minv = np.min(field) maxv = np.max(field) meanv = np.mean(field) mean2v = np.mean(field*field) varv = np.sqrt(mean2v - meanv*meanv) print ' Fstats shape:',field.shape print ' Fstats min:', minv print ' Fstats max:', maxv print ' Fstats mean:', meanv print ' Fstats mean2:', mean2v print ' Fstats variability:', varv if countVals is not None: for icV in range(NcountV): convals = counts[icV] print " Fstats N values '" + convals[1] + "'", convals[2], \ ':', convals[0] else: print errormsg print ' ' + fname + ": kind of statistics '" + values + "' not ready!!" print ' valid statistics: full' quit(-1) ncobj.close() return #field_stats('full', 'geo_em.d01.nc', 'HGT_M') def get_namelist_vars(values, namelist): """ Function to get namelist-like values ([varname] = [value]) get_namelist_vars(namelist) values= [sectionname],[kout] [sectionname]: name of the section from which one want the values ('all', for all) [kout]: kind of output 'tex3': printed output as LaTeX table of three columns \verb+namelist_name+ & value 'column': printed output as namelist_name value 'dict': as two python dictionary object (namelistname, value and namelistname, sectionname) namelist= namelist_like file to retrieve values >>> get_namelist_vars('geogrid,dic','/home/lluis/etudes/domains/medic950116/namelist.wps') {'e_sn': '32, 97,', 'stand_lon': '0.', 'opt_geogrid_tbl_path': "'./'", 'geog_data_path': "'/home/lluis/DATA/WRF/geog'", 'pole_lat': '90.0', 'ref_lat': '35.0,', 'map_proj': "'lat-lon',", 'parent_id': '1, 1,', 'geog_data_res': "'2m','2m',", 'e_we': '32, 112,', 'dx': '0.35,', 'dy': '0.35,', 'parent_grid_ratio': '1, 3,', 'pole_lon': '0.0', 'ref_lon': '20,', 'j_parent_start': '1, 17,', 'i_parent_start': '1, 31,'} """ fname = 'get_namelist_vars' # List of characters which split pairs name/value valuessep = ['='] # List of characters which indicate a comment commentchars = ['#'] if values == 'h': print fname + '_____________________________________________________________' print get_namelist_vars.__doc__ quit() expectargs = '[sectionname],[kout]' check_arguments(fname,values,expectargs,',') secname = values.split(',')[0] kout = values.split(',')[1] if not os.path.isfile(namelist): print errormsg print ' ' + fname + ": namelist file '" + namelist + "' does not exist !!" quit(-1) ncml = open(namelist, 'r') sections = {} namelistvals = {} namelistsecs = {} sectionnames = [] namessec = [] allnames = [] namelistvalssec = {} namelistsecssec = {} nmlname = '' sectionname = '' for line in ncml: linevals = reduce_spaces(line) Nvals = len(linevals) if Nvals >= 1 and linevals[0][0:1] == '&': if len(sectionnames) > 1: sections[sectionname] = namessec sectionname = linevals[0][1:len(linevals[0])+1] # print ' ' + fname + ": new section '" + sectionname + "' !!!" sectionnames.append(sectionname) namessec = [] nmlname = '' elif Nvals >= 1 and not searchInlist(commentchars,linevals[0][0:1]): if Nvals >= 3 and searchInlist(valuessep,linevals[1]): nmlname = linevals[0] nmlval = numVector_String(linevals[2:Nvals],' ') elif Nvals == 1: for valsep in valuessep: if linevals[0].find(valsep) != -1: nmlname = linevals[0].split(valsep)[0] nmlval = linevals[0].split(valsep)[1] break elif Nvals == 2: for valsep in valuessep: if linevals[0].find(valsep) != -1: nmlname = linevals[0].split(valsep)[0] nmlval = linevals[1] break elif linevals[1].find(valsep) != -1: nmlname = linevals[0] nmlval = linevals[1].split(valsep)[0] break else: print warnmsg print ' ' + fname + ': wrong number of values', Nvals, \ 'in the namelist line!' print ' line: ',line print ' line values:',linevals # quit(-1) namelistvals[nmlname] = nmlval namelistsecs[nmlname] = sectionname namessec.append(nmlname) allnames.append(nmlname) if len(sectionname) > 1: sections[sectionname] = namessec if secname != 'all': if not searchInlist(sections.keys(),secname): print errormsg print ' ' + fname + ": section '" + values + "' does not exist !!" print ' only found:',sectionnames quit(-1) namestouse = [] for nml in allnames: for nnml in sections[secname]: namelistvalssec[nnml] = namelistvals[nnml] namelistsecssec[nnml] = secname if nml == nnml: namestouse.append(nml) else: namestouse = allnames namelistvalssec = namelistvals namelistsecssec = namelistsecs if kout == 'tex3': ofile='get_namelist_vars_3col.tex' fobj = open(ofile, 'w') vals = namestouse Nvals = len(vals) Nvals3 = int(Nvals/3) latextab = '\\begin{center}\n\\begin{tabular}{lclclc}\n' for il in range(2): latextab = latextab + '{\\bfseries{name}} & {\\bfseries{value}} & ' latextab= latextab+ '{\\bfseries{name}} & {\\bfseries{value}} \\\\ \\hline\n' if np.mod(Nvals,3) != 0: Nvals0 = Nvals - np.mod(Nvals,3) else: Nvals0 = Nvals for ival in range(0,Nvals0,3): line = '' print ' ival:',ival for il in range(2): line = line + '\\verb+' + vals[ival+il] + '+ & ' + \ namelistvalssec[vals[ival+il]].replace('_','\\_') +' & ' line = line + '\\verb+' + vals[ival+2] + '+ & ' + \ namelistvalssec[vals[ival+2]].replace('_','\\_') + ' \\\\\n' latextab = latextab + line latextab = latextab + '%not multiple of three!!!!\n' print 'mod:', np.mod(Nvals,3),Nvals0,Nvals if np.mod(Nvals,3) != 0: ival = Nvals0 line = '' for il in range(np.mod(Nvals,3)): print 'ival:',ival + il line = line + '\\verb+' + vals[ival+il] + '+ & ' + \ namelistvalssec[vals[ival+il]].replace('_','\\_') + ' & ' for il in range(2-np.mod(Nvals,3)): line = line + ' & & ' latextab = latextab + line + ' & \\\\\n' latextab = latextab + '\\end{tabular}\n\\end{center}\n' # print latextab fobj.write(latextab) fobj.close() print fname + ": successful writen '" + ofile + "' LaTeX tale file !!" return elif kout == 'column': for dictv in namestouse: print dictv + ' = ' + namelistvalssec[dictv] return elif kout == 'dict': return namelistvalssec, namelistsecssec else: print errormsg print ' ' + fname + ": kind of output '" + kout + "' not ready!!!" quit(-1) return def WRF_d0Nref(values, geofold): """ Function for the generation of an extra WRF domain from a given one field_stats(values, ncfile, varn) [values]= [namelist],[minLON],[minLAT],[maxLON],[maxLAT],[Ndomref],[factor],[geogres] [namelist]: namelist.wps to use [minLON],[minLAT]: minimum longitude and latitude [maxLON],[maxLAT]: maximum longitude and latitude [Ndomref]: domain reference number [factor]: resolution factor with respect d01 [geogres]: which topography resolution to use [geofold]= folder where to found the geo_em.d0[N-1].nc file """ fname='WRF_d0Nref' if values == 'h': print fname + '_____________________________________________________________' print WRF_d0Nref.__doc__ quit() expectargs = '[namelist],[minLON],[minLAT],[maxLON],[maxLAT],[Ndomref],[factor],' expectargs = expectargs + '[geogres]' check_arguments(fname,values,expectargs,',') namelist = values.split(',')[0] minLON = np.float(values.split(',')[1]) minLAT = np.float(values.split(',')[2]) maxLON = np.float(values.split(',')[3]) maxLAT = np.float(values.split(',')[4]) Ndomref = int(values.split(',')[5]) factor = int(values.split(',')[6]) geogres = values.split(',')[7] onml = 'namelist.wps_d0' + str(Ndomref+1) # Namelist variables to which new value is a copy of the last one repeatvalue = ['start_date' ,'end_date'] # Namelist variables to which value is increased by 1 (single value) add1 = ['max_dom'] # Namelist variables to which new value is an increase by 1 of the last one add1value = ['parent_id'] # ncfile = geofold + '/geo_em.d' + zeros(Ndomref,2) + '.nc' ncfile = geofold + '/geo_em.d0' + str(Ndomref) + '.nc' if not os.path.isfile(ncfile): print errormsg print ' ' + fname + ': geo_em file "' + ncfile + '" does not exist !!' quit(-1) ncobj = NetCDFFile(ncfile, 'r') nlon='XLONG_M' nlat='XLAT_M' lonobj = ncobj.variables[nlon] latobj = ncobj.variables[nlat] lon = lonobj[0,:,:] lat = latobj[0,:,:] ncobj.close() diffmin = np.sqrt((lon - minLON)**2. + (lat - minLAT)**2.) diffmax = np.sqrt((lon - maxLON)**2. + (lat - maxLAT)**2.) posdiffmin = index_mat(diffmin,np.min(diffmin)) posdiffmax = index_mat(diffmax,np.min(diffmax)) # print 'min vals. min:',np.min(diffmin),'max:',np.min(diffmax) # print 'min:', minLON, ',', minLAT, ':', posdiffmin, '.', lon[tuple(posdiffmin)], \ # ',', lat[tuple(posdiffmin)] # print 'max:', maxLON, ',', maxLAT, ':', posdiffmax, '.', lon[tuple(posdiffmax)], \ # ',', lat[tuple(posdiffmax)] print ' ' + fname +': coordinates of the SW corner:', posdiffmin, ' NE corner:',\ posdiffmax nmlobj = open(namelist, 'r') onmlobj = open(onml, 'w') for line in nmlobj: listline = reduce_spaces(line) nvals = len(listline) if nvals > 0: if searchInlist(add1,listline[0]): nmlval = int(listline[nvals-1].replace(',','')) + 1 onmlobj.write(' ' + listline[0] + ' = ' + str(nmlval) + ',\n') elif searchInlist(add1value,listline[0]): if Ndomref == 1: nmlval = 1 else: nmlval = int(listline[nvals-1].replace(',','')) + 1 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1], ' ') + ' ' +str(nmlval)+',\n') elif searchInlist(repeatvalue,listline[0]): nmlval = listline[nvals-1] onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+'\n') elif listline[0] == 'parent_grid_ratio': nmlval = factor onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'i_parent_start': nmlval = posdiffmin[1] - 10 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'j_parent_start': nmlval = posdiffmin[0] - 10 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'e_we': nmlval = (posdiffmax[1] - posdiffmin[1] + 20)*factor nmlval = nmlval + np.mod(nmlval,factor) + 1 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'e_sn': nmlval = (posdiffmax[0] - posdiffmin[0] + 20)*factor nmlval = nmlval + np.mod(nmlval,factor) + 1 onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') elif listline[0] == 'geog_data_res': nmlval = "'" + geogres + "'" onmlobj.write(' ' + listline[0] + ' = ' + \ numVector_String(listline[2:nvals+1],' ') + ' ' + str(nmlval)+',\n') else: onmlobj.write(line) else: onmlobj.write('\n') nmlobj.close() onmlobj.close() print fname + ": successfully written new namelist '" + onml + "' !!" return def filter_2dim(values, ncfile, varn): """ Function to filter along 2 dimensions (moving grid-box means of a given size) values of the netCDF file filter_2dim(values, ncfile, variable) values= [Ngrid],[dimnh1],[dimnh2],[dimvnh1],[dimvnh2] Ngrid: box size to make means dimnh[1/2]: name of the 2 dimensions to make the averages dimvnh[1/2]: name of the 2 variables with the values of the dimensions to make the averages ncfile= netCDF file to use varn= name of the variable to filter ('all' for all variables) filter_2dim('80,y,x,lon,lat', 'tahiti_5m_ll.grd', 'z') """ fname = 'filter_2dim' if values == 'h': print fname + '_____________________________________________________________' print filter_2dim.__doc__ quit() expectargs = '[Ngrid],[dimnh1],[dimnh2],[dimvnh1],[dimvnh2]' check_arguments(fname,values,expectargs,',') Ngrid = int(values.split(',')[0]) dimnh1 = values.split(',')[1] dimnh2 = values.split(',')[2] dimvnh1 = values.split(',')[3] dimvnh2 = values.split(',')[4] Ngrid2 = Ngrid/2 dimhorns = [dimnh1, dimnh2] dimvhorns = [dimvnh1, dimvnh2] if varn == 'all': varns = ncobj.variables ofile = 'filter_hor.nc' else: varns = [varn] ofile = 'filter_hor_' + varn + '.nc' ncobj = NetCDFFile(ncfile,'r') newnc = NetCDFFile(ofile, 'w') for vn in varns: varvobj = ncobj.variables[vn] vardims = varvobj.dimensions print ' ' + fname + ' var:', vn, 'variable shape:', varvobj.shape for dimn in vardims: dimobj = ncobj.dimensions[dimn] if not newnc.dimensions.has_key(dimn): if dimobj.isunlimited(): newnc.createDimension(dimn, None) else: newnc.createDimension(dimn, len(dimobj)) if searchInlist(varvobj.ncattrs(),'_FillValue'): vfillValue = varvobj._FillValue else: vfillValue = fillValue vtype = varvobj.dtype if searchInlist(dimvhorns,vn): print ' ' + fname + ': dimension values variable!' print " keeping original values as '" + vn + "' and filtered as '" + \ vn+'_flt'+str(Ngrid)+'x'+str(Ngrid) + "'" newvar = newnc.createVariable(vn, vtype, vardims) newvar[:] = varvobj[:] vattr = varvobj.ncattrs() for nattr in vattr: attrv = varvobj.getncattr(nattr) if nattr != '_FillValue': newattr = set_attribute(newvar,nattr,attrv) newvar = newnc.createVariable(vn+'_flt'+str(Ngrid)+'x'+str(Ngrid), \ vtype, vardims) else: newvar = newnc.createVariable(vn, vtype, vardims, fill_value = fillValue) if searchInlist(vardims,dimnh1) or searchInlist(vardims,dimnh2): varv = varvobj[:] varfilt = varvobj[:] dimh1id = -1 dimh2id = -1 if searchInlist(vardims,dimnh1): dimh1id= vardims.index(dimnh1) if searchInlist(vardims,dimnh2): dimh2id= vardims.index(dimnh2) if dimh1id != -1 and dimh2id != -1: print ' ' + fname + ': 2D filtered variable' dh1 = varvobj.shape[dimh1id] dh2 = varvobj.shape[dimh2id] for i in range(Ngrid2,dh1-Ngrid2+1): for j in range(Ngrid2,dh2-Ngrid2+1): percendone(i*(dh1-Ngrid)*1.+j,(dh1-Ngrid)*(dh2-Ngrid)*1., 1., \ vn + ' filtered: ' + str(i) + ', ' + str(j)) varslice = [] varslicesum = [] for dimn in vardims: if dimn == dimnh1: varslicesum.append(slice(i-Ngrid2,i+Ngrid2+1)) varslice.append(slice(i,i+1)) elif dimn == dimnh2: varslicesum.append(slice(j-Ngrid2,j+Ngrid2+1)) varslice.append(slice(j,j+1)) else: varslicesum.append(slice(0, \ len(ncobj.dimensions[dimn]))) varslice.append(slice(0, len(ncobj.dimensions[dimn]))) # Doing nothing for _FillValue values varvalssum = varv[tuple(varslicesum)] if varvalssum.mask.all(): varfilt[tuple(varslice)] = fillValue else: numberValues = len(varvalssum.compressed()) if dimh1id < dimh2id: varfilt[tuple(varslice)] = np.sum(np.sum(varvalssum, \ axis=dimh1id), axis=dimh2id-1)/(numberValues*1.) else: varfilt[tuple(varslice)] = np.sum(np.sum(varvvlssum, \ axis=dimh2id), axis=dimh1id-1)/(numberValues*1.) print ' ' + fname + ': 2D Finished!!' elif dimh1id != -1 and dimh2id == -1: print ' ' + fname + ': 1D filtered variable' for i in range(Ngrid2,varvobj.shape[dimh1id]-Ngrid2+1): percendone(i,(dh1-Ngrid), 1, vn + ' filtered') varslice = [] varslicesum = [] for dimn in vardims: if dimn == dimhn1: varslicesum.append(slice(i-Ngrid2,i+Ngrid2+1)) else: varslicesum.append(slice(0, len(ncobj.dimensions[dimn]))) varvalssum = varv[tuple(varslicesum)] if varvalssum.mask.all(): varfilt[tuple(varslice)] = fillValue else: numberValues = len(varvalssum.compressed()) varfilt[tuple(varslice)] = np.sum(varvalssum, \ axis=dimh1id)/numberValues*1. print ' ' + fname + ': 1D Finished!!' elif dimh2id != -1 and dimh1id == -1: print ' ' + fname + ': 1D filtered variable' for j in range(Ngrid2,varvobj.shape[dimh2id]-Ngrid2+1): percendone(j,(dh2-Ngrid), 1, vn + ' filtered') varslice = [] varslicesum = [] for dimn in vardims: if dimn == dimhn2: varslicesum.append(slice(j-Ngrid2,j+Ngrid2+1)) else: varslicesum.append(slice(0, len(ncobj.dimensions[dimn]))) varvalssum = varv[tuple(varslicesum)] if varvalssum.mask.all(): varfilt[tuple(varslice)] = fillValue else: numberValues = len(varvalssum.compressed()) varfilt[tuple(varslice)] = np.sum(varvalssum, \ axis=dimh2id)/numberValues print ' ' + fname + ': 1D Finished!!' newvar[:] = varfilt else: newvar[:] = varvobj[:] vattr = varvobj.ncattrs() for nattr in vattr: attrv = varvobj.getncattr(nattr) if nattr != '_FillValue': newattr = set_attribute(newvar, nattr, attrv) gattr = ncobj.ncattrs() for nattr in gattr: attrv = ncobj.getncattr(nattr) newattr = set_attribute(newnc, nattr, attrv) ncobj.close() newnc.sync() newnc.close() print fname + ": successfull creation of '" + ofile + "' !!!" return #filter_2dim('80,y,x,lon,lat', 'tahiti_5m_ll.grd', 'z') def ncreplace(values, ncfile, varn): """ Function to replace something from a netCDF file values= [kind],[kindn],[netcdffileref] kind: element to substitute 'var': variable 'vattr': variable attributes 'gattr': global attributes kindn: name of the 'something' to replace ('all', for all possible values, [varn]@[attrn] for attribute of a given variable) netcdffileref: netCDF file from which replace values will be taken ncfile= netCDF file to replace something varn= name to use (accordingly to '[kind]') ncreplace('var,HGT_M,Polynesie_filtered_nn.nc', 'geo_em.d04.nc', 'HGT_M') """ fname = 'ncreplace' if values == 'h': print fname + '_____________________________________________________________' print ncreplace.__doc__ quit() expectargs = '[kind],[kindn],[netcdffileref]' check_arguments(fname,values,expectargs,',') kind = values.split(',')[0] kindn = values.split(',')[1] netcdffileref = values.split(',')[2] print ' ' + fname + ": replacing '" + kind + "' in '" + ncfile + "' using '" + \ netcdffileref + "' ..." if not os.path.isfile(netcdffileref): print errormsg print ' ' + fname + ": reference file '" + netcdffileref + \ "' does not exist !!" quit(-1) ncrefobj = NetCDFFile(netcdffileref, 'r') ncobj = NetCDFFile(ncfile, 'a') if kind == 'var' and kindn != 'all': if not ncrefobj.variables.has_key(kindn): print errormsg print ' ' + fname + ": reference file does not have variable '" + kindn \ + "' !!" print ' it has: ',ncrefobj.variables.keys() quit(-1) elif kind == 'vattr' and kindn != 'all': if kindn.index('@') == -1: print errormsg print ' ' + fname + ": an pair [var]@[attrn] is not given! '" + kindn + \ + "' !!" quit(-1) varn = kindn.split('@')[0] attrn = kindn.split('@')[1] if not ncrefobj.variables.has_key(varn): print errormsg print ' ' + fname + ": reference file does not have variable '" + varn +\ "' !!" quit(-1) varobj = ncrefobj.variables[varn] if not searchInlist(varobj.ncattrs(), attrn): print errormsg print ' ' + fname + ": reference variable '" + varn + \ "' does not have attribute '" + attrn + "' !!" quit(-1) else: print errormsg print ' ' + fname + ": kind to replace '" + kind + "' not ready !!" quit(-1) if kind == 'var': if kindn == 'all': varrefns = ncrefobj.variables varns = ncobj.variables else: varrefns = [kindn] varns = [varn] # On kindn= 'all' & varn= 'all' one has to assume same variables' names for vnref in varrefns: if not ncrefobj.variables.has_key(vnref): print errormsg print ' ' + fname + ": reference file does not have variable '" + \ vnref + "' !!" quit(-1) for vn in varns: if not ncobj.variables.has_key(vn): print errormsg print ' ' + fname + ": file does not have variable '" + vn + \ "' !!" quit(-1) if kindn == 'all' and vnref == vn: print ' ' + fname + ": replacing all now '" + vn + "' with '"+\ vnref + "' ..." varv = ncobj.variables[vn] varrefv = ncrefobj.variables[vnref][:] varv[:] = varrefv ncobj.sync() break else: vdimns = numVector_String(ncobj.variables[vn].shape,':') vrefdimns = numVector_String(ncrefobj.variables[vnref].shape,':') if vdimns != vrefdimns: print errormsg print ' ' + fname + ': values can not be replaced!' print ' different shapes! original:', \ ncobj.variables[vn].shape,'reference:', \ ncrefobj.variables[vnref].shape # In case of HGT_M... # quit(-1) quit(-1) print ' ' + fname + ": replacing '" + vn + "' with '" + \ vnref + "' ..." varv = ncobj.variables[vn] varrefv = ncrefobj.variables[vnref][:] # In case of HGT_M... varv[0,:,:] = varrefv varv[:] = varrefv ncobj.sync() else: print errormsg print ' ' + fname + ": kind to replace '" + kind + "' not exactly ready !!" quit(-1) ncobj.sync() ncobj.close() ncrefobj.close() return def ncstepdiff(values, ncfile, varn): """ Function to compute differencies between time-steps (deacumulate) a netCDF file new variables labelled [varn] + 'de'will be added to the file values= [tdimn],[tvarn] tdimn: name of the dimension time tvarn: name of the variable time ncfile= netCDF file to replace something varn= ',' list of variables to deaccumulate ('all' for all) ncstepdiff('Time,Times', '/home/lluis/PY/wrfout_d01_2001-11-11_00:00:00.tests', 'Q2') """ fname = 'ncstepdiff' if values == 'h': print fname + '_____________________________________________________________' print ncstepdiff.__doc__ quit() expectargs = '[tdimn],[tvarn]' check_arguments(fname,values,expectargs,',') tdimn = values.split(',')[0] tvarn = values.split(',')[1] onc = NetCDFFile(ncfile, 'a') if varn.find(',') != -1: if varn != 'all': varns = varn.split(',') else: varns = onc.variables else: varns = [varn] otdim = onc.dimensions[tdimn] otvar = onc.variables[tvarn] timeinf = cls_time_information(onc, tvarn) dimt = len(otdim) print ' ' + fname + ': deaccumulating _______' for vn in varns: print " '" + vn + "'" ovar = onc.variables[vn] varinf = variable_inf(ovar) if not searchInlist(varinf.dimns, tdimn): print warnmsg print ' ' + fname + ": variable '" + vn + "' has not time dimension '"+\ tdimn + "' !!" print ' skipping it' else: slicefwd, dfwd = slice_variable(ovar, tdimn + ':1@'+str(dimt)) slicebck, dbck = slice_variable(ovar, tdimn + ':0@'+str(dimt-1)) deaccum = np.zeros((varinf.dims), dtype=varinf.dtype) ddeaccum = [] for idv in range(varinf.Ndims): if varinf.dimns[idv] == tdimn: ddeaccum.append(slice(1,varinf.dims[idv])) else: ddeaccum.append(slice(0,varinf.dims[idv])) deaccum[tuple(ddeaccum)] = slicefwd - slicebck if not searchInlist(onc.variables, vn + 'de'): if varinf.FillValue is not None: newvar = onc.createVariable(vn + 'de', varinf.dtype, \ tuple(varinf.dimns), fill_value = varinf.FillValue) else: newvar = onc.createVariable(vn + 'de', varinf.dtype, \ tuple(varinf.dimns)) else: print warnmsg print ' ' + fname + ": variable '" + vn + "de' already exist " + \ 'replacing values !!' newvar = onc.variables[vn + 'de'] if timeinf.dt is not None: # Not sure of this... # newvar[:] = deaccum / timeinf.dt newvar[:] = deaccum else: newvar[:] = deaccum newattr = set_attribute(newvar, 'deaccumulated', 'True') if timeinf.dt is not None: newattr = set_attributek(newvar, 'time_step_diff', timeinf.dt, \ 'npfloat') newattr = set_attribute(newvar, 'values', 'divided by time_step_diff') for attrn in varinf.attributes: attrv = ovar.getncattr(attrn) newattr = set_attribute(newvar, attrn, attrv) onc.sync() onc.close() return #ncstepdiff('Time,Times', '/home/lluis/PY/wrfout_d01_2001-11-11_00:00:00.tests', 'Q2,T2') def file_creation(values, ncfile, varn): """ Operation to create a file with one variable with a given set of dimensions values= [dimensions]|[varattributes]|[kind] [dimensions]: [dimn1]:[dsize1],...,[dimnN]:[dsizeN], ',' pairs of variable name [dimn] and [size] if [dsize] = 'None' (for UNLIMITED), give a third value with the real size [attributes]: [std_name]@[long_name]@[units], standard name, long name and units of the variable [kind]: type of variable (standard netCDF4/C-like values, 'c', 'i', 'f', 'f8',...) ncfile= name of the file varn= name of the variables """ fname = 'file_creation' if values == 'h': print fname + '_____________________________________________________________' print file_creation.__doc__ quit() expectargs = '[dimensions]|[varattributes]|[kind]' check_arguments(fname,values,expectargs,'|') dimensions = values.split('|')[0].split(',') attributes = values.split('|')[1] kind = values.split('|')[2] onc = NetCDFFile(ncfile, 'w') # Dimensions dnames = [] dsize = [] for dim in dimensions: # print " Adding: '" + dim + "' ..." dimn = dim.split(':')[0] dimv = dim.split(':')[1] if dimv == 'None': if len(dim.split(':')) != 3: print errormsg print ' ' + fname + ": dimension '" + dimn + "' is None but the " + \ 'size is requried!!' quit(-1) else: dv = None dsize.append(int(dim.split(':')[2])) else: dv = int(dimv) dsize.append(dv) dnames.append(dimn) newdim = onc.createDimension(dimn, dv) onc.sync() # Variable if kind == 'c': newvar = onc.createVariable(varn, 'c', tuple(dnames)) # newvar[:] = np.zeros(tuple(dsize), dtype=np.float) if kind == 'f' or kind == 'f4': newvar = onc.createVariable(varn, 'f4', tuple(dnames), fill_value=fillValue) newvar[:] = np.zeros(tuple(dsize), dtype=np.float) elif kind == 'f8': newvar = onc.createVariable(varn, 'f8', tuple(dnames), \ fill_value= np.float64(fillValue)) newvar[:] = np.zeros(tuple(dsize), dtype=np.float64) elif kind == 'i': newvar = onc.createVariable(varn, 'i', tuple(dnames), fill_value=-999999) newvar[:] = np.zeros(tuple(dsize), dtype=int) else: print errormsg print ' ' + fname + ": variable kind '" + kind + "' not ready!!" quit(-1) sname = attributes.split('@')[0] lname = attributes.split('@')[1] u = attributes.split('@')[2] newattr = basicvardef(newvar, sname, lname, u) onc.sync() # Global attributes newattr = set_attribute(onc, 'description', "file creation using " + fname ) onc.sync() onc.close() #file_creation('time_counter:12,sizes:24|time@time@seconds since 1949-12-01 00:00:00|f8', 'test.nc', 't_instant') def var_creation(values, ncfile, varn): """ Operation to create a new variable in a file with a given set of dimensions values= [dimensions]|[varattributes]|[kind] [dimensions]: [dimn1]:[dsize1],...,[dimnN]:[dsizeN], ',' pairs of variable name [dimn] and [size] if [dsize] = 'None', give a third value with the real size [attributes]: [std_name]@[long_name]@[units], standard name, long name and units of the variable ('!' for spaces) [kind]: type of variable (standard netCDF4/C-like values, 'c', 'i', 'f', 'f8',...) ncfile= name of the file varn= name of the variables """ fname = 'var_creation' if values == 'h': print fname + '_____________________________________________________________' print var_creation.__doc__ quit() expectargs = '[dimensions]|[varattributes]|[kind]' check_arguments(fname,values,expectargs,'|') dimensions = values.split('|')[0].split(',') attributes = values.split('|')[1].replace('!',' ') kind = values.split('|')[2] onc = NetCDFFile(ncfile, 'a') # Dimensions dnames = [] dsize = [] for dim in dimensions: # print " Adding: '" + dim + "' ..." dimn = dim.split(':')[0] dimv = dim.split(':')[1] if dimv == 'None': if len(dim.split(':')) != 3: print errormsg print ' ' + fname + ": dimension '" + dimn + "' is None but the " + \ 'size is requried!!' quit(-1) else: dv = None dsize.append(int(dim.split(':')[2])) else: dv = int(dimv) dsize.append(dv) dnames.append(dimn) if not searchInlist(onc.dimensions, dimn): newdim = onc.createDimension(dimn, dv) onc.sync() # Variable if kind == 'c': newvar = onc.createVariable(varn, 'c', tuple(dnames)) # newvar[:] = np.zeros(tuple(dsize), dtype=np.float) elif kind == 'f' or kind == 'f4': newvar = onc.createVariable(varn, 'f4', tuple(dnames), fill_value=fillValue) newvar[:] = np.zeros(tuple(dsize), dtype=np.float) elif kind == 'f8': newvar = onc.createVariable(varn, 'f8', tuple(dnames), \ fill_value=np.float64(fillValue)) newvar[:] = np.zeros(tuple(dsize), dtype=np.float64) elif kind == 'i': newvar = onc.createVariable(varn, 'i', tuple(dnames), fill_value=-99999) newvar[:] = np.zeros(tuple(dsize), dtype=int) else: print errormsg print ' ' + fname + ": variable kind '" + kind + "' not ready!!" quit(-1) sname = attributes.split('@')[0] lname = attributes.split('@')[1] u = attributes.split('@')[2] newattr = basicvardef(newvar, sname, lname, u) onc.sync() onc.close() #var_creation('dim1:12,sizes:24|time@time@seconds since 1949-12-01 00:00:00|f4', 'test.nc', 'var2') #quit() def dimToUnlimited(values, ncfile): """ Operation to create an unlimited dimension from an existing one values= Name of the dimension to transform ncfile= name of the file """ fname = 'dimToUnlimited' if values == 'h': print fname + '_____________________________________________________________' print dimToUnlimited.__doc__ quit() dimn = values ncobj = NetCDFFile(ncfile, 'r') dimnames = ncobj.dimensions.keys() variables = ncobj.variables.keys() if not searchInlist(dimnames,dimn): print errormsg print ' ' + fname + ": dimension '" + dimn + "' does not exist in file '" + \ ncfile + "' !!" print ' existing ones are:',ncobj.dimensions.keys() quit(-1) # Creation of a new file newnc = NetCDFFile('new_' + ncfile, 'w') # Creation of dimensions for dn in dimnames: print 'adding dim:',dn if ncobj.dimensions[dn].isunlimited() or dn == dimn: newdim = newnc.createDimension(dn, None) else: newdim = newnc.createDimension(dn, len(ncobj.dimensions[dn])) # ncobj.close() newnc.sync() # newnc.close() # Adding variables for vn in variables: print 'adding variable:',vn vno = ncobj.variables[vn] vdim = vno.dimensions vtype = vno.dtype varatts = vno.ncattrs() if searchInlist(varatts, '_FillValue'): fillv = vno.getncattr('_FillValue') newvar = newnc.createVariable(vn, vtype, vdim, fill_value=fillv) else: newvar = newnc.createVariable(vn, vtype, vdim) newvar[:] = vno[:] for nattr in vno.ncattrs(): if not nattr == '_FillValue': nattrv = vno.getncattr(nattr) newattr = newvar.setncattr(nattr, nattrv) # Adding attributes fgaddattr(ncfile, 'new_' + ncfile) sub.call(['mv', 'new_'+ncfile, ncfile]) return #dimToUnlimited('lon', 'test.nc') def increaseDimvar(values, ncfile, varn): """ Function to increase with 1 dimension an existing variable within a netcdf file. Values of the variable will be repeated along the new dimension values='[dimname]:[size]:[position]:[unlimited]' dimname: name of the new dimension size: size of the new dimension position: position for the dimensions (starting from 0, outter most [left]) unlimited: y/n value for an unlimited dimension ncfile= netCDF file to use varn= variable to change its dimensions """ import subprocess as sub fname = 'increaseDimvar' if values == 'h': print fname + '_____________________________________________________________' print increaseDimvar.__doc__ quit() expectargs = '[dimname]:[size]:[position]:[unlimited]' check_arguments(fname,values,expectargs,':') dimname = values.split(':')[0] size = int(values.split(':')[1]) position = int(values.split(':')[2]) unlimited = values.split(':')[3] ofile = ncfile + '_new.nc' onc = NetCDFFile(ncfile, 'r') dims = onc.dimensions if searchInlist(dims, dimname): print errormsg print ' ' + fname + ": file '" + ncfile + "' already has dimension '" + \ dimname + "' !!" quit(-1) varns = onc.variables if not searchInlist(varns, varn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not variable '" + varn + \ "' !!" quit(-1) varobj = onc.variables[varn] vardims = varobj.dimensions varkind = varobj.dtype Norigdims = len(vardims) if position > Norigdims - 1: print errormsg print ' ' + fname + ": wrong position ", position," for the new dimension ",\ "on variable '" + varn + "' has only", Norigdims," dimensions !!" quit(-1) newdimns = [] newdimvs = [] idorig = 0 for idn in range(Norigdims + 1): if idn == position: newdimns.append(dimname) newdimvs.append(size) else: newdimns.append(vardims[idorig]) newdimvs.append(varobj.shape[idorig]) idorig = idorig + 1 # New variable newvarv = np.zeros(tuple(newdimvs), dtype=varkind) for inewdim in range(size): slicev = [] idorig = 0 for idn in range(Norigdims + 1): if idn == position: slicev.append(inewdim) else: slicev.append(slice(0,varobj.shape[idorig])) idorig = idorig + 1 newvarv[tuple(slicev)] = varobj[:] # Creation of a new file with the new variable dimensions newnc = NetCDFFile(ofile, 'w') # Creation of dimensions for dn in dims: print 'adding dim:',dn,' ...' if onc.dimensions[dn].isunlimited(): newdim = newnc.createDimension(dn, None) else: newdim = newnc.createDimension(dn, len(onc.dimensions[dn])) if unlimited == 'y': newdim = newnc.createDimension(dimname, None) else: newdim = newnc.createDimension(dimname, size) newnc.sync() # Adding variables for vn in varns: print 'adding variable:',vn,'...' vno = onc.variables[vn] if vn == varn: vdim = tuple(newdimns) else: vdim = vno.dimensions vtype = vno.dtype varatts = vno.ncattrs() if searchInlist(varatts, '_FillValue'): fillv = vno.getncattr('_FillValue') newvar = newnc.createVariable(vn, vtype, vdim, fill_value=fillv) else: newvar = newnc.createVariable(vn, vtype, vdim) for nattr in vno.ncattrs(): if not nattr == '_FillValue': nattrv = vno.getncattr(nattr) newattr = newvar.setncattr(nattr, nattrv) if vn == varn: newvar[:] = newvarv[:] else: newvar[:] = vno[:] onc.close() newnc.sync() newnc.close() # Adding attributes fgaddattr(ncfile, ofile) sub.call(['mv',ofile,ncfile]) return #increaseDimvar('shan:29:1:y', 'test.nc', 'var') def changevartype(values, ncfile, varn): """ Function to change the type of a variable (when possible) values=[newtype] type to use ('c', character; 'i', integer, 'f', float, 'd', double) ncfile= netCDF file to use varn= variable to change its type """ import subprocess as sub fname = 'changevartype' if values == 'h': print fname + '_____________________________________________________________' print changevartype.__doc__ quit() newtype = values ofile = ncfile + '_new.nc' onc = NetCDFFile(ncfile, 'r') if not searchInlist(onc.variables, varn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not variable '" + varn + \ "' !!" quit(-1) dims = onc.dimensions varns = onc.variables # Creation of a new file with the new variable dimensions newnc = NetCDFFile(ofile, 'w') # Creation of dimensions for dn in dims: print 'adding dim:',dn,' ...' if onc.dimensions[dn].isunlimited(): newdim = newnc.createDimension(dn, None) else: newdim = newnc.createDimension(dn, len(onc.dimensions[dn])) newnc.sync() # Transforming variable varo = onc.variables[varn] vartype = varo.dtype vardims = varo.dimensions varatts = varo.ncattrs() varshape = varo.shape varv = varo[:] if newtype == 'c': print errormsg print ' ' + fname + ": new type '" + newtype + "' not ready!!'" elif newtype == 'i': newvarv = varv.astype(int) elif newtype == 'f': newvarv = varv.astype(np.float) elif newtype == 'd': newvarv = varv.astype(np.float64) else: print errormsg print ' ' + fname + ": type '" + newtpe + "' not ready !!" print ' availables: i, f, d' quit(-1) if searchInlist(varatts, '_FillValue'): fillv = varo.getncattr('_FillValue') newvar = newnc.createVariable(varn, newtype, vardims, fill_value=fillv) else: newvar = newnc.createVariable(varn, newtype, vardims) for nattr in varo.ncattrs(): if not nattr == '_FillValue': nattrv = varo.getncattr(nattr) newattr = newvar.setncattr(nattr, nattrv) newvar[:] = newvarv # Adding variables for vn in varns: print 'adding variable:',vn,'...' vno = onc.variables[vn] if vn != varn: vdim = vno.dimensions vtype = vno.dtype varatts = vno.ncattrs() if searchInlist(varatts, '_FillValue'): fillv = vno.getncattr('_FillValue') newvar = newnc.createVariable(vn, vtype, vdim, fill_value=fillv) else: newvar = newnc.createVariable(vn, vtype, vdim) for nattr in vno.ncattrs(): if not nattr == '_FillValue': nattrv = vno.getncattr(nattr) newattr = newvar.setncattr(nattr, nattrv) newvar[:] = vno[:] onc.close() newnc.sync() newnc.close() # Adding attributes fgaddattr(ncfile, ofile) sub.call(['mv',ofile,ncfile]) return #changevartype('i', 'test.nc', 'var') def add_dims(oc,nc,dnames): """ Function to add dimensions from a given netCDF object to another one oc: source netcdfile object nc: netcdfile object to add dimensions dnames: list of name of dimensions to add """ fname = 'add_dims' for dn in dnames: print ' ' + fname + ': adding dim:',dn,' ...' if oc.dimensions[dn].isunlimited(): newdim = nc.createDimension(dn, None) else: newdim = nc.createDimension(dn, len(oc.dimensions[dn])) nc.sync() return def add_vars(oc,nc,vnames): """ Function to add variables from a given netCDF object to another one oc: source netcdfile object nc: netcdfile object to add dimensions vnames: list of name of variables to add """ fname = 'add_vars' for vn in vnames: if not searchInlist(nc.variables,vn): print ' ' + fname + ': adding var:',vn,' ...' varo = oc.variables[vn] vartype = varo.dtype vardims = varo.dimensions varattrs = varo.ncattrs() for vdn in vardims: if not searchInlist(nc.dimensions,vdn): print warnmsg print ' ' + fname + ": adding dimension '" + vdn + \ "' from variable '" + vdn + "' which is not in file !!" add_dims(oc,nc,[vdn]) if searchInlist(varattrs,'_FillValue'): newvar = nc.createVariable(vn, vartype, vardims, \ fill_value=varo.getncattr('_FillValue')) else: newvar = nc.createVariable(vn, vartype, vardims) for attrn in varattrs: attrv = varo.getncattr(attrn) newattr = set_attribute(newvar, attrn, attrv) newvar[:] = varo[:] nc.sync() return def add_globattrs(oc,nc,attrns): """ Function to add global attributes from a given netCDF object to another one oc: source netcdfile object nc: netcdfile object to add dimensions vnames: list of name of dimensions to add ('all', for all attributes) """ fname = 'add_globattrs' allfattrs = oc.ncattrs() if attrns == 'all': fattrns = allfattrs else: fattrns = attrns for an in fattrns: if not searchInlist(allfattrs,an): print errormsg print ' ' + fname + ": file has not global attribute '" + an + "' !!" quit(-1) print ' ' + fname + ': adding attr:',an,' ...' av = oc.getncattr(an) newattr = set_attribute(nc, an, av) nc.sync() return def selvar(values, ncfile, varn): """ Function to select a series of variables from a netcdf file. Variables with the values of the associated dimensions will be also retrieved values= ',' list of couples [dimname]@[vardimname] [dimname]: name of the dimension [vardimname]: name of the variable with values for the dimension ('int', for counter of integers with the dimension length) ncfile= netCDF file to use varnames= ',' list of variables to retrieve """ fname = 'selvar' if values == 'h': print fname + '_____________________________________________________________' print selvar.__doc__ quit() varnames = varn.split(',') dimvals = values.split(',') ofile = fname + '_new.nc' onc = NetCDFFile(ncfile, 'r') for varn in varnames: if not searchInlist(onc.variables, varn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not variable '" + \ varn + "' !!" quit(-1) dimsv = {} vardims = [] for dimv in dimvals: dimn = dimv.split('@')[0] if not searchInlist(onc.dimensions, dimn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not dimension '" + \ dimn + "' !!" quit(-1) varn = dimv.split('@')[1] if not varn == 'int' and not searchInlist(onc.variables, varn): print errormsg print ' ' + fname + ": file '" + ncfile + "' has not variable '" + \ varn + "' !!" quit(-1) dimsv[dimn] = varn if varn != 'int': vardims.append(varn) newnc = NetCDFFile(ofile, 'w') # Creation of dimensions add_dims(onc,newnc,dimsv.keys()) # Getting variables from dimensions add_vars(onc,newnc,vardims) # Getting variables for varn in varnames: varobj = onc.variables[varn] vardims = varobj.dimensions for dimn in vardims: if not searchInlist(vardims,dimn): print errormsg print ' ' + fname + ": no assigned variable to dimension '" + \ dimn + "' !!" quit(-1) add_vars(onc,newnc,varnames) # Getting global attributes add_globattrs(onc,newnc,'all') onc.close() newnc.close() print "Successfull written of file '" + ofile + "' !!" return #selvar('x@lon,y@lat,time@time,shan@int', '/home/lluis/PY/test.nc', 'var') def coincident_CFtimes(tvalB, tunitA, tunitB): """ Function to make coincident times for two different sets of CFtimes tvalB= time values B tunitA= time units times A to which we want to make coincidence tunitB= time units times B >>> coincident_CFtimes(np.arange(10),'seconds since 1949-12-01 00:00:00', 'hours since 1949-12-01 00:00:00') [ 0. 3600. 7200. 10800. 14400. 18000. 21600. 25200. 28800. 32400.] >>> coincident_CFtimes(np.arange(10),'seconds since 1949-12-01 00:00:00', 'hours since 1979-12-01 00:00:00') [ 9.46684800e+08 9.46688400e+08 9.46692000e+08 9.46695600e+08 9.46699200e+08 9.46702800e+08 9.46706400e+08 9.46710000e+08 9.46713600e+08 9.46717200e+08] """ import datetime as dt fname = 'coincident_CFtimes' trefA = tunitA.split(' ')[2] + ' ' + tunitA.split(' ')[3] trefB = tunitB.split(' ')[2] + ' ' + tunitB.split(' ')[3] tuA = tunitA.split(' ')[0] tuB = tunitB.split(' ')[0] if tuA != tuB: if tuA == 'microseconds': if tuB == 'microseconds': tB = tvalB*1. elif tuB == 'seconds': tB = tvalB*10.e6 elif tuB == 'minutes': tB = tvalB*60.*10.e6 elif tuB == 'hours': tB = tvalB*3600.*10.e6 elif tuB == 'days': tB = tvalB*3600.*24.*10.e6 else: print errormsg print ' ' + fname + ": combination of time untis: '" + tuA + \ "' & '" + tuB + "' not ready !!" quit(-1) elif tuA == 'seconds': if tuB == 'microseconds': tB = tvalB/10.e6 elif tuB == 'seconds': tB = tvalB*1. elif tuB == 'minutes': tB = tvalB*60. elif tuB == 'hours': tB = tvalB*3600. elif tuB == 'days': tB = tvalB*3600.*24. else: print errormsg print ' ' + fname + ": combination of time untis: '" + tuA + \ "' & '" + tuB + "' not ready !!" quit(-1) elif tuA == 'minutes': if tuB == 'microseconds': tB = tvalB/(60.*10.e6) elif tuB == 'seconds': tB = tvalB/60. elif tuB == 'minutes': tB = tvalB*1. elif tuB == 'hours': tB = tvalB*60. elif tuB == 'days': tB = tvalB*60.*24. else: print errormsg print ' ' + fname + ": combination of time untis: '" + tuA + \ "' & '" + tuB + "' not ready !!" quit(-1) elif tuA == 'hours': if tuB == 'microseconds': tB = tvalB/(3600.*10.e6) elif tuB == 'seconds': tB = tvalB/3600. elif tuB == 'minutes': tB = tvalB/60. elif tuB == 'hours': tB = tvalB*1. elif tuB == 'days': tB = tvalB*24. else: print errormsg print ' ' + fname + ": combination of time untis: '" + tuA + \ "' & '" + tuB + "' not ready !!" quit(-1) elif tuA == 'days': if tuB == 'microseconds': tB = tvalB/(24.*3600.*10.e6) elif tuB == 'seconds': tB = tvalB/(24.*3600.) elif tuB == 'minutes': tB = tvalB/(24.*60.) elif tuB == 'hours': tB = tvalB/24. elif tuB == 'days': tB = tvalB*1. else: print errormsg print ' ' + fname + ": combination of time untis: '" + tuA + \ "' & '" + tuB + "' not ready !!" quit(-1) else: print errormsg print ' ' + fname + ": time untis: '" + tuA + "' not ready !!" quit(-1) else: tB = tvalB*1. if trefA != trefB: trefTA = dt.datetime.strptime(trefA, '%Y-%m-%d %H:%M:%S') trefTB = dt.datetime.strptime(trefB, '%Y-%m-%d %H:%M:%S') difft = trefTB - trefTA diffv = difft.days*24.*3600.*10.e6 + difft.seconds*10.e6 + difft.microseconds print ' ' + fname + ': different reference refA:',trefTA,'refB',trefTB print ' difference:',difft,':',diffv,'microseconds' if tuA == 'microseconds': tB = tB + diffv elif tuA == 'seconds': tB = tB + diffv/10.e6 elif tuA == 'minutes': tB = tB + diffv/(60.*10.e6) elif tuA == 'hours': tB = tB + diffv/(3600.*10.e6) elif tuA == 'dayss': tB = tB + diffv/(24.*3600.*10.e6) else: print errormsg print ' ' + fname + ": time untis: '" + tuA + "' not ready !!" quit(-1) return tB def wdismean(pos,val4): """ Function to compute the mean value weighted to its 4 distances pos= x,y position within the 'square' val4= values at the four vertexs (2,2) >>>position = [0.005,0.005] >>>val4 = np.zeros((2,2), dtype=np.float) >>>val4 = np.arange(4).reshape(2,2) 0.035707955234 """ fname = 'wdismean' dist = np.zeros((2,2), dtype=np.float) dist[0,0] = np.sqrt(pos[0]*pos[0]+pos[1]*pos[1]) dist[0,1] = np.sqrt(pos[0]*pos[0]+(1.-pos[1])*(1.-pos[1])) dist[1,0] = np.sqrt((1.-pos[0])*(1.-pos[0])+pos[1]*pos[1]) dist[1,1] = np.sqrt((1.-pos[0])*(1.-pos[0])+(1.-pos[1])*(1.-pos[1])) if np.min(dist) == 0.: pos0 = index_mat(dist, 0.) dist[:,:] = 0. posmean = val4[pos0[0], pos0[1]] else: totdist = np.sum(1./dist) posmean = np.sum(val4.flatten()/dist.flatten())/totdist return posmean def read_ASCIIlist(filen): """ Function to build a list from an ASCII lines file filen= name of the ASCII file """ import os fname = 'read_ASCIIlist' if not os.path.isfile(filen): print errormsg print ' ' + fname + ": ASCII types file '" + filen + "' does not exist !!" quit(-1) of = open(filen, 'r') filevals = [] for linef in of: filevals.append(linef.replace('\n','')) of.close() return filevals def radial_points(angle,dist): """ Function to provide a number of grid point positions for a given angle angle= desired angle (rad) dist= number of grid points >>> radial_points(0.5*np.pi,5) [[ 6.12323400e-17 1.00000000e+00] [ 1.22464680e-16 2.00000000e+00] [ 1.83697020e-16 3.00000000e+00] [ 2.44929360e-16 4.00000000e+00] [ 3.06161700e-16 5.00000000e+00]] """ fname = 'radial_points' radpos = np.zeros((dist,2), dtype=np.float) for ip in range(dist): radpos[ip,0] = (ip+1.)*np.cos(angle) radpos[ip,1] = (ip+1.)*np.sin(angle) return radpos def read_SQradii(filename='SQradii.dat', Npt=-1, maxradii=None): """ Function to read the outcome of the function `squared_radial' filename= Name of the outcome file of the function Npt= Number of points on each grid maxradii= allowed maximum radii 'SQradii.dat': file with the radii and paris of integers after ran squared_radial(1000) """ fname = 'read_SQradii' if not os.path.isfile(filename): print errormsg print ' ' + fname + ": file '" + filename + "' not found!!" quit(-1) infile = open(filename, 'r') gridradii = {} Nradii = {} radii = [] for line in infile: values = values_line(line, ' ', ['[', ']']) rad = np.float(values[0]) # Limited by a given radii if maxradii is not None and rad > maxradii: break Nrad = int(values[2]) couples = [] Nradgood = 0 if Nrad > 1: Ncouples = 0 for ic in range(Nrad): ix = int(values[3 + ic*2]) iy = int(values[3 + ic*2 + 1]) if ix <= Npt and iy <= Npt: couples.append(np.array([ix,iy])) Nradgood = Nradgood + 1 elif ix > Npt and iy > Npt: break else: ix = int(values[3]) iy = int(values[4]) if ix <= Npt and iy <= Npt: couples.append(np.array([ix,iy])) Nradgood = Nradgood + 1 elif ix > Npt and iy > Npt: break if Nradgood > 0: Nradii[rad] = Nradgood finalcouples = np.zeros((Nradgood,2), dtype=int) for ic in range(Nradgood): finalcouples[ic,:] = couples[ic] gridradii[rad] = finalcouples # for rad in sorted(gridradii.keys()): # print rad, gridradii[rad] return gridradii def grid_combinations(x,y): """ Function to provide all the possible grid points combination for a given pair of values x,y= pair of grid points >>>grid_combinations(1,2) [[1, 2], [-1, 2], [-1, -2], [1, -2], [2, 1], [-2, 1], [-2, -1], [2, -1]] """ fname = 'grid_combinations' gridcomb = [] gridcomb.append([x,y]) if x != 0: gridcomb.append([-x,y]) if x != 0 and y != 0: gridcomb.append([-x,-y]) if y != 0: gridcomb.append([x,-y]) if x != y: gridcomb.append([y,x]) if y != 0: gridcomb.append([-y,x]) if x != 0 and y != 0: gridcomb.append([-y,-x]) if x != 0: gridcomb.append([y,-x]) return gridcomb def radii_points(xpos,ypos,Lrad,Nang,dx,dy): """ Function to provide the grid points for radial cross sections, by angle and radii and `squared' radii xpos= x position of the center ypos= y position of the center Lrad= length of the maximum radi in grid points Nang= number of equivalent angles dx= x size of the domain of values dy= y size of the domain of values >>> radpos, SQradpos = radii_points(12,12,1,8,25,25) radpos: [[[ 13. 12. ]] [[ 12.70710678 12.70710678]] [[ 12. 13. ]] [[ 11.29289322 12.70710678]] [[ 11. 12. ]] [[ 11.29289322 11.29289322]] [[ 12. 11. ]] [[ 12.70710678 11.29289322]]] SQradpos: {1.0: [[13, 12], [11, 12], [12, 13], [12, 11]], 1.41421356237: [[13, 13], [11, 13], [11, 11], [13, 11]]} """ fname = 'radii_points' radiipos = np.zeros((Nang, Lrad, 2), dtype = np.float) SQradiipos = {} # Getting the range of radii: pairsSQradii = read_SQradii(Npt=Lrad) SQradii = pairsSQradii.keys() Nradii = len(SQradii) # Angle loop for ia in range(Nang): angle = 2.*np.pi*ia/Nang posangle = np.zeros((Lrad,2), dtype=np.float) # Points at that given angle pangle = radial_points(angle,Lrad) posangle[:,0] = pangle[:,0] + xpos posangle[:,1] = pangle[:,1] + ypos for ip in range(Lrad): iipos = int(posangle[ip,0]) jjpos = int(posangle[ip,1]) # Creation of a matrix with all the grid points at each time-step if iipos >= 0 and iipos < dx-1 and jjpos >= 0 and jjpos < dy-1: radiipos[ia,ip,0] = posangle[ip,0] radiipos[ia,ip,1] = posangle[ip,1] # SQ radii values # Radii loop (avoiding 0) pairswithin = {} for ir in range(1,Nradii): pairs = pairsSQradii[SQradii[ir]] # pairs loop Npairs = len(pairs.flatten())/2 pairsw = [] for ip in range(Npairs): if Npairs == 0: break else: allpairs = grid_combinations(pairs[ip,0],pairs[ip,1]) for iap in range(len(allpairs)): ppos = allpairs[iap] iipos = xpos + ppos[0] jjpos = ypos + ppos[1] # Creation of a matrix with all the grid points at each time-step if iipos >= 0 and iipos < dx and jjpos >= 0 and jjpos < dy: pairsw.append([iipos,jjpos]) SQradiipos[SQradii[ir]] = pairsw return radiipos, SQradiipos def hor_weight_int(SWval, SEval, NWval, NEval, xpos, ypos): """ Function to weighted by distance interpolate a horizontal value from it closest 4 neighbourgs field= a 4 points representing the environment of a given point xpos, ypos: position to which one want to interpolate (relative to 0,0 at the corner SW) >>> hor_weight_int(1., 1., 1., 2., 0.75, 0.75) 1.44888128193 """ fname = 'hor_weight_int' vals = np.array([SWval, SEval, NWval, NEval]) print vals if xpos > 1.: print errormsg print ' ' + fname + ': Wrong position to interpolate! (',xpos,',',ypos, \ ') must be within (1,1) !!' quit(-1) vertexs = np.zeros((4,2), dtype=np.float) vertexs[0,0] = 0. vertexs[0,1] = 0. vertexs[1,0] = 1. vertexs[1,1] = 0. vertexs[2,0] = 0. vertexs[2,1] = 1. vertexs[3,0] = 1. vertexs[3,1] = 1. print vertexs dist = np.sqrt((vertexs[:,0]-xpos)**2 + (vertexs[:,1]-ypos)**2) print dist done = False for id in range(4): if dist[id] == 0.: intval = vals[id] done = True if not done: intval = np.sum(vals/dist)/np.sum(1./dist) return intval def compute_tevolboxtraj_radialsec(values, ncfile, varn): """ Function to compute tevolboxtraj_radialsec: temporal evolution at a given point along a number of radii at a given frequency following a trajectory values= [trajfile]@[Tbeg],[lonname],[latname],[zname],[timename],[timekind],[Nangle],[radii] [trajfile]: ASCII file with the trajectory ('#' not readed) [time] [xpoint] [ypoint] [Tbeg]: equivalent first time-step of the traqjectory within the netCDF file [lonname],[latname],[zname],[timename]: longitude, latitude, z and time variables names [timekind]: kind of time 'cf': cf-compilant 'wrf': WRF kind [Nangle]: Number of angles [radii]: length in grid points of the radius ncfile= netCDF file to use varn= ',' list of variables' name ('all', for all variables) """ import numpy.ma as ma import subprocess as sub fname='compute_tevolboxtraj_radialsec' if values == 'h': print fname + '_____________________________________________________________' print compute_tevolboxtraj_radialsec.__doc__ quit() arguments = '[trajfile]@[Tbeg],[lonname],[latname],[zname],[timename],' + \ '[timekind],[Nangle],[radii]]' check_arguments(fname,values,arguments,',') trajfile = values.split(',')[0].split('@')[0] Tbeg = int(values.split(',')[0].split('@')[1]) lonn = values.split(',')[1] latn = values.split(',')[2] zn = values.split(',')[3] timn = values.split(',')[4] timekind = values.split(',')[5] Nangle = int(values.split(',')[6]) radii = int(values.split(',')[7]) if not os.path.isfile(ncfile): print errormsg print ' ' + fname + " netCDF file: '" + ncfile + "' does not exist !!" quit(-1) if not os.path.isfile(trajfile): print errormsg print ' ' + fname + " trajectory file: '" + trajfile + "' does not exist !!" quit(-1) objfile = NetCDFFile(ncfile, 'r') lonobj = objfile.variables[lonn] latobj = objfile.variables[latn] timobj = objfile.variables[timn] if varn.find(',') != -1: varns = varn.split(',') else: if varn == 'all': varns = objfile.variables else: varns = [varn] lonv, latv = lonlat2D(lonobj[:],latobj[:]) dimx = lonv.shape[1] dimy = lonv.shape[0] timv = timobj[:] # Selecting accordingly a trajectory ## Ttraj = file_nlines(trajfile,'#') if timekind == 'wrf': dimt = objfile.variables[timn].shape[0] else: dimt = objfile.variables[timn].shape # Removing this checking # if Tbeg + Ttraj > dimt: # print errormsg # print ' ' + fname + ': trajectory has ', Ttraj, ' time steps and starts ' + \ # ' at',Tbeg,' and data',varobj.shape[0], '" !!!!!' # quit(-1) print ' ' + fname + ': Number of time-steps in trajectory file: ',Ttraj trajobj = open(trajfile,'r') # Trajectory values/slices ## # This is how we're going to proceed: # Creation of 1 matrix and 1 dictionay with the grid points where values have to be taken # trjradii(dimt,Nangle,Nrad,2): x,y pairs at each radii, angle and time step. # Ditance interpolation might be required! # SQtrjradii(dimt,2): x,y pairs at each radii, number of pairs (Npair) and time # step. Flip/rotation of points might be required! trjradii = np.zeros((dimt, Nangle, radii, 2), dtype = int) trjSQradii = {} TOTradii = [] iline = 0 it = 0 trajpos = np.zeros((Ttraj,Nangle,radii), dtype=np.float) gtrajvals = np.zeros((Ttraj,3), dtype=int) trajvals = np.zeros((Ttraj,3), dtype=np.float) # Grid position at each time-step and angle # varvalst = np.ones((Ttraj, Nangle, radii, 2), dtype=np.float)*fillValue # varvalstSQ = np.zeros((Ttraj, radii, radii), dtype=bool) it = 0 iline = 0 for line in trajobj: ## Skipping first line ## if not iline == 0: ## it = iline - 1 ## it = iline # Slicing brings to reduce 1 time-step.... ??? if line[0:1] != '#': gtrajvals[it,0] = Tbeg + iline gtrajvals[it,1] = int(line.split(' ')[1]) gtrajvals[it,2] = int(line.split(' ')[2]) print it,'t:',gtrajvals[it,0],'x y:', gtrajvals[it,1], gtrajvals[it,2] if timekind == 'wrf': gdate = datetimeStr_conversion(timv[gtrajvals[it,0]],'WRFdatetime', \ 'matYmdHMS') trajvals[it,0] = realdatetime1_CFcompilant(gdate, '19491201000000', \ 'hours') tunits = 'hours since 1949-12-01 00:00:00' elif timekind == 'cf': trajvals[it,0] = timv[gtrajvals[it,0]] tunits = timobj.getncattr('units') else: print errormsg print ' ' + fname + ' time kind: "' + timekind + '" not ready !!' quit(-1) # print iline, it,'time:',trajvals[it,0],'lon lat:', trajvals[it,1], \ # trajvals[it,2] # Assuming time as the first dimension in a fortran like way (t=0 as 1) # trajcut[0] = tstept - 1 # Assuming time as the first dimension in a C/python like way (t=0) posangle = np.zeros((radii,2), dtype=np.float) trjradii[it,:,:,:], trjSQradii[it] = radii_points(gtrajvals[it,1], \ gtrajvals[it,2],radii,Nangle,dimx,dimy) if it == 0: TOTradii = list(trjSQradii[it].keys()) else: for ir in trjSQradii[it].keys(): if searchInlist(TOTradii,ir): TOTradii.append(ir) it = it + 1 iline = iline + 1 trajobj.close() # Creation of the netCDF file ## NtotSQradii = len(TOTradii) TOTSQradii = sorted(TOTradii) print ' ' + fname + ':',NtotSQradii,' squared radii:',TOTSQradii if varn.find(',') != -1: varnS = 'multi-var' else: varnS = varn.replace(',','-') ofile = 'tevolradiitraj_' + varnS + '.nc' objofile = NetCDFFile(ofile, 'w') boxs = radii*2+1 # Dimensions newdim = objofile.createDimension('x', boxs) newdim = objofile.createDimension('y', boxs) newdim = objofile.createDimension('time', None) newdim = objofile.createDimension('radii', radii) newdim = objofile.createDimension('SQradii', NtotSQradii) stsn = ['min', 'max', 'mean', 'mean2', 'stdev', 'ac'] vstlname = ['minimum value within', 'maximum value within', 'mean value within', \ 'squared mean value within', 'standard deviation value within', \ 'accumulated value within'] Nsts = len(stsn) statnames = [] cstatnames = [] for i in range(Nsts): statnames.append(stsn[i] + 'radii') cstatnames.append(stsn[i] + 'SQradii') # Getting values ## ivar = 0 box2=0 for vn in varns: vnst = variables_values(vn)[0] if not searchInlist(objfile.variables, vn): print errormsg print ' ' + fname + ": variable name '" + vn + "' is not in file " + \ ncfile + '" !!!!!' quit(-1) varobj = objfile.variables[vn] Nvardims = len(varobj.shape) print ' ' + fname + ": getting '" + vn + "' ... .. ." slicev = [] slice2D = [] slicevnoT = [] cslicev = [] cslice2D = [] cslicevnoT = [] if Nvardims == 4: # Too optimistic, but at this stage... if not objofile.dimensions.has_key('z'): varzobj = objfile.variables[zn] if len(varzobj.shape) == 1: dimz = varzobj.shape zvals = varzobj[:] elif len(varzobj.shape) == 2: dimz = varzobj.shape[1] zvals = varzobj[0,:] elif len(varzobj.shape) == 3: dimz = varzobj.shape[2] zvals = varzobj[0,0,:] objofile.createDimension('z', dimz) newvar = objofile.createVariable(zn, 'f4', ('z'),fill_value=fillValue) if searchInlist(varzobj.ncattrs(),'standard_name'): vsname = varzobj.getncattr('standard_name') else: vsname = variables_values(zn)[1] if searchInlist(varzobj.ncattrs(),'long_name'): vlname = varzobj.getncattr('long_name') else: vlname = variables_values(zn)[4].replace('|',' ') if searchInlist(varzobj.ncattrs(),'units'): vunits = varzobj.getncattr('units') else: vunits = variables_values(zn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newvar[:] = zvals varvals = np.ones(tuple([Ttraj,dimz,boxs,boxs]), dtype=np.float) lonvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) latvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) radiivarvals = np.ones(tuple([Ttraj,dimz,radii]), dtype=np.float)*fillValueF SQradiivarvals = np.ones(tuple([Ttraj,dimz,NtotSQradii]), dtype=np.float)*fillValueF # One dimension plus for the values at the center of the trajectory statvarvals = np.ones(tuple([Ttraj,dimz,Nsts+1]), dtype=np.float) rstatvarvals = np.ones(tuple([Ttraj,dimz,Nsts+1]), dtype=np.float) for it in range(Ttraj): it0 = Tbeg + it # Radii values # trjradii[it,:,:,:], trjSQradii[it] for ir in range(radii): for ia in range(Nangle): xp = trjradii[it,ia,ir,0] yp = trjradii[it,ia,ir,1] xpI = int(xp*1.) ypI = int(yp*1.) print 'xp yp:',xp,yp,'xpI ypI:',xpI,ypI quit() mean = 0. for iz in range(dimz): SWv = varobj[it0,iz,ypI,xpI] SEv = varobj[it0,iz,ypI,xpI+1] NWv = varobj[it0,iz,ypI+1,xpI] NEv = varobj[it0,iz,ypI+1,xpI+1] val = hor_weight_int(SEv, SWv, NEv, NWv, xp-xpI, yp-ypI) print 'Lluis hor_weight_int(',SEv,',', SWv,',', NEv,',', NWv,',', xp-xpI,',', yp-ypI,')' print val mean = mean + val print 'mean:',mean quit() slicev = [] slice2D = [] slicevnoT = [] cslicev = [] cslice2D = [] cslice2Dhor = [] cslicevnoT = [] cslicevnoThor = [] slicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-box2 < 0 or gtrajvals[it,2]+box2 + 1 > dimy + 1 \ or gtrajvals[it,1]-box2 < 0 or gtrajvals[it,1]+box2 + 1 > dimx +1: # box values slicev.append(slice(0,dimz)) slicev.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicev.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slicevnoT.append(slice(0,dimz)) slicevnoT.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicevnoT.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slice2D.append(slice(0,dimz)) slice2D.append(slice(0,yrangeslice[it][1]-yrangeslice[it][0])) slice2D.append(slice(0,xrangeslice[it][1]-xrangeslice[it][0])) rvarvalst = np.ones((dimz, Nrad*2+1, Nrad*2+1),dtype=np.float)* \ fillValue varvalst[tuple(slice2D)] = varobj[tuple(slicev)] varvals[it,:,:,:] = varvalst # box stats values maskedvals = ma.masked_values (varvalst, fillValue) maskedvals2 = maskedvals*maskedvals for iz in range(dimz): statvarvals[it,iz,0] = varvalst[iz,box2,box2] statvarvals[it,iz,1] = np.min(varvalst[iz,:,:]) statvarvals[it,iz,2] = np.max(varvalst[iz,:,:]) statvarvals[it,iz,3] = np.mean(varvalst[iz,:,:]) statvarvals[it,iz,4] = maskedvals2[iz,:,:].mean() statvarvals[it,iz,5] = np.sqrt(statvarvals[it,iz,4] - \ statvarvals[it,iz,3]*statvarvals[it,iz,3]) statvarvals[it,iz,6] = np.sum(varvalst[iz,:,:]) else: slicev.append(slice(0,dimz)) slicev.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2+1)) slicev.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2+1)) slicevnoT.append(slice(0,dimz)) slicevnoT.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+ \ box2+1)) slicevnoT.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+ \ box2+1)) slice2D.append(slice(0,dimz)) slice2D.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2] + \ box2 + 1)) slice2D.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1] + \ box2 + 1)) varvalst = varobj[tuple(slicev)] # box values varvals[it,:,:,:] = varvalst # print 'values at time t______' # print varvalst # box stats values for iz in range(dimz): statvarvals[it,:,0] = varvalst[:,box2,box2] statvarvals[it,iz,1] = np.min(varvalst[iz,:,:]) statvarvals[it,iz,2] = np.max(varvalst[iz,:,:]) statvarvals[it,iz,3] = np.mean(varvalst[iz,:,:]) statvarvals[it,iz,4] = np.mean(varvalst[iz,:,:]* \ varvalst[iz,:,:]) statvarvals[it,iz,5] = np.sqrt(statvarvals[it,iz,4] - \ statvarvals[it,iz,3]*statvarvals[it,iz,3]) statvarvals[it,iz,6] = np.sum(varvalst[iz,:,:]) # Circle values cslicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-Nrad < 0 or gtrajvals[it,2]+Nrad + 1 >= dimy \ or gtrajvals[it,1]-Nrad < 0 or gtrajvals[it,1]+Nrad + 1 >= dimx: maxx = np.min([cxrangeslice[it][1], dimx]) maxy = np.min([cyrangeslice[it][1], dimy]) cslicev.append(slice(0,dimz)) cslicev.append(slice(cyrangeslice[it][0], maxy)) cslicev.append(slice(cxrangeslice[it][0], maxx)) cslicevnoT.append(slice(0,dimz)) cslicevnoT.append(slice(cyrangeslice[it][0], cyrangeslice[it][1])) cslicevnoT.append(slice(cxrangeslice[it][0], cxrangeslice[it][1])) cslice2D.append(slice(0,dimz)) cslice2D.append(slice(0,maxy-cyrangeslice[it][0])) cslice2D.append(slice(0,maxx-cxrangeslice[it][0])) cslice2Dhor.append(slice(0, maxy - cyrangeslice[it][0])) cslice2Dhor.append(slice(0, maxx -cxrangeslice[it][0])) rvarvalst = np.ones((dimz,Nrad*2+1,Nrad*2+1),dtype=np.float)* \ fillValue rvarvalst[tuple(cslice2D)] = varobj[tuple(cslicev)] for iz in range(dimz): tslice = [slice(it,it)]+cslice2Dhor zslice = [slice(iz,iz)]+cslice2Dhor rvarvalst[tuple(zslice)] = np.where(circdist[tuple(tslice)] > \ np.float(Nrad), fillValue, rvarvalst[tuple(zslice)]) rvarvals[it,:,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) maskedvals2 = maskedvals*maskedvals for iz in range(dimz): rstatvarvals[it,iz,0] = varvalst[iz,box2,box2] rstatvarvals[it,iz,1] = np.min(varvalst[iz,:,:]) rstatvarvals[it,iz,2] = np.max(varvalst[iz,:,:]) rstatvarvals[it,iz,3] = np.mean(varvalst[iz,:,:]) rstatvarvals[it,iz,4] = maskedvals2[iz,:,:].mean() rstatvarvals[it,iz,5] = np.sqrt(rstatvarvals[it,iz,4] - \ rstatvarvals[it,iz,3]*rstatvarvals[it,iz,3]) rstatvarvals[it,iz,6] = np.sum(varvalst[iz,:,:]) else: cslicev.append(slice(0,dimz)) cslicev.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslicev.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) cslicevnoT.append(slice(0,dimz)) cslicevnoT.append(slice(gtrajvals[it,2]-Nrad, gtrajvals[it,2]+ \ Nrad+1)) cslicevnoT.append(slice(gtrajvals[it,1]-Nrad, gtrajvals[it,1]+ \ Nrad+1)) cslicevnoThor.append(slice(gtrajvals[it,2]-Nrad, \ gtrajvals[it,2] + Nrad+1)) cslicevnoThor.append(slice(gtrajvals[it,1]-Nrad, \ gtrajvals[it,1] + Nrad+1)) cslice2D.append(slice(0,dimz)) cslice2D.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2] + \ Nrad+1)) cslice2D.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1] + \ Nrad+1)) rvarvalst = varobj[tuple(cslicev)] # circle values for iz in range(dimz): tslice = [it]+cslicevnoThor rvarvalst[iz,:,:] = np.where(circdist[tuple(tslice)] > \ np.float(Nrad), fillValue, rvarvalst[iz,:,:]) rvarvals[it,:,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) maskedvals2 = maskedvals*maskedvals for iz in range(dimz): rstatvarvals[it,iz,0] = rvarvalst[iz,Nrad,Nrad] rstatvarvals[it,iz,1] = maskedvals[iz,:,:].min() rstatvarvals[it,iz,2] = maskedvals[iz,:,:].max() rstatvarvals[it,iz,3] = maskedvals[iz,:,:].mean() rstatvarvals[it,iz,4] = maskedvals2[iz,:,:].mean() rstatvarvals[it,iz,5] = np.sqrt(rstatvarvals[it,iz,4] - \ rstatvarvals[it,iz,3]*rstatvarvals[it,iz,3]) rstatvarvals[it,iz,6] = maskedvals[iz,:,:].sum() # print 'statistics:',rstatvarvals[it,:] # variable box values newvar = objofile.createVariable(vnst + 'box', 'f4', ('time','z','y','x'),\ fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = varvals # center of the trajectory newvar = objofile.createVariable('trj_' + vnst, 'f', ('time','z'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'trj_' + vsname, 'value along the ' + \ 'trajectory of '+ vn, vunits) newvar[:] = statvarvals[:,:,0] # variable box statistics ist = 0 for statn in statnames: newvar = objofile.createVariable(statn + '_' + vnst,'f',('time','z'),\ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the box ('+str(boxs)+'x'+str(boxs)+') of ' + vnst, vunits) newvar[:] = statvarvals[:,:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 # variable circle values newvar = objofile.createVariable(vnst + 'circle','f4',('time','z','yr', \ 'xr'), fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = rvarvals # variable circle statistics ist = 0 for statn in cstatnames: newvar = objofile.createVariable(statn + '_' + vnst,'f',('time','z'),\ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the circle of radius ('+ str(circler)+') of ' + vnst, vunits) newvar[:] = rstatvarvals[:,:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 elif Nvardims == 3: # Too optimistic, but at this stage... dimt = varobj.shape[0] Nrad=1 varvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) lonvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) latvals = np.ones(tuple([Ttraj,boxs,boxs]), dtype=np.float) rvarvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) rlonvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) rlatvals = np.ones(tuple([Ttraj,Nrad*2+1,Nrad*2+1]), dtype=np.float) # One dimension plus for the values at the center of the trajectory statvarvals = np.ones(tuple([Ttraj,Nsts+1]), dtype=np.float) rstatvarvals = np.ones(tuple([Ttraj,Nsts+1]), dtype=np.float) for it in range(Ttraj): it0 = Tbeg + it slicev = [] slice2D = [] slicevnoT = [] cslicev = [] cslice2D = [] cslicevnoT = [] cslicevC = [] slicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-box2 < 0 or gtrajvals[it,2]+box2 + 1 > dimy + 1 \ or gtrajvals[it,1]-box2 < 0 or gtrajvals[it,1]+box2 + 1 > dimx + 1: # box values slicev.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicev.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slicevnoT.append(slice(yrangeslice[it][0],yrangeslice[it][1])) slicevnoT.append(slice(xrangeslice[it][0],xrangeslice[it][1])) slice2D.append(slice(0,yrangeslice[it][1]-yrangeslice[it][0])) slice2D.append(slice(0,xrangeslice[it][1]-xrangeslice[it][0])) rvarvalst = np.ones((Nrad*2+1, Nrad*2+1),dtype=np.float)*fillValue varvalst[tuple(slice2D)] = varobj[tuple(slicev)] varvals[it,:,:] = varvalst # box stats values maskedvals = ma.masked_values (varvalst, fillValue) statvarvals[it,0] = varvalst[box2,box2] statvarvals[it,1] = maskedvals.min() statvarvals[it,2] = maskedvals.max() statvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals statvarvals[it,4] = maskedvals2.mean() statvarvals[it,5] = np.sqrt(statvarvals[it,4] - \ statvarvals[it,3]*statvarvals[it,3]) statvarvals[it,6] = maskedvals.sum() varvalst[tuple(slice2D)] = lonv[tuple(slicevnoT)] lonvals[it,:,:] = varvalst varvalst[tuple(slice2D)] = latv[tuple(slicevnoT)] latvals[it,:,:] = varvalst else: slicev.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2+1)) slicev.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2+1)) slicevnoT.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+ \ box2+1)) slicevnoT.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+ \ box2+1)) slice2D.append(slice(gtrajvals[it,2]-box2, gtrajvals[it,2]+box2 +\ 1)) slice2D.append(slice(gtrajvals[it,1]-box2, gtrajvals[it,1]+box2 +\ 1)) varvalst = varobj[tuple(slicev)] # box values varvals[it,:,:] = varvalst # print 'values at time t______' # print varvalst # box stats values statvarvals[it,0] = varvalst[box2,box2] statvarvals[it,1] = np.min(varvalst) statvarvals[it,2] = np.max(varvalst) statvarvals[it,3] = np.mean(varvalst) statvarvals[it,4] = np.mean(varvalst*varvalst) statvarvals[it,5] = np.sqrt(statvarvals[it,4] - \ statvarvals[it,3]*statvarvals[it,3]) statvarvals[it,6] = np.sum(varvalst) varvalst = lonv[tuple(slicevnoT)] lonvals[it,:,:] = varvalst varvalst = latv[tuple(slicevnoT)] latvals[it,:,:] = varvalst # Circle values cslicev.append(gtrajvals[it,0]) if gtrajvals[it,2]-Nrad < 0 or gtrajvals[it,2]+Nrad + 1 >= dimy \ or gtrajvals[it,1]-Nrad < 0 or gtrajvals[it,1]+Nrad + 1 >= dimx: maxx = np.min([cxrangeslice[it][1], dimx]) maxy = np.min([cyrangeslice[it][1], dimy]) cslicev.append(slice(cyrangeslice[it][0],maxy)) cslicev.append(slice(cxrangeslice[it][0],maxx)) cslicevnoT.append(slice(cyrangeslice[it][0],cyrangeslice[it][1])) cslicevnoT.append(slice(cxrangeslice[it][0],cxrangeslice[it][1])) cslice2D.append(slice(0,maxy - cyrangeslice[it][0])) cslice2D.append(slice(0,maxx - cxrangeslice[it][0])) rvarvalst = np.ones((Nrad*2+1,Nrad*2+1),dtype=np.float)*fillValue rvarvalst[tuple(cslice2D)] = varobj[tuple(cslicev)] rvarvalst[tuple(cslice2D)] = np.where(circdist[tuple(cslicev)] >\ np.float(Nrad), fillValue, rvarvalst[tuple(cslice2D)]) rvarvals[it,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) rstatvarvals[it,0] = rvarvalst[Nrad,Nrad] rstatvarvals[it,1] = maskedvals.min() rstatvarvals[it,2] = maskedvals.max() rstatvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals rstatvarvals[it,4] = maskedvals2.mean() rstatvarvals[it,5] = np.sqrt(rstatvarvals[it,4] - \ rstatvarvals[it,3]*rstatvarvals[it,3]) rstatvarvals[it,6] = maskedvals2.sum() else: cslicev.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslicev.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) cslicevnoT.append(slice(gtrajvals[it,2]-Nrad, gtrajvals[it,2]+ \ Nrad+1)) cslicevnoT.append(slice(gtrajvals[it,1]-Nrad, gtrajvals[it,1]+ \ Nrad+1)) cslice2D.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslice2D.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) cslicevC.append(it) cslicevC.append(slice(gtrajvals[it,2]-Nrad,gtrajvals[it,2]+Nrad+1)) cslicevC.append(slice(gtrajvals[it,1]-Nrad,gtrajvals[it,1]+Nrad+1)) rvarvalst = varobj[tuple(cslicev)] cdist = circdist[tuple(cslicevC)] # circle values rvarvalst = np.where(cdist > np.float(Nrad),fillValue,rvarvalst) rvarvals[it,:,:] = rvarvalst # circle stats values maskedvals = ma.masked_values (rvarvalst, fillValue) rstatvarvals[it,0] = rvarvalst[Nrad,Nrad] rstatvarvals[it,1] = maskedvals.min() rstatvarvals[it,2] = maskedvals.max() rstatvarvals[it,3] = maskedvals.mean() maskedvals2 = maskedvals*maskedvals rstatvarvals[it,4] = maskedvals2.mean() rstatvarvals[it,5] = np.sqrt(rstatvarvals[it,4] - \ rstatvarvals[it,3]*rstatvarvals[it,3]) rstatvarvals[it,6] = maskedvals.sum() # print 'statistics:',rstatvarvals[it,:] # variable box values newvar = objofile.createVariable(vnst + 'box', 'f4', ('time', 'y', 'x'), \ fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = varvals # center of the trajectory newvar = objofile.createVariable('trj_' + vnst, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'trj_' + vsname, 'value along the ' + \ 'trajectory of '+ vnst, vunits) newvar[:] = statvarvals[:,0] # variable box statistics ist = 0 for statn in statnames: newvar = objofile.createVariable(statn + '_' + vnst, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the box ('+str(boxs)+'x'+str(boxs)+') of ' + vnst, vunits) newvar[:] = statvarvals[:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 # variable circle values newvar = objofile.createVariable(vnst + 'circle','f4',('time','yr','xr'),\ fill_value=fillValue) if searchInlist(varobj.ncattrs(),'standard_name'): vsname = varobj.getncattr('standard_name') else: vsname = variables_values(vn)[1] if searchInlist(varobj.ncattrs(),'long_name'): vlname = varobj.getncattr('long_name') else: vlname = variables_values(vn)[4].replace('|',' ') if searchInlist(varobj.ncattrs(),'units'): vunits = varobj.getncattr('units') else: vunits = variables_values(vn)[5].replace('|',' ') newattr = basicvardef(newvar, vsname, vlname, vunits) newattr = newvar.setncattr('projection','lon lat') newvar[:] = rvarvals # variable circle statistics ist = 0 for statn in cstatnames: newvar = objofile.createVariable(statn + '_' + vnst, 'f', ('time'), \ fill_value=fillValue) newattr = basicvardef(newvar, statn + '_' + vsname, vstlname[ist] + \ ' the circle of radius ('+ str(circler)+') of ' + vnst, vunits) newvar[:] = rstatvarvals[:,ist+1] # newattr = set_attributek(newvar,'_FillValue',fillValue,'npfloat') ist = ist + 1 else: # Other variables print warnmsg print ' ' + fname + ": variable '" + vn + "' shape:",varobj.shape,' not'\ + ' ready!!' print ' skipping variable' if len(varns) == 1: objofile.close() sub.call(['rm', ofile]) print ' uniq variable! removing file and finishing program' quit() if not objofile.variables.has_key('trlon') and Nvardims == 3: # var dimensions newvar = objofile.createVariable('trlon', 'f8', ('time')) newattr = basicvardef(newvar,'trlon','trajectory longitude', \ 'degrees west_east') newvar[:] = trajvals[:,1] newvar = objofile.createVariable('trlat', 'f8', ('time')) newattr = basicvardef(newvar,'trlat','trajectory latitude', \ 'degrees north_south') newvar[:] = trajvals[:,2] newvar = objofile.createVariable('time', 'f8', ('time')) newattr = basicvardef(newvar, 'time', 'time', tunits) newvar[:] = trajvals[:,0] newvar = objofile.createVariable('lon', 'f8', ('time', 'y', 'x'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'longitude', 'longitude', \ 'degrees west_east') newvar[:] = lonvals newvar = objofile.createVariable('lat', 'f8', ('time', 'y', 'x'), \ fill_value=fillValue) newattr = basicvardef(newvar, 'latitude', 'latitude', \ 'degrees north_south') newvar[:] = latvals ivar = ivar + 1 # global attributes objofile.setncattr('author', 'L. Fita') objofile.setncattr('institution', 'Laboratire de Meteorologie Dynamique') objofile.setncattr('university', 'Pierre Marie Curie - Jussieu') objofile.setncattr('center', 'Centre National de Recherches Scientifiques') objofile.setncattr('city', 'Paris') objofile.setncattr('country', 'France') objofile.setncattr('script', 'nc_var_tools.py') objofile.setncattr('function', 'compute_tevolboxtraj') objofile.setncattr('version', '1.0') objofile.setncattr('data_file',ncfile) objfile.close() objofile.sync() objofile.close() print ' ' + fname + ': successful creation of file "' + ofile + '" !!!' return # [trajfile]: ASCII file with the trajectory ('#' not readed) # [time] [xpoint] [ypoint] # [Tbeg]: equivalent first time-step of the trajectory within the netCDF file # [lonname],[latname],[zname],[timename]: longitude, latitude, z and time variables names # [timekind]: kind of time # 'cf': cf-compilant # 'wrf': WRF kind # [Nangle]: Number of angles # [radii]: length in grid points of the radius #compute_tevolboxtraj_radialsec('/bdd/PCER/workspace/lfita/etudes/FF/medic051215_2km/run/trajectory_shorten.dat@0,XLONG,XLAT,ZNU,Times,wrf,4,2', \ # '/bdd/PCER/workspace/lfita/etudes/FF/medic051215_2km/run/wrfout/wrfout_d02_2005-12-13_00:00:00', 'QVAPOR') #quit() def DatesFiles(values, ncfile, varn): """ Function to find different time values on a series of WRF files in a folder [values]= [dates],[folder] [dates]: ':' list of [YYYY][MM][DD][HH][MI][SS] dates [fold]: folder with the location of the netCDF files [ncfile]= header of the name of the files to concatenate [ncfile]* [varn]= name of the variable Time ('WRFt', for WRF time) """ import subprocess as sub import datetime as dt fname='DatesFiles' if values == 'h': print fname + '_____________________________________________________________' print DatesFiles.__doc__ quit() arguments = '[dates],[fold]' check_arguments(fname, values, arguments, ',') dates = values.split(',')[0].split(':') fold = values.split(',')[1] confiles = files_folder(fold,ncfile) Nfiles = len(confiles) print ' ' + fname +': looking on:',Nfiles,'files' # Opening all files and getting times ncobjs = {} for filen in confiles: filename = fold + '/' + filen ncf = NetCDFFile(filename, 'r') datesv = [] if varn == 'WRFt': fdates = ncf.variables['Times'][:] for it in range(fdates.shape[0]): datesv.append(datetimeStr_conversion(fdates[it,:], 'WRFdatetime', \ 'YmdHMS')) else: tunits = ncf.variables[varn].getncattr('units') fdates = ncf.variables[varn][:] for it in range(fdates.shape[0]): datesv.append(datetimeStr_conversion(fdates[it], 'cfTime,' + tunits ,\ 'YmdHMS')) ncobjs[filename] = datesv ncf.close() # Computing time-step inside files (assuming regular) if filen == confiles[0]: ini=datesv[0] end=datesv[1] iniT = dt.datetime.strptime(ini,'%Y%m%d%H%M%S') endT = dt.datetime.strptime(end,'%Y%m%d%H%M%S') DT = endT - iniT diffT = DT.total_seconds() print ' ' + fname + ': seconds between time-steps in files:', diffT Efounddates = {} Cfounddates = {} for dtv in dates: foundclosest = False for fn in ncobjs.keys(): if searchInlist(ncobjs[fn], dtv): istep = ncobjs[fn].index(dtv) print fname + 'Exact:',dtv,fn,istep,'(date, filname, time-step)' Efounddates[dtv] = [dtv,fn,istep] for it in ncobjs[fn]: iniT = dt.datetime.strptime(it,'%Y%m%d%H%M%S') endT = dt.datetime.strptime(dtv,'%Y%m%d%H%M%S') DT = endT - iniT diffitv = np.abs(DT.total_seconds()) if diffT/2. >= diffitv: istep = ncobjs[fn].index(it) print fname + 'Closest:',dtv,fn,istep,diffitv,it,'(date, ' + \ 'filname, time-step, distance in seconds, closest date)' Cfounddates[dtv] = [dtv,fn,istep,diffitv] foundclosest = True break if foundclosest: break return def TimeSplitmat(values, ncfile, vn): """ Function to transform a file with CFtimes to a matrix [Nyear,Nmonth,Nday,Nhour,Nminute,Nsecond] values= [tdim]:[tvdim]:[seltimes] tdim= name of the dimension time tvdim= name of the variable with the time values seltimes= ',' list of which times ('y', 'm', 'd', 'H','M', 'S') one wants to split the values desired final order ncfile= netcdf file to use vn= name of the variable, 'all' for all variables """ import datetime as dt fname = 'TimeSplitmat' ofile = 'TimeSplitmat.nc' if values == 'h': print fname + '_____________________________________________________________' print TimeSplitmat.__doc__ quit() arguments = '[tdim]:[tvdim]:[seltimes]' check_arguments(fname, values, arguments, ':') tdim = values.split(':')[0] tvdim = values.split(':')[1] seltimes = values.split(':')[2] of = NetCDFFile(ncfile, 'r') stimes = seltimes.split(',') if len(stimes) != 2: print errormsg print fname + ': number of splitting times',len(stimes),'not ready!!' quit(-1) if vn == 'all': vns = of.variables.keys() else: vns = [vn] otime = of.variables[tvdim] dt = otime.shape[0] if not searchInlist(otime.ncattrs(), 'units'): print errormsg print ' ' + fname + ": varible time '" + dimtv + "' has not units!!" quit(-1) tunits = otime.getncattr('units') mattimes = CFtimes_datetime(of,tvdim) Nyears = len(np.unique(mattimes[:,0])) Nmonths = len(np.unique(mattimes[:,1])) Ndays = len(np.unique(mattimes[:,2])) Nhours = len(np.unique(mattimes[:,3])) Nminutes = len(np.unique(mattimes[:,4])) Nseconds = len(np.unique(mattimes[:,5])) newdimns = [] newdimvs = [] firstmat = -1 secondmat = -1 for st in stimes: if st == 'y': newdimns.append('years') newdimvs.append(Nyears) if firstmat == -1: firstmat = 0 else: secondmat = 0 elif st == 'm': newdimns.append('months') newdimvs.append(Nmonths) if firstmat == -1: firstmat = 1 else: secondmat = 1 elif st == 'd': newdimns.append('days') newdimvs.append(Ndays) if firstmat == -1: firstmat = 2 else: secondmat = 2 elif st == 'H': newdimns.append('hours') newdimvs.append(Nhours) if firstmat == -1: firstmat = 3 else: secondmat = 3 elif st == 'M': newdimns.append('minutes') newdimvs.append(Nminutes) if firstmat == -1: firstmat = 4 else: secondmat = 4 elif st == 'S': newdimns.append('seconds') newdimvs.append(Nseconds) if firstmat == -1: firstmat = 5 else: secondmat = 5 else: print errormsg print ' ' + fname + ": selected time '" + st + "' not ready!!" quit(-1) print ' ' + fname + ':initial selection of:',newdimvs print ' ' + newdimns[0],':',np.unique(mattimes[:,firstmat]) print ' ' + newdimns[1],':',np.unique(mattimes[:,secondmat]) # We can have still multiple values (for given days, multiple months...). Assuming # 30days months newdimvs[1] = dt/newdimvs[0] print ' re-arranged to:',newdimvs newtime = np.zeros(tuple(newdimvs), dtype=np.float) # Openning new file onewfile = NetCDFFile(ofile, 'w') # Dimensions newdim = onewfile.createDimension(newdimns[0], newdimvs[0]) newdim = onewfile.createDimension(newdimns[1], newdimvs[1]) newdim = onewfile.createDimension('time', None) # Variable time newvar = onewfile.createVariable('time', 'f8', tuple(newdimns)) newvar[:] = newtime basicvardef(newvar, 'time', 'time', tunits) splitS = seltimes.replace(',','') for Vn in vns: print Vn + '...' oVn = of.variables[Vn] # Removing time dimension from the variable dimensions Vndims = oVn.dimensions if searchInlist(Vndims, tdim) and Vn != tvdim: varNOtdimns = [] varNOtdimvs = [] for idn in Vndims: if idn != tdim: varNOtdimns.append(idn) varNOtdimvs.append(len(of.dimensions[idn])) if not searchInlist(onewfile.dimensions.keys(), idn): newdim = onewfile.createDimension(idn, \ len(of.dimensions[idn])) newvarv = np.ones(tuple(newdimvs+varNOtdimvs), dtype=np.float)*fillValueF d1 = 0 d2 = -1 d1val = mattimes[0,firstmat] d2val = mattimes[0,secondmat] secondvals = [] for ddt in range(dt): if mattimes[ddt,firstmat] > d1val: d1 = d1 + 1 d1val = mattimes[ddt,firstmat] d2 = 0 else: d2 = d2 + 1 if d1 == 0: secondvals.append(mattimes[ddt,secondmat]) slicevar = [] slicenewvar = [] slicenewvar.append(d1) slicenewvar.append(d2) for idn in Vndims: if idn != tdim: slicevar.append(slice(0,len(of.dimensions[idn]))) slicenewvar.append(slice(0,len(of.dimensions[idn]))) else: slicevar.append(ddt) # print ddt,d1,d2,mattimes[ddt,:],'var:',slicevar,'new:',slicenewvar newvarv[tuple(slicenewvar)] = oVn[tuple(slicevar)] newtime[d1,d2] = otime[ddt] newvar = onewfile.createVariable(Vn + '_' + splitS, 'f', \ tuple(newdimns + varNOtdimns), fill_value = fillValueF) newattr = newvar.setncattr('desc', Vn + 'splitted by:' + newdimns[0] + \ ' & ' + newdimns[1]) newvar[:] = newvarv onewfile.sync() elif Vn != tvdim: for dvn in Vndims: if not searchInlist(onewfile.dimensions,dvn): dsize = len(of.dimensions[dvn]) newdim = onewfile.createDimension(dvn, dsize) # From http://guziy.blogspot.fr/2014/01/netcdf4-python-copying-variables-from.html vartype = oVn.dtype newvar = onewfile.createVariable(Vn, vartype, Vndims) for attrn in oVn.ncattrs(): attrv = oVn.getncattr(attrn) newattr = set_attribute(newvar, attrn, attrv) newvar[:] = oVn[:] # Variable time newvar = onewfile.variables['time'] newvar[:] = newtime # Variables selection idn=0 for newd in newdimns: print newd, onewfile.dimensions.keys() newvar = onewfile.createVariable(newd, 'f', (newd)) if idn == 0: newvar[:] = np.unique(mattimes[:,firstmat]) else: newvar[:] = secondvals set_attribute(newvar, 'desc', 'Selected ' + newdimns[idn]) idn = idn + 1 # Global attributes onewfile.setncattr('script', fname) onewfile.setncattr('version', '1.0') onewfile.setncattr('author', 'L. Fita') newattr = set_attributek(onewnfile, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') onewfile.setncattr('university', 'Pierre et Marie Curie') onewfile.setncattr('country', 'France') onewfile.setncattr('description', 'transform a file with CFtimes to a matrix ' +\ '[Nyear,Nmonth,Nday,Nhour,Nminute,Nsecond]') onewfile.sync() onewfile.close() print fname + ": Successful writting of file '" + ofile + "' !!" return #fileobj = NetCDFFile('/home/lluis/PY/ERAI_pl199501_130-133.nc', 'r') #TimeSplitmat(fileobj, 'time', 'time', 'all', 'd,H') def rmNOnum(string): """ Removing from a string all that characters which are not numbers # From: http://stackoverflow.com/questions/4289331/python-extract-numbers-from-a-string """ fname = 'rmNOnum' newstr = str(re.findall("[-+]?\d+[\.]?\d*", string)[0]) return newstr def values_fortran_fmt(lin,fFMT): """ Function to give back the values of an ASCII line according to its fortran printing format lin= ASCII line fFMT= list with the fortran FORMAT formats forline = 'Natchitoches (RGNL) 1 11 0011 ( 31.733, -93.100) ( 28, 157) ( 31.761, -93.113) 41.2 meters' formats = ['A26', 'I2', 'I3', 'A6', 'A2', 'F7.3', 'A1', 'F8.3', 'A3', 'I4', 'A1', 'I4', 'A3', 'F7.3', 'A1', 'F8.3', 'A2', 'F6.1', 'A7'] >>> values_fortran_fmt(forline, formats) ['Natchitoches (RGNL) ', 1, 11, ' 0011 ', ' ( ', 31.733, ', ', -93.1, ') ( ', 28, ', ', 157, ') ( ', 31.761, ', ', -93.113, ') ', 41.2, ' meters'] """ fname = 'values_fortran_fmt' afmts = ['A', 'D', 'F', 'I'] if lin == 'h': print fname + '_____________________________________________________________' print values_fortran_fmt.__doc__ quit() fvalues = [] ichar=0 ival = 0 for ft in fFMT: Lft = len(ft) if ft[0:1] == 'A' or ft[0:1] == 'a': Lfmt = int(ft[1:Lft+1]) fvalues.append(lin[ichar:ichar+Lfmt+1]) ichar = ichar + Lfmt elif ft[0:1] == 'F' or ft[0:1] == 'f': if ft.find('.') != -1: Lft = len(ft.split('.')[0]) Lfmt = int(ft[1:Lft]) fvalues.append(np.float(rmNOnum(lin[ichar:ichar+Lfmt+1]))) ichar = ichar + Lfmt elif ft[0:1] == 'D' or ft[0:1] == 'd': if ft.find('.') != -1: Lft = len(ft.split('.')[0]) Lfmt = int(ft[1:Lft]) fvalues.append(np.float64(rmNOnum(lin[ichar:ichar+Lfmt+1]))) ichar = ichar + Lfmt elif ft[0:1] == 'I' or ft[0:1] == 'i': Lfmt = int(ft[1:Lft+1]) fvalues.append(int(rmNOnum(lin[ichar:ichar+Lfmt+1]))) ichar = ichar + Lfmt elif ft.find('X') != -1 or ft.find('x') != -1: print ' ' + fname + ': skipping space' ichar = ichar + int(rmNOnum(ft)) else: print errormsg print ' ' + fname + ": format '" + ft[0:1] + "' not ready!!" print ' Available formats:',afmts quit(-1) ival = ival + 1 return fvalues def reduce_last_spaces(string): """ Function to reduce the last right spaces from a string string= string to remove the spaces at the righ side of the string >>> reduce_last_spaces('Hola Lluis ') 'Hola Lluis' """ fname = 'reduce_last_spaces' Lstring = len(string) firstspace = -1 for istr in range(Lstring-1,0,-1): if string[istr:istr+1] == ' ': firstspace = istr else: break if firstspace == -1: print errormsg print ' ' + fname + ": string '" + string + "' is not ended by spaces!!" print ' char. number of last character:',ord(string[Lstring:Lstring+1]) quit(-1) newstr = string[0:firstspace] return newstr def squared_radial(Npt): """ Function to provide the series of radii as composite of pairs (x,y) of gid cells Npt= largest amount of grid points on x and y directions >>> squared_radial(5) [[0.0, 0, 0], [1.0, 1, 0], [1.4142135623730951, 1, 1], [2.0, 2, 0], [2.2360679774997898, 2, 1], [2.8284271247461903, 2, 2], [3.0, 3, 0], [3.1622776601683795, 3, 1], [3.6055512754639891, 3, 2], [4.0, 4, 0], [4.1231056256176606, 4, 1], [4.2426406871192848, 3, 3], [4.4721359549995796, 4, 2], [5.0, array([4, 3]), array([5, 0])], [5.0990195135927845, 5, 1], [5.3851648071345037, 5, 2], [5.6568542494923806, 4, 4], [5.8309518948453007, 5, 3], [6.4031242374328485, 5, 4], [7.0710678118654755, 5, 5]] """ fname = 'squared_radial' radii = [] gridradii = {} singradii = [] Nradii = {} # Computing all radii ## for i in range(Npt+1): if np.mod(i,100) == 0: print 'done: ',i for j in range(i+1): # ir = np.float(i) # jr = np.float(j) ir = np.float64(i) jr = np.float64(j) rad = np.sqrt(ir*ir + jr*jr) if not searchInlist(singradii, rad): singradii.append(rad) gridradii[rad] = np.array([i,j], dtype=int) Nradii[rad] = 1 else: # print warnmsg # print ' ' + fname + ': repeated radii',rad,'!!' # print ' Previous values:', gridradii[rad] oldvalues = gridradii[rad] Nradii[rad] = Nradii[rad] + 1 newvalues = np.zeros((Nradii[rad],2), dtype=int) if Nradii[rad] == 2: newvalues[0,:] = oldvalues Nstart = 1 else: Nstart = 0 for Nr in range(Nstart,Nradii[rad]-1): newvalues[Nr,:] = oldvalues[Nr,:] newvalues[Nradii[rad]-1,:] = np.array([i,j], dtype=int) gridradii[rad] = newvalues # Sorting values ## ## Only required to downwrite the output file # radiif = open('SQradii.dat', 'w') # radii = [] # prevrad = 0. # for rad in sorted(gridradii.keys()): # dradii = rad - prevrad ## Check fo repeated values # radii.append([rad, gridradii[rad][0], gridradii[rad][1]]) # radiif.write(str(rad) + ' ' + str(dradii) + ' ' + str(Nradii[rad]) + ' [ '+ \ # numVector_String(gridradii[rad].flatten(), ' ') + ' ] \n') # prevrad = rad.copy() # radiif.close() return gridradii def getvalues_lonlat(values, ncfile): """ Function to retrieve the values from the closest grid point to a set of longitude, latitude values values=[longitude]:[latitude]:[dnames]:[vdnames] [longitude]: value for the longitude [latitude]: value for the latitude [dnames]: [xdimname],[ydimname] names of the dimensions on the 'x' and 'y' axis [dvnames]: [xvdimname],[yvdimname] names of the variables with the values for the dimensions on the 'x' and 'y' axis ncfile= netcdf file to use """ import subprocess as sub fname = 'getvalues_lonlat' ofile = fname + '.nc' if values == 'h': print fname + '_____________________________________________________________' print getvalues_lonlat.__doc__ quit() arguments = '[longitude]:[latitude]:[dnames]:[vdnames]' check_arguments(fname, values, arguments, ':') longitude = np.float(values.split(':')[0]) latitude = np.float(values.split(':')[1]) dnames = values.split(':')[2].split(',') dvnames = values.split(':')[3].split(',') onc = NetCDFFile(ncfile, 'r') for dn in dnames: if not searchInlist(onc.dimensions, dn): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have dimension '"+\ dn + "' !!" quit(-1) for vdn in dvnames: if not searchInlist(onc.variables, vdn): print errormsg print ' ' + fname + ": file '" + ncfile + "' does not have variable '"+ \ vdn + "' !!" quit(-1) # Getting grid point localization ## lon2d, lat2d = lonlat2D(onc.variables[dvnames[0]], onc.variables[dvnames[1]]) dimx = lon2d.shape[1] dimy = lon2d.shape[0] mappairs = np.ones((2), dtype=int) dist=np.sqrt((longitude - lon2d)**2. + (latitude - lat2d)**2.) mindist = np.min(dist) minydist, minxdist = index_mat(dist,mindist) mappairs[:] = [minydist, minxdist] print ' ' + fname + ': nearest point: ',mappairs, 'distance:', mindist onc.close() vals = dnames[1] + ',' + str(mappairs[1]) + ',' + str(mappairs[1]) + ',1@' vals = vals + dnames[0] + ',' + str(mappairs[0]) + ',' + str(mappairs[0]) + ',1' DataSetSection_multidims(vals, ncfile) of = ncfile.split('.')[0] + '_' + dnames[1] + '_B' + str(mappairs[1]) + '-E' + \ str(mappairs[1]) + '-I1_' + dnames[0] + '_B' + str(mappairs[0]) + '-E' + \ str(mappairs[0]) + '-I1.nc' sub.call(['mv',of,ofile]) onc = NetCDFFile(ofile, 'a') newdim = onc.createDimension('lLd',1) newvar = onc.createVariable('lon_point','f',('lLd')) newattr = basicvardef(newvar,'lon_point','longitude of selection of the values', \ 'degrees_East') newvar[:] = longitude newvar = onc.createVariable('lat_point','f',('lLd')) newattr = basicvardef(newvar,'lat_point','latitude of selection of the values', \ 'degrees_North') newvar[:] = latitude newvar = onc.createVariable('flon_point','f',('lLd')) newattr = basicvardef(newvar,'flon_point','closest longitude of selection of ' + \ 'the values', 'degrees_East') newvar[:] = lon2d[tuple(mappairs)] newvar = onc.createVariable('flat_point','f',('lLd')) newattr = basicvardef(newvar,'flat_point','closest latitude of selection of ' + \ 'the values', 'degrees_North') newvar[:] = lat2d[tuple(mappairs)] newvar = onc.createVariable('fdist_point','f',('lLd')) newattr = basicvardef(newvar,'fdist_point','distance to the point of selection '+\ 'of the values', 'degrees') newvar[:] = mindist onc.sync() onc.close() print fname + ": successful writing of file '" + ofile + "' !!" return #vals='20.5:36.0:west_east,south_north:XLONG_M,XLAT_M' #ncf='/home/lluis/etudes/domains/medic051215_2km/geo_em.d01.nc' def put_variable_slice(inc, onc, vn, sliced): """ Function to add a variable from a netcdf object to a new one following a slice inc= input netcdf object onc= output netcdf object vn= variable name to include sliced= dictioary with the dimension name as key and [beg, end, int] as values """ fname = 'put_variable_slice' if not searchInlist(inc.variables,vn): print errormsg print ' ' + fname + ": variable '" + vn + "' is not in file!!" quit(-1) ov = inc.variables[vn] dnvs = ov.dimensions varslice = [] # Getting variable's dimensions for dnv in dnvs: if not searchInlist(onc.dimensions,dnv): od = inc.dimensions[dnv] if od.isunlimited(): dsize = None else: dsize = len(od) newdim = onc.createDimension(dnv, dsize) if searchInlist(sliced.keys(),dnv): varslice.append(slice(sliced[dnv][0], sliced[dnv][1], sliced[dnv][2])) else: varslice.append(slice(0,len(inc.dimensions[dnv]))) # Getting variable varattrs = ov.ncattrs() if searchInlist(varattrs, '_FillValue'): varfil = ov._FillValue else: varfil = False vartype = ov.dtype newvar = onc.createVariable(vn, vartype, dnvs, fill_value=varfil) # print 'Lluis shapes newvar:',newvar.shape,'slice:',varslice newvar[:] = ov[tuple(varslice)] for attrs in varattrs: if not attrs == '_FillValue': attrv = ov.getncattr(attrs) attr = set_attribute(newvar, attrs, attrv) return def DataSetSection_multivars(values, filen, varn): """ Function to get a section (values along multiple variables) of a given data-set values= [varn1],[beg1],[end1],[int1]@[...[[varnM],[begM],[endM],[intM]]] [varni]: name of the variable ('WRFt', for WRF time varibale) [begi],[endi],[inti]: beginning, end and interval along the dimension-axis [endi] = -1, maximum value [inti] = -1, all the values within the range NOTE: dimensions without section by the variables are taken allover their size filen= netCDF with the data-set varn= ',' list of variables ('all', for all variables) """ import numpy.ma as ma fname = 'DataSetSection_multivars' # Variables not to check in the file NOcheck = ['WRFt'] if values == 'h': print fname + '_____________________________________________________________' print DataSetSection_multivars.__doc__ quit() Nvars = len(values.split('@')) # Slicing variables varns = [] ofiletile = '' slicevalS = '' begvs = np.zeros((Nvars), dtype=int) endvs = np.zeros((Nvars), dtype=int) intvs = np.zeros((Nvars), dtype=int) for ivar in range(Nvars): val = values.split('@')[ivar] varns.append(val.split(',')[0]) begvs[ivar] = np.float(val.split(',')[1]) endvs[ivar] = np.float(val.split(',')[2]) intvs[ivar] = np.float(val.split(',')[3]) ofiletile = ofiletile + '_' + varns[ivar] + '_B' + str(begvs[ivar]) + '-E' + \ str(endvs[ivar]) + '-I' + str(intvs[ivar]) if intvs[ivar] != -1: slicevalS = slicevalS + varns[ivar] + ' (' + str(begvs[ivar]) + ',' + \ str(endvs[ivar]) + ',' + str(intvs[ivar]) + '); ' else: slicevalS = slicevalS + varns[ivar] + ' (' + str(begvs[ivar]) + ',' + \ str(endvs[ivar]) + ',1); ' ofile = ofile=filen.split('.')[0] + ofiletile + '.nc' nciobj = NetCDFFile(filen,'r') ncoobj = NetCDFFile(ofile,'w') # Output variables ## if varn == 'all': variables = nciobj.variables else: if varn.find(',') != -1: variables = varn.split(',') else: variables = [varn] # Looking for limits due to the variables ## for ivar in range(Nvars): print ' ' + fname + ": masking with '" + varns[ivar] + "':", begvs[ivar], \ ',', endvs[ivar], ',', intvs[ivar] if not searchInlist(nciobj.variables,varns[ivar]) and not \ searchInlist(NOcheck,varns[ivar]): print errormsg print ' ' + fname + ": variable '" + varns[ivar] + "' is not in ' + \ 'input file!!" quit(-1) if varns[ivar] == 'WRFt': ovar = nciobj.variables['Times'][:] vals = [] for it in range(ovar.shape[0]): vals.append(datetimeStr_conversion(ovar[it,:], 'WRFdatetime', \ 'YmdHMS')) vals[it] = int(vals[it]) begvs[ivar] = int(begvs[ivar]) endvs[ivar] = int(endvs[ivar]) intvs[ivar] = int(intvs[ivar]) vals = np.array(vals) vardims = ['Time'] else: ovar = nciobj.variables[varns[ivar]] vals = ovar[:] vardims = ovar.dimensions if endvs[ivar] == -1: endvs[ivar] = np.max(vals) maskinf = ma.masked_less(vals, begvs[ivar]) masksup = ma.masked_greater(vals, endvs[ivar]) if intvs[ivar] != -1: print errormsg print ' ' + fname + ': non-consecutive slices not ready!!' quit(-1) finalmask = maskinf.mask + masksup.mask idn = 0 slicedims = {} for dn in vardims: ddn = len(nciobj.dimensions[dn]) dinds = np.arange(ddn) rightvals = np.where(finalmask, False, True) slicedims[dn]=[np.min(dinds[rightvals]), np.max(dinds[rightvals])+1, None] print ' ' + fname + ': slicing variables with______' print slicedims # Creating dimensions ## print ' ' + fname + ': adding dimensions...' for nd in slicedims.keys(): objdim = nciobj.dimensions[nd] if objdim.isunlimited(): dimsize = None else: dimsize = slicedims[nd][1] - slicedims[nd][0] + 1 newdim = ncoobj.createDimension(nd, dimsize) ncoobj.sync() for var in variables: put_variable_slice(nciobj, ncoobj, var, slicedims) # Global attributes ## for attrs in nciobj.ncattrs(): attrv = nciobj.getncattr(attrs) attr = set_attribute(ncoobj, attrs, attrv) attr = set_attribute(ncoobj, 'sliced_variables', slicevalS) nciobj.close() ncoobj.sync() ncoobj.close() print ' ' + fname + ' succesfull creation of file "' + ofile + '" !!!' return #DataSetSection_multivars('XTIME,4380,4800,-1','/home/lluis/PY/wrfout_d01_2001-11-11_00:00:00','all') #DataSetSection_multivars('WRFt,20011111060000,20011111180000,-1','/home/lluis/PY/wrfout_d01_2001-11-11_00:00:00','all') def get_attribute(values, filen, varn): """ Function to get an attribute from a netCDF file values= [attrname] attrname: attribute name filen= name of the netCDF file varn= name of the variable ('global' for a global attribute) """ fname = 'get_attribute' attrn = values if values == 'h': print fname + '_____________________________________________________________' print get_attribute.__doc__ quit() onc = NetCDFFile(filen, 'r') if varn == 'global': if not searchInlist(onc.ncattrs(), attrn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have atribute '" + \ attrn + "' !!" quit(-1) else: attrv = onc.getncattr(attrn) else: if not searchInlist(onc.variables, varn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have variable '" + \ varn + "' !!" quit(-1) else: ovar = onc.variables[varn] if not searchInlist(ovar.ncattrs(), attrn): print errormsg print ' ' + fname + ": variable '" + varn + "' does not have " + \ "atribute '" + attrn + "' !!" print ' it has:',ovar.ncattrs() quit(-1) else: attrv = ovar.getncattr(attrn) onc.close() print attrv return def DimsLoop(ovar,seldims): """ Function to provide the shape of the dimensions which are not selected ova= variable object seldims= list of dimensions to select """ fname = 'DimsLoop' newshape = [] noloopdims = [] idim = 0 for dim in list(ovar.dimensions): if not searchInlist(seldims,dim): newshape.append(ovar.shape[idim]) noloopdims.append(dim) idim = idim + 1 return newshape, noloopdims def SliceVar(ovar,dloop,dloopindex): """ Function to slice a given variable throughout a given list of dimensions ovar= variable object to slice dloop= list of dimensions to get a single value dloopindex= value of the loop dimenion to get """ fname = 'SliceVar' varshape = ovar.shape slicevals = [] idim = 0 for dim in ovar.dimensions: if searchInlist(dloop,dim): slicevals.append(dloopindex[idim]) else: slicevals.append(slice(0,varshape[idim])) idim = idim + 1 return slicevals def varDimension(oncf, dname): """ Function to find the variable with the values of a dimension. It is assumed, that given variable might have a shape with only the dimension, double precision oncf: netCDF object dname: name of the dimension """ fname = 'varDimension' if not oncf.dimensions.has_key(dname): print errormsg print ' ' + fname + ": NetCDF object does not have dimension '" + dname + "' !!" quit(-1) # 1D variables with the dimension name ## d1var = [] for vn in oncf.variables: ovn = oncf.variables[vn] if len(ovn.shape) == 1 and ovn.dimensions[0] == dname: d1var.append(vn) if len(d1var) != 0: print ' ' + fname + ": 1D variables with dimension '" + dname + "':",d1var else: print warnmsg print ' ' + fname + ": Any 1D variables with dimension '" + dname + "' !!" return # Dpouble precision ## if len(d1var) == 0: dimvarname = None if len(d1var) > 1: d1vardouble = [] for vn in d1var: ovn = oncf.variables[vn] if ovn.dtype == type(np.float64(1.)): d1vardouble.append(vn) if len(d1vardouble) > 1: print errormsg print ' ' + fname + ": Found more than 1D double variable with " + \ "dimension '" + dname + "' !!" print ' found:', d1vardouble quit(-1) elif len(d1vardouble) == 0: print warnmsg print ' ' + fname + ": Found any 1D double variable with dimension '" + \ dname + "' !!" dimvarname = None else: dimvarname = d1vardouble[0] else: dimvarname = d1var[0] return dimvarname def ovar_onc(incf, ovar, oncf): """ Function to copy an object variable to a nother netcdf object incf: input netCDF object ovar= variable object oncf= netCDF object """ fname = 'ovar_onc' varname = ovar._name if oncf.variables.has_key(varname): print warnmsg print ' ' + fname + ": netCDF object already has variable '" + ovar.name + \ "' !!" print ' doing noting' return vardims = ovar.dimensions vartype = ovar.dtype for dn in vardims: if not searchInlist(oncf.dimensions, dn): if incf.dimensions[dn].isunlimited(): newdim = oncf.createDimension(dn, None) else: newdim = oncf.createDimension(dn, len(incf.dimensions[dn])) newvar = oncf.createVariable(varname, vartype, vardims) newvar[:] = ovar[:] for attrn in ovar.ncattrs(): attrv = ovar.getncattr(attrn) set_attribute(newvar, attrn, attrv) oncf.sync() return def SpatialWeightedMean(values, filen, varn): """ Function to compute the spatial mean using weights from a netCDF file values= [weightskind],[xdimname],[ydimname],[addvars] [weightskind]: type of weights: * 'varnames',[varname],[oper]: using a variable [varname] with an operation [oper] as area size [oper]: available operations 'inv': inverse of the values 1/[varname] 'nothing': direct values 'prodinv': inverse of the product between 2 variables ([varname1]*[varname2]), NOTE: [varname] = [varname1]:[varname2] * 'reglonlat',[lonname],[latname]: regular lon/lat projection to compute the area size [xdimname]: name of the dimension for the x-axis [ydimname]: name of the dimension for the y-axis [addvars]: ':', separetd list of name of variables to add to the output file filen= name of the netCDF file varn= name of the variable """ fname = 'SpatialWeightedMean' if values == 'h': print fname + '_____________________________________________________________' print SpatialWeightedMean.__doc__ quit() operations = ['inv','nothing','prodinv'] weightk = values.split(',')[0] xdimname = values.split(',')[3] ydimname = values.split(',')[4] addvars = values.split(',')[5] ofile = 'SpatialWeightedMean_' + weightk + '.nc' if weightk == 'varnames': arguments = '[weightk],[varname],[oper],[xdimname],[ydimname],[addvars]' check_arguments(fname, values, arguments, ',') varname = values.split(',')[1] oper = values.split(',')[2] if not searchInlist(operations, oper): print errormsg print ' ' + fname + ": operation '" + oper + "' not ready!!" print ' ready operations:',operations quit(-1) elif weightk == 'reglonlat': arguments = '[weightk],[lonname],[latname],[xdimname],[ydimname],[addvars]' check_arguments(fname, values, arguments, ',') lonname = values.split(',')[1] latname = values.split(',')[2] else: print errormsg print ' ' + fname + ": wieght type '" + weightk + "' not ready!!" quit(-1) # Operations ## onc = NetCDFFile(filen, 'r') if not onc.variables.has_key(varn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have variable '" + \ varn + "' !!" quit(-1) iovar = onc.variables[varn] if not searchInlist(list(iovar.dimensions),xdimname) and not \ searchInlist(list(iovar.dimensions),ydimname): print errormsg print ' ' + fname + ": variable '" + varn + "' does not have dimension '" + \ xdimname + "' neither '" + ydimname + "' !!" quit(-1) # Variable attributes filemiss = False ivarattrs = iovar.ncattrs() if searchInlist(ivarattrs, 'missing_value'): filemissv = iovar.getncattr('missing_value') filemiss = True print ' ' + fname + ' input data with missing value:',filemissv,'!!' filefill = False if searchInlist(ivarattrs, '_FillValue'): filefillv = iovar.getncattr('_FillValue') filefill = True print ' ' + fname + ' input data with fill value:',filefillv,'!!' # Getting shape of output variables for future average loop ## loopshape, dimsloop = DimsLoop(iovar,[xdimname, ydimname]) if len(loopshape) > 3: print erromsg print ' ' + fname + ': output size not ready!!' quit(-1) ivarv = iovar[:] if weightk == 'varnames': if oper == 'inv': if not onc.variables.has_key(varname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varname + "' !!" quit(-1) iovarwgt = onc.variables[varname] ivarwgtv = iovarwgt[:] elif oper == 'nothing': if not onc.variables.has_key(varname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varname + "' !!" quit(-1) iovarwgt = onc.variables[varname] ivarwgtv = iovarwgt[:] elif oper == 'prodinv': varname1 = varname.split(':')[0] varname2 = varname.split(':')[1] if not onc.variables.has_key(varname1): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varname1 + "' !!" quit(-1) if not onc.variables.has_key(varname2): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varname2 + "' !!" quit(-1) iovarwgt1 = onc.variables[varname1] iovarwgt2 = onc.variables[varname2] ivarwgtv = iovarwgt1[:]*iovarwgt2[:] if oper == 'inv': longvarname = 'using inverse of variable ' + varname + ' as space weights' if len(loopshape) == 1: newvals = np.ones((loopshape[0]), dtype=np.float)*fillValueF for id1 in range(loopshape[0]): slicevalues = SliceVar(iovar,dimsloop,[id1]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1]) TOTsumwgt = np.sum(1./ivarwgtv[tuple(slicewgt)]) newvals[id1] = np.sum(ivarv[tuple(slicevalues)] / \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt elif len(loopshape) == 2: newvals = np.ones((loopshape[0],loopshape[1]),dtype=np.float)* \ fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1,id2]) TOTsumwgt = np.sum(1./ivarwgtv[tuple(slicewgt)]) newvals[id1,id2] = np.sum(ivarv[tuple(slicevalues)] / \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt elif len(loopshape) == 3: newvals = np.ones((loopshape[0],loopshape[1],loopshape[2]), \ dtype=np.float)*fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): for id3 in range(loopshape[1]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2,id3]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1,id2,id3]) TOTsumwgt = np.sum(1./ivarwgtv[tuple(slicewgt)]) newvals[id1,id2,id3]= np.sum(ivarv[tuple(slicevalues)]/ \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt outweightvals = 1./ivarwgtv[tuple(slicewgt)] elif oper == 'nothing': longvarname = 'using variable ' + varname + ' as space weights' if len(loopshape) == 1: newvals = np.ones((loopshape[0]), dtype=np.float)*fillValueF for id1 in range(loopshape[0]): slicevalues = SliceVar(iovar,dimsloop,[id1]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1]) TOTsumwgt = np.sum(ivarwgtv[tuple(slicewgt)]) newvals[id1] = np.sum(ivarv[tuple(slicevalues)]* \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt elif len(loopshape) == 2: newvals = np.ones((loopshape[0],loopshape[1]), dtype=np.float)* \ fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1,id2]) TOTsumwgt = np.sum(ivarwgtv[tuple(slicewgt)]) newvals[id1,id2] = np.sum(ivarv[tuple(slicevalues)]* \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt elif len(loopshape) == 3: newvals = np.ones((loopshape[0],loopshape[1],loopshape[2]), \ dtype=np.float)*fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): for id3 in range(loopshape[2]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2.id3]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1,id2,id3]) TOTsumwgt = np.sum(ivarwgtv[tuple(slicewgt)]) newvals[id1,id2,id3]=np.sum(ivarv[tuple(slicevalues)]* \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt outweightvals = ivarwgtv[tuple(slicewgt)] if oper == 'prodinv': longvarname = 'using inverse of product of variables ' + varname1 + '*' \ + varname2 + ' as space weights' if len(loopshape) == 1: newvals = np.ones((loopshape[0]), dtype=np.float)*fillValueF for id1 in range(loopshape[0]): slicevalues = SliceVar(iovar,dimsloop,[id1]) slicewgt = SliceVar(iovarwgt1,dimsloop,[id1]) TOTsumwgt = np.sum(1./ivarwgtv[tuple(slicewgt)]) newvals[id1] = np.sum(ivarv[tuple(slicevalues)] / \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt elif len(loopshape) == 2: newvals = np.ones((loopshape[0],loopshape[1]),dtype=np.float)* \ fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1,id2]) TOTsumwgt = np.sum(1./ivarwgtv[tuple(slicewgt)]) newvals[id1,id2] = np.sum(ivarv[tuple(slicevalues)] / \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt elif len(loopshape) == 3: newvals = np.ones((loopshape[0],loopshape[1],loopshape[2]), \ dtype=np.float)*fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): for id3 in range(loopshape[1]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2,id3]) slicewgt = SliceVar(iovarwgt,dimsloop,[id1,id2,id3]) TOTsumwgt = np.sum(1./ivarwgtv[tuple(slicewgt)]) newvals[id1,id2,id3]= np.sum(ivarv[tuple(slicevalues)]/ \ ivarwgtv[tuple(slicewgt)]) / TOTsumwgt outweightvals = 1./ivarwgtv[tuple(slicewgt)] elif weightk == 'reglonlat': longvarname = 'using cos(latitude) variable ' + latname + ' as space weights' if not onc.variables.has_key(lonname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have variable '" + \ lonname + "' !!" quit(-1) if not onc.variables.has_key(latname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have variable '" + \ latname + "' !!" quit(-1) iolon = onc.variables[lonname] iolat = onc.variables[latname] ilonv = iolon[:] ilatv = iolat[:] lonv, latv = lonlat2D(ilonv, ilatv) ivarwgtv = np.abs(np.cos(latv*np.pi/180.)) TOTsumwgt = np.sum(ivarwgtv) if len(loopshape) == 1: newvals = np.ones((loopshape[0]), dtype=np.float)*fillValueF newsumvals = np.ones((loopshape[0]), dtype=np.float)*fillValueF for id1 in range(loopshape[0]): slicevalues = SliceVar(iovar,dimsloop,[id1]) vals = ivarv[tuple(slicevalues)] if filemiss: vals = np.where(vals == filemissv, 0., vals) if filefill: vals = np.where(vals == filefillv, 0., vals) newsumvals[id1] = np.sum(vals*ivarwgtv) newvals[id1] = newsumvals[id1]/TOTsumwgt elif len(loopshape) == 2: newvals = np.ones((loopshape[0],loopshape[1]), dtype=np.float)*fillValueF newsumvals = np.ones((loopshape[0],loopshape[1]), dtype=np.float)*fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2]) vals = ivarv[tuple(slicevalues)] if filemiss: vals = np.where(vals == filemissv, 0., vals) if filefill: vals = np.where(vals == filefillv, 0., vals) newsumvals[id1,id2] = np.sum(vals*ivarwgtv) newvals[id1,id2] = newsumvals[id1,id2] / TOTsumwgt elif len(loopshape) == 3: newvals = np.ones((loopshape[0],loopshape[1],loopshape[2]), \ dtype=np.float)*fillValueF newsumvals = np.ones((loopshape[0],loopshape[1],loopshape[2]), \ dtype=np.float)*fillValueF for id1 in range(loopshape[0]): for id2 in range(loopshape[1]): for id3 in range(loopshape[2]): slicevalues = SliceVar(iovar,dimsloop,[id1,id2,id3]) vals = ivarv[tuple(slicevalues)] if filemiss: vals = np.where(vals == filemissv, 0., vals) if filefill: vals = np.where(vals == filefillv, 0., vals) newsumvals[id1,id2,id3] = np.sum(vals*ivarwgtv) newvals[id1,id2,id3] = newsumvals[id1,id2,id3] / TOTsumwgt outweightvals = ivarwgtv # Writting output file ## onewnc = NetCDFFile(ofile, 'w') # Dimensions creation ## for dim in dimsloop: newdim = onewnc.createDimension(dim, len(onc.dimensions[dim])) vardim = varDimension(onc, dim) if vardim is not None: ovar_onc(onc, onc.variables[vardim], onewnc) # Output variable ## newvar = onewnc.createVariable(varn + 'spaceweightsum', 'f4', tuple(dimsloop), \ fill_value=fillValueF) basicvardef(newvar, varn + 'spaceweightsum', 'space summed ' + varn + ' ' + \ longvarname, iovar.getncattr('units')) newvar[:] = newsumvals onewnc.sync() newvar = onewnc.createVariable(varn + 'spaceweightmean', 'f4', tuple(dimsloop), \ fill_value=fillValueF) basicvardef(newvar, varn + 'spaceweightmean', 'space weighted ' + varn + ' ' + \ longvarname, iovar.getncattr('units')) newvar[:] = newvals onewnc.sync() print 'newsumvals:',newsumvals,'newvals:',newvals,'TOTsumwgt:',TOTsumwgt # Spatial weight ## newdim = onewnc.createDimension(ydimname, outweightvals.shape[0]) newdim = onewnc.createDimension(xdimname, outweightvals.shape[1]) newvar = onewnc.createVariable('spatialweight', 'f4',tuple([ydimname, xdimname]),\ fill_value=fillValueF) basicvardef(newvar, 'spatialweight', 'space weight ' + longvarname, '-') newvar[:] = outweightvals onewnc.sync() # Additional variables ## for vn in addvars.split(':'): if not onc.variables.has_key(vn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have variable '" + \ vn + "' !!" quit(-1) ovar_onc(onc, onc.variables[vn], onewnc) onewnc.sync() # Global attributes ## onewnc.setncattr('script', fname) onewnc.setncattr('version', '1.0') onewnc.setncattr('author', 'L. Fita') newattr = set_attributek(onewnc, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') onewnc.setncattr('university', 'Pierre et Marie Curie') onewnc.setncattr('country', 'France') for attrs in onc.ncattrs(): attrv = onc.getncattr(attrs) attr = set_attribute(onewnc, attrs, attrv) onc.close() onewnc.sync() onewnc.close() print fname + ": Successfull written of file: '" + ofile + "' !!" return #SpatialWeightedMean('varnames,MAPFAC_M,inv,west_east,south_north','wrfout_d01_2001-11-11_00:00:00','HGT') #SpatialWeightedMean('reglonlat,XLONG,XLAT,west_east,south_north','wrfout_d01_2001-11-11_00:00:00','HGT') def fillvalue_kind(vartype, fillval0): """ Function to provide the fiven fill_Value for a kind of variable [vartype]= type of the variable [fillval0]= value to take as fill value, 'std' for the standard value Float = 1.e20 Character = '-' Integer = -99999 Integer16 = -99999 Float64 = 1.e20 Integer32 = -99999 """ fname = 'fillvalue_kind' if vartype == type('c'): if fillval0 == 'std': fillval = fillvalueC else: fillval = fillval0 elif vartype == type(int(1)): if fillval0 == 'std': fillval = fillValueI else: fillval = int(fillval0) elif vartype == type(np.int16(1)): if fillval0 == 'std': fillval = np.int(9999999) else: fillval = np.int16(fillval0) elif vartype == type(np.int32(1)): if fillval0 == 'std': fillval = np.int32(fillValueI) else: fillval = np.int32(fillval0) elif vartype == type(np.float(1.)): if fillval0 == 'std': fillval = np.float(fillValueF) else: fillval = np.float(fillval0) elif vartype == type(np.float32(1.)): if fillval0 == 'std': fillval = np.float32(fillValueF) else: fillval = np.float32(fillval0) elif vartype == type(np.float64(1.)): if fillval0 == 'std': fillval = np.float64(fillValueF) else: fillval = np.float64(fillval0) else: print errormsg print ' ' + fname + ': variable type', vartype, 'not ready !!' quit(-1) return fillval def nctype(vartype): """ Function to provide the string for the variable creation in a netCDF file [vartype]= type of the variable """ fname = 'nctype' if vartype == type('c'): ncs = 'c' elif vartype == '|S1': ncs = 'c' elif vartype == type(int(1)): ncs = 'i' elif vartype == type(np.int16(1)): ncs = 'i' elif vartype == type(np.int32(1)): ncs = 'i8' elif vartype == type(np.float(1.)): ncs = 'f' elif vartype == type(np.float32(1.)): ncs = 'f4' elif vartype == type(np.float64(1.)): ncs = 'f8' else: print errormsg print ' ' + fname + ': variable type', vartype, 'not ready !!' quit(-1) return ncs def varval_ind(var, ind): """ Function to provide a value from a variable providing a value for each index var= variable ind= list of indices for each dimension of the variable """ fname = 'varval_ind' if len(var.shape) != len(ind): print errormsg print ' ' + fname + ': different shape:', var.shape,'and indices:', ind, '!!' quit(-1) varslice=[] for iv in ind: varslice.append(iv) val = varval_ind[slice(varslice)] return val def VarVal_FillValue(values, filen, varn): """ Function to transform a given value from a given variable to _FillValue in a netCDF file values= [value],[fillVal] [value]: value to pass to '_FillValue' [fillVal]: value for '_FillValue', 'std' for the standard value Float = 1.e20 Character = '-' Integer = -99999 Float64 = np.float(Float) Integer32 = np.int32(Integer) filen= name of the netCDF file varn= name of the variable """ fname = 'VarVal_FillValue' if values == 'h': print fname + '_____________________________________________________________' print VarVal_FillValue.__doc__ quit() onc = NetCDFFile(filen, 'a') if not onc.variables.has_key(varn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varn + "' !!" quit(-1) value = values.split(',')[0] fval0 = values.split(',')[1] varo = onc.variables[varn] varvals = varo[:] vartype = varo.dtype if vartype == '|S': valchk = value elif vartype == type(int(1)): valchk = int(value) elif vartype == type(np.int32(1)): valchk = np.int32(value) elif vartype == type(np.float(1.)): valchk = np.float(value) elif vartype == type(np.float32(1.)): valchk = np.float32(value) elif vartype == type(np.float64(1.)): valchk = np.float64(value) else: print errormsg print ' ' + fname + ': variable type', vartype, 'not ready !!' quit(-1) fval = fill_value(vartype, fval0) newvarvals = np.where(varvals==valchk, fval, varvals) if searchInlist(varo.ncattrs(), 'missing_value'): missval = varo.getncattr('missing_value') difvals = fval*10.**(-np.log10(valchk)) - missval*10.**(-np.log10(missval)) if missval == valchk or np.abs(difvals) < 1.e-6: print warnmsg print ' ' + fname + ': same missing and checking Value!' print ' renaming missing_value' varo.delncattr('missing_value') set_attribute(varo, '_FillValue', fval) else: set_attribute(varo, '_FillValue', fval) else: set_attribute(varo, '_FillValue', fval) varo[:] = newvarvals onc.sync() onc.close() print fname +': Successfull change to _FillValue !!' return #VarVal_FillValue('0.,std', '/home/lluis/etudes/DYNAMICO/ORCHIDEE/interpolation/data/new_lai2D.nc', 'LAI') def lonlatProj(dxres, dyres, project, wfile): """ Function to compute longitudes and latitudes for a given projection following subroutine calc_resolution_in from ORCHIDEE/src_global/module_InterpWeight dxres= resolution on the WE axis [m] dyres= resolution on the SN axis [m] project= projection 'lonlat': standard lon/lat projection tacking number of points from [dx/dy]res at the Equator 'lonlat_dxdyFix': projection with a fixed grid distance wfile= Should an output file be written (True/False) """ fname = 'lonlatProj' ofile = 'lonlatProj.nc' # Given constants EarthR = 6378000. RadDeg = 180./np.pi Npts = 1 if project == 'lonlat': Nlon = int(2. * np.pi * EarthR / dxres) + 1 Nlat = int(np.pi * EarthR / dyres) + 1 if np.mod(Nlon,2) == 0: Nlon = Nlon + 1 if np.mod(Nlat,2) == 0: Nlat = Nlat + 1 lonProj = np.zeros((Nlon), dtype=np.float) latProj = np.zeros((Nlat), dtype=np.float) lonProj = -180. + np.arange(Nlon) * 2. * np.pi * RadDeg / (Nlon - 1) latProj = -90. + np.arange(Nlat) * np.pi * RadDeg / (Nlat - 1) dimx = Nlon dimy = Nlat Npts = dimx*dimy Npoints = np.arange((Npts)).reshape(dimy,dimx) elif project == 'lonlat_dxdyFix': lonProj = [] latProj = [] lonProjcirc = [] latProjcirc = [] Nlonmaxcirc = -10. lonProj.append(0.) latProj.append(-90.) lonProjcirc.append(0.) latProjcirc.append(-90.) # Differential on latitudes Nlat = int(np.pi * EarthR / dyres) + 3 latarc = np.pi / (Nlat - 1) Nloncirc = np.zeros((Nlat), dtype=int) Nloncirc[0] = 1 lat=-np.pi/2. ilat = 1 while lat + latarc <= np.pi/2.: lon=0. lat = lat + latarc lonarc = dxres / (EarthR * np.cos(lat)) lonscirc = [] latProjcirc.append(lat*RadDeg) while lon <= 2.*np.pi: # print lonarc, latarc, lon, lat, [lon*RadDeg, lat*RadDeg] lonProj.append(-180. + lon*RadDeg) latProj.append(lat*RadDeg) lonscirc.append(-180. + lon*RadDeg) lon = lon + lonarc Npts = Npts + 1 lonProjcirc.append(lonscirc) Nloncirc[ilat] = len(lonscirc) if Nloncirc[ilat] > Nlonmaxcirc: Nlonmaxcirc = Nloncirc[ilat] ilat = ilat + 1 lonProj.append(0.) latProj.append(90.) lonProjcirc.append(0.) latProjcirc.append(90.) Nloncirc[ilat] = 1 Npts = Npts + 1 dimx = Npts dimy = Npts Npoints = np.arange(Npts) else: print errormsg print ' ' + fnamne + ": projection '" + project + "' not ready !!" quit(-1) if ofile: # netCDF creation onewnc = NetCDFFile(ofile, 'w') # Dimensions newdim = onewnc.createDimension('x', dimx) newdim = onewnc.createDimension('y', dimy) newdim = onewnc.createDimension('Npts', Npts) if project == 'lonlat_dxdyFix': newdim = onewnc.createDimension('xcirc', Nlonmaxcirc) newdim = onewnc.createDimension('ycirc', Nlat) # Variables newvar = onewnc.createVariable('lon', 'f8', ('x')) basicvardef(newvar, 'longitude', 'Longitude', 'degrees_East') newvar[:] = lonProj newvar.setncattr('axis', 'X') newvar.setncattr('_CoordinateAxisType', 'Lon') newvar = onewnc.createVariable('lat', 'f8', ('y')) basicvardef(newvar, 'latitude', 'Latitude', 'degrees_North') newvar[:] = latProj newvar.setncattr('axis', 'Y') newvar.setncattr('_CoordinateAxisType', 'Lat') # Specific variables if project == 'lonlat_dxdyFix': newvar = onewnc.createVariable('Nloncirc', 'i', ('ycirc')) newvar[:] = Nloncirc basicvardef(newvar, 'Nloncirc', 'Number of longitudes by latitude circs', '-') newvar = onewnc.createVariable('loncirc', 'f8', ('ycirc','xcirc')) basicvardef(newvar, 'longitude', 'Longitude by latitude circs', 'degrees_East') for iL in range(Nlat): newvar[iL,0:Nloncirc[iL]] = np.asarray(lonProjcirc[iL]) newvar = onewnc.createVariable('latcirc', 'f8', ('ycirc')) basicvardef(newvar, 'latitude', 'Latitude', 'degrees_North') newvar[:] = latProjcirc if len(Npoints.shape) == 2: newvar = onewnc.createVariable('points', 'i8', ('y', 'x')) else: newvar = onewnc.createVariable('points', 'i8', ('Npts')) basicvardef(newvar, 'points', '# point of the projection', '-') newvar[:] = Npoints newvar.setncattr('coordinates', 'lon lat') # Global attributes onewnc.setncattr('script', fname) onewnc.setncattr('version', '1.0') onewnc.setncattr('author', 'L. Fita') newattr = set_attributek(onewnc, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') onewnc.setncattr('university', 'Pierre et Marie Curie') onewnc.setncattr('country', 'France') onewnc.setncattr('Projection','project') onewnc.sync() onewnc.close() print fname + "Successfull written of '" + ofile + "' !!'" if project == 'lonlat': return lonProj, latProj elif project == 'lonlat_dxdyFix': return lonProjcirc, latProj #lonlatProj(100000.,100000.,'lonlat',True) #quit() def CoarselonlatFind(ilon, ilat, lonv, latv, per): """ Function to search a given value from a coarser version of the data ilon, ilat= original 2D matrices with the longitudes and the latitudes lonv, latv= longitude and latitude to find per= period (as fraction over 1) of the fractions of the original grid to use to explore >>> npt=100 >>> lonval0 = np.arange(0.,npt*1.,1.) >>> latval0 = np.arange(0.,npt*1.,1.) >>> lonval = np.zeros((npt,npt), dtype=np.float) >>> latval = np.zeros((npt,npt), dtype=np.float) >>> for i in range(npt): >>> lonval[:,i] = lonval0[i] >>> latval[i,:] = latval0[i] >>> CoarselonlatFind(lonval, latval, 5.25, 5.25, .1) (array([5, 5]), 0.35355339059327379) """ fname = 'CoarselonlatFind' dx = ilon.shape[1] dy = ilon.shape[0] nlon = np.min(ilon) xlon = np.max(ilon) nlat = np.min(ilat) xlat = np.max(ilat) if lonv < nlon or lonv > xlon: print errormsg print ' ' + fname + ': longitude outside data range!!' print ' given value:', lonv,'outside [',nlon,',',xlon,']' quit(-1) if latv < nlat or latv > xlat: print errormsg print ' ' + fname + ': latitude outside data range!!' print ' given value:', latv,'outside [',nlat,',',xlat,']' quit(-1) fracx = int(dx*per) fracy = int(dy*per) fraclon = ilon[::fracy,::fracx] fraclat = ilat[::fracy,::fracx] fracdx = fraclon.shape[1] fracdy = fraclon.shape[0] # print 'fraclon _______' # print fraclon # print 'fraclat _______' # print fraclat # Fraction point difffraclonlat = np.sqrt((fraclon-lonv)**2. + (fraclat-latv)**2.) mindifffracLl = np.min(difffraclonlat) ilatlonfrac = index_mat(difffraclonlat, mindifffracLl) # print 'mindifffracLl:', mindifffracLl, 'ilatlonfrac:', ilatlonfrac # print 'frac lon, lat:', fraclon[ilatlonfrac[0],ilatlonfrac[1]], ',', \ # fraclat[ilatlonfrac[0],ilatlonfrac[1]] # print 'values lon, lat:', lonv, latv # Providing fraction range if fraclon[ilatlonfrac[0],ilatlonfrac[1]] >= lonv and \ fraclat[ilatlonfrac[0],ilatlonfrac[1]] >= latv: # ifracbeg = [ilatlonfrac[0]-1, ilatlonfrac[1]-1] # ifracend = [ilatlonfrac[0], ilatlonfrac[1]] if ilatlonfrac[0] > 0: iybeg = (ilatlonfrac[0]-1)*fracy iyend = ilatlonfrac[0]*fracy+1 else: iybeg = 0 iyend = fracy+1 if ilatlonfrac[1] > 0: ixbeg = (ilatlonfrac[1]-1)*fracx ixend = ilatlonfrac[1]*fracx+1 else: ixbeg = 0 ixend = fracx+1 elif fraclon[ilatlonfrac[0],ilatlonfrac[1]] < lonv and \ fraclat[ilatlonfrac[0],ilatlonfrac[1]] >= latv: # ifracbeg = [ilatlonfrac[0]-1, ilatlonfrac[1]] # ifracend = [ilatlonfrac[0], ilatlonfrac[1]+1] if ilatlonfrac[0] > 0: iybeg = (ilatlonfrac[0]-1)*fracy iyend = ilatlonfrac[0]*fracy+1 else: iybeg = 0 iyend = fracy+1 if ilatlonfrac[1] < fracdx: if ilatlonfrac[1] != 0: ixbeg = (ilatlonfrac[1]-1)*fracx ixend = ilatlonfrac[1]*fracx+1 else: ixbeg = 0 ixend = fracx+1 else: ixbeg = fracdx*fracx ixend = dx+1 elif fraclon[ilatlonfrac[0],ilatlonfrac[1]] < lonv and \ fraclat[ilatlonfrac[0],ilatlonfrac[1]] < latv: # ifracbeg = [ilatlonfrac[0], ilatlonfrac[1]] # ifracend = [ilatlonfrac[0]+1, ilatlonfrac[1]+1] if ilatlonfrac[0] < fracdy: if ilatlonfrac[0] != 0: iybeg = (ilatlonfrac[0]-1)*fracy iyend = ilatlonfrac[0]*fracy+1 else: iybeg = 0 iyend = fracy+1 else: iybeg = fracdy*fracy iyend = dy+1 if ilatlonfrac[1] < fracdx: if ilatlonfrac[1] != 0: ixbeg = (ilatlonfrac[1]-1)*fracx ixend = ilatlonfrac[1]*fracx+1 else: ixbeg = 0 ixend = fracx+1 else: ixbeg = fracdx*fracx ixend = dx+1 elif fraclon[ilatlonfrac[0],ilatlonfrac[1]] >= lonv and \ fraclat[ilatlonfrac[0],ilatlonfrac[1]] < latv: # ifracbeg = [ilatlonfrac[0], ilatlonfrac[1]-1] # ifracend = [ilatlonfrac[0]+1, ilatlonfrac[1]] if ilatlonfrac[0] < fracdy: if ilatlonfrac[0] != 0: iybeg = (ilatlonfrac[0]-1)*fracy iyend = ilatlonfrac[0]*fracy+1 else: iybeg = 0 iyend = fracy+1 else: iybeg = fracdy*fracy iyend = dy+1 if ilatlonfrac[1] > 0: ixbeg = (ilatlonfrac[1]-1)*fracx ixend = ilatlonfrac[1]*fracx+1 else: ixbeg = 0 ixend = fracx+1 # ibeg = [iybeg, ixbeg] # iend = [iyend, ixend] # print 'find window; beginning', ibeg, 'end:', iend lon = ilon[iybeg:iyend,ixbeg:ixend] lat = ilat[iybeg:iyend,ixbeg:ixend] # print 'lon _______' # print lon # print 'lat _______' # print lat # Find point difflonlat = np.sqrt((lon-lonv)**2. + (lat-latv)**2.) mindiffLl = np.min(difflonlat) ilatlon = index_mat(difflonlat, mindiffLl) ilatlon[0] = ilatlon[0] + iybeg ilatlon[1] = ilatlon[1] + ixbeg # print 'mindiffLl:', mindiffLl, 'ilatlon:', ilatlon # print 'lon, lat:', lon[ilatlon[0],ilatlon[1]], ',', lat[ilatlon[0],ilatlon[1]] # quit() return ilatlon, mindiffLl def CoarselonlatFindAll(onc, ilon, ilat, longvs, latvs, per, frac, out): """ Function to search all values from a coarser version of the data onc= netCDF object from an existing file ilon, ilat= original 2D matrices with the longitudes and the latitudes lonv, latv= longitude and latitude to find per= period (as fraction over 1) of the fractions of the original grid to use to explore frac= frequency of points at which syncronize file with calculations out= True/False if outcome is required >>> npt=100 >>> lonval0 = np.arange(0.,npt*1.,1.) >>> latval0 = np.arange(0.,npt*1.,1.) >>> lonval = np.zeros((npt,npt), dtype=np.float) >>> latval = np.zeros((npt,npt), dtype=np.float) >>> lonvalues = np.arange(0.25,10.,0.25) >>> latvalues = np.arange(0.25,10.,0.25) >>> for i in range(npt): >>> lonval[:,i] = lonval0[i] >>> latval[i,:] = latval0[i] >>> CoarselonlatFindAll(onewnc, lonval, latval, lovalues, latvalues, .1, 100) (array([5, 5]), 0.35355339059327379) """ fname = 'CoarselonlatFindAll' dx = ilon.shape[1] dy = ilon.shape[0] nlon = np.min(ilon) xlon = np.max(ilon) nlat = np.min(ilat) xlat = np.max(ilat) Nallpts = len(longvs) fracx = int(dx*per) fracy = int(dy*per) fraclon = ilon[::fracy,::fracx] fraclat = ilat[::fracy,::fracx] fracdx = fraclon.shape[1] fracdy = fraclon.shape[0] # print 'fraclon _______' # print fraclon # print 'fraclat _______' # print fraclat if out: ilatlon = np.zeros((2,Nallpts), dtype=int) mindiffLl = np.zeros((Nallpts), dtype=np.float) iptpos=0 for iv in range(Nallpts): lonv = longvs[iv] latv = latvs[iv] if lonv < nlon or lonv > xlon: print errormsg print ' ' + fname + ': longitude outside data range!!' print ' given value:', lonv,'outside [',nlon,',',xlon,']' quit(-1) if latv < nlat or latv > xlat: print errormsg print ' ' + fname + ': latitude outside data range!!' print ' given value:', latv,'outside [',nlat,',',xlat,']' quit(-1) # Fraction point difffraclonlat = np.sqrt((fraclon-lonv)**2. + (fraclat-latv)**2.) mindifffracLl = np.min(difffraclonlat) ilatlonfrac = index_mat(difffraclonlat, mindifffracLl) # print 'values lon, lat:', lonv, latv # print 'mindifffracLl:', mindifffracLl, 'ilatlonfrac:', ilatlonfrac # print 'frac lon, lat:', fraclon[ilatlonfrac[0],ilatlonfrac[1]], ',', \ # fraclat[ilatlonfrac[0],ilatlonfrac[1]] # Providing fraction range if fraclon[ilatlonfrac[0],ilatlonfrac[1]] >= lonv and \ fraclat[ilatlonfrac[0],ilatlonfrac[1]] >= latv: # ifracbeg = [ilatlonfrac[0]-1, ilatlonfrac[1]-1] # ifracend = [ilatlonfrac[0], ilatlonfrac[1]] if ilatlonfrac[0] > 0: iybeg = (ilatlonfrac[0]-1)*fracy iyend = ilatlonfrac[0]*fracy+1 else: iybeg = 0 iyend = fracy+1 if ilatlonfrac[1] > 0: ixbeg = (ilatlonfrac[1]-1)*fracx ixend = ilatlonfrac[1]*fracx+1 else: ixbeg = 0 ixend = fracx+1 elif fraclon[ilatlonfrac[0],ilatlonfrac[1]] < lonv and \ fraclat[ilatlonfrac[0],ilatlonfrac[1]] >= latv: # ifracbeg = [ilatlonfrac[0]-1, ilatlonfrac[1]] # ifracend = [ilatlonfrac[0], ilatlonfrac[1]+1] if ilatlonfrac[0] > 0: iybeg = (ilatlonfrac[0]-1)*fracy iyend = ilatlonfrac[0]*fracy+1 else: iybeg = 0 iyend = fracy+1 if ilatlonfrac[1] < fracdx: if ilatlonfrac[1] != 0: ixbeg = (ilatlonfrac[1]-1)*fracx ixend = ilatlonfrac[1]*fracx+1 else: ixbeg = 0 ixend = fracx+1 else: ixbeg = fracdx*fracx ixend = dx+1 elif fraclon[ilatlonfrac[0],ilatlonfrac[1]] < lonv and \ fraclat[ilatlonfrac[0],ilatlonfrac[1]] < latv: # ifracbeg = [ilatlonfrac[0], ilatlonfrac[1]] # ifracend = [ilatlonfrac[0]+1, ilatlonfrac[1]+1] if ilatlonfrac[0] < fracdy: if ilatlonfrac[0] != 0: iybeg = (ilatlonfrac[0]-1)*fracy iyend = ilatlonfrac[0]*fracy+1 else: iybeg = 0 iyend = fracy+1 else: iybeg = fracdy*fracy iyend = dy+1 if ilatlonfrac[1] < fracdx: if ilatlonfrac[1] != 0: ixbeg = (ilatlonfrac[1]-1)*fracx ixend = ilatlonfrac[1]*fracx+1 else: ixbeg = 0 ixend = fracx+1 else: ixbeg = fracdx*fracx ixend = dx+1 elif fraclon[ilatlonfrac[0],ilatlonfrac[1]] >= lonv and \ fraclat[ilatlonfrac[0],ilatlonfrac[1]] < latv: # ifracbeg = [ilatlonfrac[0], ilatlonfrac[1]-1] # ifracend = [ilatlonfrac[0]+1, ilatlonfrac[1]] if ilatlonfrac[0] < fracdy: if ilatlonfrac[0] != 0: iybeg = (ilatlonfrac[0]-1)*fracy iyend = ilatlonfrac[0]*fracy+1 else: iybeg = 0 iyend = fracy+1 else: iybeg = fracdy*fracy iyend = dy+1 if ilatlonfrac[1] > 0: ixbeg = (ilatlonfrac[1]-1)*fracx ixend = ilatlonfrac[1]*fracx+1 else: ixbeg = 0 ixend = fracx+1 # ibeg = [iybeg, ixbeg] # iend = [iyend, ixend] # print 'find window; beginning', ibeg, 'end:', iend lon = ilon[iybeg:iyend,ixbeg:ixend] lat = ilat[iybeg:iyend,ixbeg:ixend] # print 'Looking within _______' # print 'lon:',lon[0,0],',',lon[-1,-1] # print 'lat:',lat[0,0],',',lat[-1,-1] # print 'lon _______' # print lon # print 'lat _______' # print lat # Find point difflonlat = np.sqrt((lon-lonv)**2. + (lat-latv)**2.) if out: mindiffLl[iv] = np.min(difflonlat) ilatlon[:,iv] = index_mat(difflonlat, mindiffLl) ilatlon[0,iv] = ilatlon[0,iv] + iybeg ilatlon[1,iv] = ilatlon[1,iv] + ixbeg else: mindiffLl = np.min(difflonlat) ilatlon = index_mat(difflonlat, mindiffLl) ilatlon[0] = ilatlon[0] + iybeg ilatlon[1] = ilatlon[1] + ixbeg # print 'mindiffLl:', mindiffLl, 'ilatlon:', ilatlon # print 'lon, lat:', lon[ilatlon[0],ilatlon[1]], ',', lat[ilatlon[0],ilatlon[1]] # quit() # if mindiffLl == 0. and type(newvar[ilatlon[0],ilatlon[1]]) == type(amsk): # percendone(iv,Ninpts,0.5,'done:') if mindiffLl == 0.: iptpos = iptpos + 1 # Speeding it up! # if mindiffLl > mindiff: # print errormsg # print ' ' + fname + ': for point #', iv,'lon,lat in ' + \ # 'incomplet map:', lonvs[iv], ',', latvs[iv], 'there is ' + \ # 'not a set of lon,lat in the completed map closer than: ', \ # mindiff, '!!' # print ' minimum difference:', mindiffLl # quit(-1) # Speeding it up! # if ilatlon[0] >= 0 and ilatlon[1] >= 0: # newvar[ilatlon[0],ilatlon[1]] = ovar[iv] # newvarin[ilatlon[0],ilatlon[1]] = iv # newvarinpt[iv] = 1 # newvarindiff[iv] = mindiffLl newvar[ilatlon[0],ilatlon[1]] = ovar[iv] newvarin[ilatlon[0],ilatlon[1]] = iv newvarinpt[iv] = 1 newvarindiff[iv] = mindiffLl # print 'Lluis iv:', newvarin[ilatlon[0],ilatlon[1]], \ # 'localized:', newvarinpt[iv], 'values:', \ # newvar[ilatlon[0],ilatlon[1]], 'invalues:', ovar[iv], \ # 'mindist:', newvarindiff[iv], 'point:',ilatlon # else: # print errormsg # print ' ' + fname + ': point iv:', iv, 'at', lonvs[iv], ',', \ # latvs[iv],' not relocated !!' # print ' mindiffl:', mindiffLl, 'ilon:', ilatlon[1], \ # 'ilatlon:', ilatlon[1] # quit(-1) if np.mod(iptpos,fracs) == 0: print 'Lluis syncronizing!' onc.sync() if out: return ilatlon, mindiffLl else: return def lonlatFind(ilon, ilat, lonv, latv): """ Function to search a given value from a lon, lat 2D data ilon, ilat= original 2D matrices with the longitudes and the latitudes lonv, latv= longitude and latitude to find >>> npt=100 >>> lonval0 = np.arange(0.,npt*1.,1.) >>> latval0 = np.arange(0.,npt*1.,1.) >>> lonval = np.zeros((npt,npt), dtype=np.float) >>> latval = np.zeros((npt,npt), dtype=np.float) >>> for i in range(npt): >>> lonval[:,i] = lonval0[i] >>> latval[i,:] = latval0[i] >>> lonlatFind(lonval, latval, 5.25, 5.25) (array([5, 5]), 0.35355339059327379) """ fname = 'lonlatFind' dx = ilon.shape[1] dy = ilon.shape[0] nlon = np.min(ilon) xlon = np.max(ilon) nlat = np.min(ilat) xlat = np.max(ilat) if lonv < nlon or lonv > xlon: print errormsg print ' ' + fname + ': longitude outside data range!!' print ' given value:', lonv,'outside [',nlon,',',xlon,']' quit(-1) if latv < nlat or latv > xlat: print errormsg print ' ' + fname + ': latitude outside data range!!' print ' given value:', latv,'outside [',nlat,',',xlat,']' quit(-1) # Find point difflonlat = np.sqrt((ilon-lonv)**2. + (ilat-latv)**2.) mindiffLl = np.min(difflonlat) ilatlon = index_mat(difflonlat, mindiffLl) # print 'mindiffLl:', mindiffLl, 'ilatlon:', ilatlon # print 'lon, lat:', lon[ilatlon[0],ilatlon[1]], ',', lat[ilatlon[0],ilatlon[1]] return ilatlon, mindiffLl def Partialmap_Entiremap(values, filen, varn): """ Function to transform from a partial global map (e.g.: only land points) to an entire one Coincidence of points is done throughout a first guess from fractions of the total domain of search values= [lonmame],[latname],[fillVal],[resolution],[kind],[lonlatProjfile],[fracd],[fracs] [lonname]: name of the longitude variable [latname]: name of the latitude variable [fillVal]: value for '_FillValue', 'std' for the standard value Float = 1.e20 Character = '-' Integer = -99999 Float64 = 1.e20 Integer32 = -99999 [resolution]: resolution of the map [kind]: kind of target projection 'lonlat': standard lon/lat projection tacking number of points from [dx/dy]res at the Equator 'lonlat_dxdyFix': projection with a fixed grid distance 'Goode': Goode projection [lonlatProjfile]: file with the lon,lat of the desired projection. 'None' to be computed and written on fly [fracd]: Percentage of the fractions within perform the first guess search [fracs]: frequency of points at which syncronize file with calculations filen= name of the netCDF file varn= name of the variable """ import numpy.ma as ma import subprocess as sub fname = 'Partialmap_Entiremap' if values == 'h': print fname + '_____________________________________________________________' print Partialmap_Entiremap.__doc__ quit() arguments = '[lonmame],[latname],[fillVal],[resolution],[kind],' + \ '[lonlatProjfile],[fracd],[fracs]' check_arguments(fname, values, arguments, ',') ofile = 'EntireGlobalMap.nc' onc = NetCDFFile(filen, 'r') if not onc.variables.has_key(varn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varn + "' !!" quit(-1) lonname = values.split(',')[0] latname = values.split(',')[1] fval0 = values.split(',')[2] resolution = np.float(values.split(',')[3]) kind = values.split(',')[4] Projfile = values.split(',')[5] fracd = np.float(values.split(',')[6]) fracs = int(values.split(',')[7]) if Projfile == 'None': lonlatProjfile = None else: lonlatProjfile = Projfile if not onc.variables.has_key(lonname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "longitude '" + lonname + "' !!" quit(-1) if not onc.variables.has_key(latname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "latitude '" + latname + "' !!" quit(-1) olon = onc.variables[lonname] olat = onc.variables[latname] lonvs = olon[:] latvs = olat[:] Ninpts = len(lonvs) nlat = np.min(latvs) nlon = np.min(lonvs) minlat = np.min(np.abs(latvs)) minlon = np.min(np.abs(lonvs)) # mindiff = np.min([minlat, minlon])/1. mindiff = np.min([minlat, minlon])*10.e6 print ' ' + fname + ': Closest latitude to Equator:', minlat print ' ' + fname + ': Closest longitude to Greenwich Meridian:', minlon print ' ' + fname + ': Minium distance for point coincidence:', mindiff # Existing longitudes along/closest to the Equator eqlons = [] for i in range(len(latvs)): if latvs[i] == minlat: eqlons.append(lonvs[i]) # Existing latitudes along/closest to the Greenwich Meridian grlats = [] for i in range(len(lonvs)): if lonvs[i] == minlon: grlats.append(latvs[i]) sorteqlons = np.sort(eqlons) sortgrlats = np.sort(grlats) Neqlons = len(sorteqlons) Ngrlats = len(sortgrlats) if Neqlons > 1: diffeqlons = sorteqlons[1:Neqlons-1] - sorteqlons[0:Neqlons-2] mindeqlon = np.min(diffeqlons) print 'N sorteqlons:',Neqlons,'min deqlon:', mindeqlon else: mindeqlon = None if Ngrlats > 1: mindgrlat = np.min(diffgrlats) diffgrlats = sortgrlats[1:Ngrlats-1] - sortgrlats[0:Ngrlats-2] print 'N sortgrlats:',Ngrlats,'min dgrlat:', mindgrlat latmap = np.range(0,360.,360./mindeqlon) else: mindgrlat = None # Fixing in case it has not worked if mindeqlon is not None and mindgrlat is None: mindgrlat = mindeqlon if mindeqlon is None and mindgrlat is not None: mindeqlon = mindgrlat if mindeqlon is None and mindgrlat is None: print errormsg print fname + ': Not enough values along the Equator and Greenwich!!' quit(-1) if lonlatProjfile is None: lonlatProjfile = kind + '.nc' print warnmsg print ' ' + fname + ": no reference map !!" print " creation of '" + lonlatProjfile + "'" print ' creating it via: lonlatProj(', resolution, ',', resolution, \ ', ' + kind + ', True)' lonmap, latmap = lonlatProj(resolution, resolution, kind, True) sub.call(['mv','lonlatProj.nc',lonlatProjfile]) oproj = NetCDFFile(lonlatProjfile, 'r') if kind == 'Goode': olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] projlon = olon[:] projlat = olat[:] omapvals = oproj.variables['ptid'] Ntotvals = omapvals.shape[0] * omapvals.shape[1] Nmaplon = olon.shape[1] Nmaplat = olat.shape[0] elif kind == 'lonlat': olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] projlon = olon[:] projlat = olat[:] omapvals = oproj.variables['points'] Ntotvals = omapvals.shape[0] * omapvals.shape[1] Nmaplon = olon.shape[0] Nmaplat = olat.shape[0] elif kind == 'lonlat_dxdyFix': oprojlon = oproj.variables['loncirc'] oprojlat = oproj.variables['latcirc'] oNprojlon = oproj.variables['Nloncirc'] projlat = oprojlat[:] Nprojlon = oNprojlon[:] Ntotvals = len(oproj.dimensions['Npts']) olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] omapvals = oproj.variables['points'] # Ntotvals = len(lonmap) Nmaplon = Ntotvals Nmaplat = Ntotvals else: print errormsg print ' ' + fname + ": projection kind '" + kind + "' not ready!!" quit(-1) # Matrix map creation ## ovar = onc.variables[varn] vartype = type(ovar[0]) varlongname = ovar.long_name varunits = ovar.units fval = fillvalue_kind(type(ovar[0]), fval0) # Final map creation ## if not os.path.isfile(ofile): print ' ' + fname + "File '" + ofile + "' does not exist !!" print ' cration of output file' newnc = NetCDFFile(ofile, 'w') # dimensions newdim = newnc.createDimension('lon',Nmaplon) newdim = newnc.createDimension('lat',Nmaplat) newdim = newnc.createDimension('inpts', Ninpts) newdim = newnc.createDimension('pts', Ntotvals) # Variables if kind == 'Goode': newvar = newnc.createVariable('lon','f8',('lat', 'lon')) else: newvar = newnc.createVariable('lon','f8',('lon')) basicvardef(newvar, 'lon', 'Longitudes','degrees_East') newvar[:] = lonmap newvar.setncattr('axis', 'X') newvar.setncattr('_CoordinateAxisType', 'Lon') if kind == 'Goode': newvar = newnc.createVariable('lat','f8',('lat','lon')) else: newvar = newnc.createVariable('lat','f8',('lat')) basicvardef(newvar, 'lat', 'Latitudes','degrees_North') newvar[:] = latmap newvar.setncattr('axis', 'Y') newvar.setncattr('_CoordinateAxisType', 'Lat') newvarinpt = newnc.createVariable('locinpt','i',('inpts')) basicvardef(newvarinpt, 'locinpt', 'input point located: 0: no, 1: yes','-') newvarindiff = newnc.createVariable('locindiff','f4',('inpts')) basicvardef(newvarindiff, 'locindiff', 'distance between input point and its final location','degree') # map variable Lmapvalsshape = len(omapvals.shape) if Lmapvalsshape == 1: newvar = newnc.createVariable(varn, nctype(vartype), ('pts'), fill_value=fval) else: newvar = newnc.createVariable(varn, nctype(vartype), ('lat','lon'), fill_value=fval) basicvardef(newvar, varn, varlongname, varunits) newvar.setncattr('coordinates', 'lon lat') if Lmapvalsshape == 1: newvarin = newnc.createVariable('inpts', 'i4', ('pts')) else: newvarin = newnc.createVariable('inpts', 'i4', ('lat','lon')) basicvardef(newvarin, 'inpts', 'Equivalent point from the input source', '-') newvar.setncattr('coordinates', 'lon lat') else: print ' ' + fname + "File '" + ofile + "' exists !!" print ' reading values from file' newnc = NetCDFFile(ofile, 'a') newvar = newnc.variables[varn] newvarin = newnc.variables['inpts'] newvarinpt = newnc.variables['locinpt'] newvarindiff = newnc.variables['locindiff'] amsk = np.arange(3) amsk = ma.masked_equal(amsk, 0) # fraclon = projlon[::Nmaplon*0.1] # fraclat = projlat[::Nmaplat*0.1] # print 'fraclon________________', fraclon.shape # print fraclon # print 'fraclat________________', fraclat.shape # print fraclat # Reducing the searching points newvarinvals = newvarinpt[:] maskpt = np.where(newvarinvals.mask == True, False, True) points = np.arange(Ninpts) mapoints = ma.array(points, mask=maskpt) ptsf = mapoints.compressed() Nptsf = len(ptsf) print Ninpts,'Npoints to find:', len(ptsf), ptsf[0:10], newvarindiff[ptsf[0:10]] # Error at 150024, 150025, 151709, 153421 print ' ' + fname + ': from:', Ninpts,'re-locating:',Nptsf,'points...' if kind == 'Goode': empty = CoarselonlatFindAll(newnc,projlon,projlat,lonvs[ptsf],latvs[ptsf], \ fracd,fracs,False) # for iv0 in range(Nptsf): # iv = mapoints[ptsf[iv0]] # if newvarinpt[iv] == 0: ## difflonlat = np.sqrt((projlon-lonvs[iv])**2. + (projlat-latvs[iv])**2.) ## mindiffLl = np.min(difflonlat) ## ilatlon = index_mat(difflonlat, mindiffLl) # ilatlon, mindiffLl = CoarselonlatFind(projlon,projlat,lonvs[iv],latvs[iv],fracd) ## if ilatlon[0] != ilatlon2[0] or ilatlon[1] != ilatlon2[1]: ## print ilatlon, '<', ilatlon2 ## print 'diff:', mindiffLl, '<', mindiffLl2 ## quit(-1) ## if mindiffLl != 0. or type(newvar[ilatlon[0],ilatlon[1]]) != type(amsk): ## print errormsg ## if mindiffLl !=0.: ## print ' ' + main + ': no zero', newvarindiff[iv], 'distance!!' ## else: ## print ' ' + main+': point',projlon[ilatlon[0],ilatlon[1]], \ ## ',', projlat[ilatlon[0],ilatlon[1]], 'already filled!!' ## print ' value:',newvar[ilatlon[0],ilatlon[1]],'distance:',\ ## newvarindiff[iv] ## ### print ' iv:', iv, 'lon:', lonvs[iv],'lat:',latvs[iv] ### print ' mindiffLl:',mindiffLl,'ilatlon:',ilatlon ### quit(-1) ### print ' Lluis; ' + fname + ' iv:', iv, 'lon:', lonvs[iv],'lat:',latvs[iv] ### print ' Lluis; ' + fname + ' mindiffLl:',mindiffLl,'ilatlon:',ilatlon ### print ' Lluis; ' + fname + ' A found lon:',projlon[ilatlon[0], ilatlon[1]], 'lat:', projlat[ilatlon[0], ilatlon[1]] ### quit() ## else: # if mindiffLl == 0. and type(newvar[ilatlon[0],ilatlon[1]]) == type(amsk): # percendone(iv,Ninpts,0.5,'done:') # if mindiffLl > mindiff: # print errormsg # print ' ' + fname + ': for point #', iv,'lon,lat in ' + \ # 'incomplet map:', lonvs[iv], ',', latvs[iv], 'there is ' + \ # 'not a set of lon,lat in the completed map closer than: ', \ # mindiff, '!!' # print ' minimum difference:', mindiffLl # quit(-1) # if ilatlon[0] >= 0 and ilatlon[1] >= 0: # newvar[ilatlon[0],ilatlon[1]] = ovar[iv] # newvarin[ilatlon[0],ilatlon[1]] = iv # newvarinpt[iv] = 1 # newvarindiff[iv] = mindiffLl ## print 'Lluis iv:', newvarin[ilatlon[0],ilatlon[1]], \ ## 'localized:', newvarinpt[iv], 'values:', \ ## newvar[ilatlon[0],ilatlon[1]], 'invalues:', ovar[iv], \ ## 'mindist:', newvarindiff[iv], 'point:',ilatlon # else: # print errormsg # print ' ' + fname + ': point iv:', iv, 'at', lonvs[iv], ',',\ # latvs[iv],' not relocated !!' # print ' mindiffl:', mindiffLl, 'ilon:', ilatlon[1], \ # 'ilatlon:', ilatlon[1] # quit(-1) # if np.mod(iv,1000) == 0: # newnc.sync() ## print ' ' + fname + 'values localized:', newvar[:].compressed() ## if iv > 10: ## newnc.sync() ## newnc.close() ## quit(-1) elif kind == 'lonlat': for iv in range(Ntotvals): difflat = np.abs(projlat - latvs[iv]) mindiffL = np.min(difflat) ilat = index_mat(difflat, mindiffL) difflon = np.abs(projlon - lonvs[iv]) mindiffl = difflon.min() ilon = index_mat(difflon, mindiffl) percendone(iv,Ntotvals,0.5,'done:') if mindiffl > mindiff or mindiffL > mindiff: print errormsg print ' ' + fname + ': for point #', iv,'lon,lat in incomplet map:', \ lonvs[iv], ',', latvs[iv], 'there is not a set of lon,lat in the ' + \ 'completed map closer than: ', mindiff, '!!' print ' minimum difflon:', np.min(difflon), 'difflat:', np.min(difflat) quit() if ilon >= 0 and ilat >= 0: newvar[ilat,ilon] = ovar[iv] newnc.sync() else: print errormsg print ' ' + fname + ': point iv:',iv,'at',lonvs[iv],',',latvs[iv],' not relocated !!' print ' mindiffl:',mindiffl,'mindiffL:',mindiffL,'ilon:',ilon,'ilat:',ilat quit(-1) newnc.sync() elif kind == 'lonlat_dxdyFix': for iv in range(Ntotvals): # for iv in range(15): # print 'Lluis:',iv,'lon lat:',lonvs[iv],',',latvs[iv] difflat = np.abs(projlat - latvs[iv]) mindiffL = np.min(difflat) ilat = index_mat(difflat, mindiffL) # print ' Lluis mindiffL:',mindiffL,'<> ilat:',ilat,'lat_ilat:',projlat[ilat],'Nprojlon:',Nprojlon[ilat],'shape oporjlon:',oprojlon[ilat,:].shape, type(oprojlon[ilat,:]) loncirc = np.zeros((Nprojlon[ilat]), dtype=np.float) loncirc[:] = np.asarray(oprojlon[ilat,0:int(Nprojlon[ilat])].flatten()) difflon = np.abs(loncirc - lonvs[iv]) mindiffl = difflon.min() ilon = index_mat(difflon, mindiffl) # print ' difflon:',type(difflon),'shape difflon',difflon.shape,'shape loncirc:', loncirc.shape # print ' Lluis mindiffl:',mindiffl,'<> ilon:',ilon,'oprojlon:', loncirc[ilon] percendone(iv,Ntotvals,0.5,'done:') if mindiffl > mindiff or mindiffL > mindiff: print errormsg print ' ' + fname + ': for point #', iv,'lon,lat in incomplet map:', \ lonvs[iv], ',', latvs[iv], 'there is not a set of lon,lat in the ' + \ 'completed map closer than: ', mindiff, '!!' print ' minimum difflon:', np.min(difflon), 'difflat:', np.min(difflat) quit() if ilon >= 0 and ilat >= 0: if Lmapvalsshape ==1: newvar[iv] = ovar[iv] newnc.sync() else: print errormsg print ' ' + fname + ': point iv:',iv,'at',lonvs[iv],',',latvs[iv],' not relocated !!' print ' mindiffl:',mindiffl,'mindiffL:',mindiffL,'ilon:',ilon,'ilat:',ilat quit(-1) newnc.sync() # Global attributes ## newnc.setncattr('script', fname) newnc.setncattr('version', '1.0') newnc.setncattr('author', 'L. Fita') newattr = set_attributek(newnc, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') newnc.setncattr('university', 'Pierre et Marie Curie') newnc.setncattr('country', 'France') for attrs in onc.ncattrs(): attrv = onc.getncattr(attrs) attr = set_attribute(newnc, attrs, attrv) newnc.sync() onc.close() newnc.close() print fname + ": Successfull written of file: '" + ofile + "' !!" return def Partialmap_EntiremapFor(values, filen, varn): """ Function to transform from a partial global map (e.g.: only land points) to an entire one usinf Fortran code Coincidence of points is done throughout a first guess from fractions of the total domain of search Using fortran codes: module_ForInterpolate.F90, module_generic.F90 foudre: f2py -m module_ForInterpolate --f90exec=/usr/bin/gfortran-4.7 -c module_ForInterpolate.F90 module_generic.F90 >& run_f2py.log ciclad: f2py --f90flags="-fPIC" -L/opt/canopy-1.3.0/Canopy_64bit/System/lib/ -L/usr/lib64/ -L/opt/canopy-1.3.0/Canopy_64bit/System/lib/ -m module_ForInterpolate -c module_ForInterpolate.F90 module_generic.F90 values= [lonmame],[latname],[fillVal],[resolution],[kind],[lonlatProjfile],[ipoint],[fracd],[fracs] [lonname]: name of the longitude variable [latname]: name of the latitude variable [fillVal]: value for '_FillValue', 'std' for the standard value Float = 1.e20 Character = '-' Integer = -99999 Float64 = 1.e20 Integer32 = -99999 [resolution]: resolution of the map [kind]: kind of target projection 'lonlat': standard lon/lat projection tacking number of points from [dx/dy]res at the Equator 'lonlat_dxdyFix': projection with a fixed grid distance 'Goode': Goode projection [lonlatProjfile]: file with the lon,lat of the desired projection. 'None' to be computed and written on fly [ipoint]: initial point to use for the interpolation (0, the first) [fracd]: Percentage of the fractions within perform the first guess search [fracs]: Number of grid points to perform the syncronization with the file and the computed values filen= name of the netCDF file varn= name of the variable """ import module_ForInterpolate as fin import numpy.ma as ma import subprocess as sub fname = 'Partialmap_Entiremap' if values == 'h': print fname + '_____________________________________________________________' print Partialmap_Entiremap.__doc__ quit() arguments = '[lonmame],[latname],[fillVal],[resolution],[kind],' + \ '[lonlatProjfile],[ipoint],[fracd],[fracs]' check_arguments(fname, values, arguments, ',') ofile = 'EntireGlobalMap.nc' onc = NetCDFFile(filen, 'r') if not onc.variables.has_key(varn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varn + "' !!" quit(-1) lonname = values.split(',')[0] latname = values.split(',')[1] fval0 = values.split(',')[2] resolution = np.float(values.split(',')[3]) kind = values.split(',')[4] Projfile = values.split(',')[5] ipoint = int(values.split(',')[6]) fracd = np.float(values.split(',')[7]) fracs = int(values.split(',')[8]) if Projfile == 'None': lonlatProjfile = None else: lonlatProjfile = Projfile if not onc.variables.has_key(lonname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "longitude '" + lonname + "' !!" quit(-1) if not onc.variables.has_key(latname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "latitude '" + latname + "' !!" quit(-1) olon = onc.variables[lonname] olat = onc.variables[latname] lonvs = olon[:] latvs = olat[:] Ninpts = len(lonvs) nlat = np.min(latvs) nlon = np.min(lonvs) minlat = np.min(np.abs(latvs)) minlon = np.min(np.abs(lonvs)) # mindiff = np.min([minlat, minlon])/1. mindiff = np.min([minlat, minlon])*10.e6 print ' ' + fname + ': Closest latitude to Equator:', minlat print ' ' + fname + ': Closest longitude to Greenwich Meridian:', minlon print ' ' + fname + ': Minium distance for point coincidence:', mindiff # Existing longitudes along/closest to the Equator eqlons = [] for i in range(len(latvs)): if latvs[i] == minlat: eqlons.append(lonvs[i]) # Existing latitudes along/closest to the Greenwich Meridian grlats = [] for i in range(len(lonvs)): if lonvs[i] == minlon: grlats.append(latvs[i]) sorteqlons = np.sort(eqlons) sortgrlats = np.sort(grlats) Neqlons = len(sorteqlons) Ngrlats = len(sortgrlats) if Neqlons > 1: diffeqlons = sorteqlons[1:Neqlons-1] - sorteqlons[0:Neqlons-2] mindeqlon = np.min(diffeqlons) print 'N sorteqlons:',Neqlons,'min deqlon:', mindeqlon else: mindeqlon = None if Ngrlats > 1: mindgrlat = np.min(diffgrlats) diffgrlats = sortgrlats[1:Ngrlats-1] - sortgrlats[0:Ngrlats-2] print 'N sortgrlats:',Ngrlats,'min dgrlat:', mindgrlat latmap = np.range(0,360.,360./mindeqlon) else: mindgrlat = None # Fixing in case it has not worked if mindeqlon is not None and mindgrlat is None: mindgrlat = mindeqlon if mindeqlon is None and mindgrlat is not None: mindeqlon = mindgrlat if mindeqlon is None and mindgrlat is None: print errormsg print fname + ': Not enough values along the Equator and Greenwich!!' quit(-1) if lonlatProjfile is None: lonlatProjfile = kind + '.nc' print warnmsg print ' ' + fname + ": no reference map !!" print " creation of '" + lonlatProjfile + "'" print ' creating it via: lonlatProj(', resolution, ',', resolution, \ ', ' + kind + ', True)' lonmap, latmap = lonlatProj(resolution, resolution, kind, True) sub.call(['mv','lonlatProj.nc',lonlatProjfile]) oproj = NetCDFFile(lonlatProjfile, 'r') if kind == 'Goode': olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] projlon = olon[:].astype('float64') projlat = olat[:].astype('float64') omapvals = oproj.variables['ptid'] Ntotvals = omapvals.shape[0] * omapvals.shape[1] Nmaplon = olon.shape[1] Nmaplat = olat.shape[0] elif kind == 'lonlat': olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] projlon = olon[:] projlat = olat[:] omapvals = oproj.variables['points'] Ntotvals = omapvals.shape[0] * omapvals.shape[1] Nmaplon = olon.shape[0] Nmaplat = olat.shape[0] elif kind == 'lonlat_dxdyFix': oprojlon = oproj.variables['loncirc'] oprojlat = oproj.variables['latcirc'] oNprojlon = oproj.variables['Nloncirc'] projlat = oprojlat[:] Nprojlon = oNprojlon[:] Ntotvals = len(oproj.dimensions['Npts']) olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] omapvals = oproj.variables['points'] # Ntotvals = len(lonmap) Nmaplon = Ntotvals Nmaplat = Ntotvals else: print errormsg print ' ' + fname + ": projection kind '" + kind + "' not ready!!" quit(-1) # Matrix map creation ## ovar = onc.variables[varn] vartype = type(ovar[0]) varlongname = ovar.long_name varunits = ovar.units fval = fillvalue_kind(type(ovar[0]), fval0) # Final map creation ## if not os.path.isfile(ofile): print ' ' + fname + "File '" + ofile + "' does not exist !!" print ' cration of output file' newnc = NetCDFFile(ofile, 'w') # dimensions newdim = newnc.createDimension('lon',Nmaplon) newdim = newnc.createDimension('lat',Nmaplat) newdim = newnc.createDimension('inpts', Ninpts) newdim = newnc.createDimension('pts', Ntotvals) # Variables if kind == 'Goode': newvar = newnc.createVariable('lon','f8',('lat', 'lon')) else: newvar = newnc.createVariable('lon','f8',('lon')) basicvardef(newvar, 'lon', 'Longitudes','degrees_East') newvar[:] = lonmap newvar.setncattr('axis', 'X') newvar.setncattr('_CoordinateAxisType', 'Lon') if kind == 'Goode': newvar = newnc.createVariable('lat','f8',('lat','lon')) else: newvar = newnc.createVariable('lat','f8',('lat')) basicvardef(newvar, 'lat', 'Latitudes','degrees_North') newvar[:] = latmap newvar.setncattr('axis', 'Y') newvar.setncattr('_CoordinateAxisType', 'Lat') newvarinpt = newnc.createVariable('locinpt','i',('inpts')) basicvardef(newvarinpt, 'locinpt', 'input point located: 0: no, 1: yes','-') newvarindiff = newnc.createVariable('locindiff','f4',('inpts')) basicvardef(newvarindiff, 'locindiff', 'distance between input point and its final location','degree') # map variable Lmapvalsshape = len(omapvals.shape) if Lmapvalsshape == 1: newvar = newnc.createVariable(varn, nctype(vartype), ('pts'), fill_value=fval) else: newvar = newnc.createVariable(varn, nctype(vartype), ('lat','lon'), fill_value=fval) basicvardef(newvar, varn, varlongname, varunits) newvar.setncattr('coordinates', 'lon lat') if Lmapvalsshape == 1: newvarin = newnc.createVariable('inpts', 'i4', ('pts')) else: newvarin = newnc.createVariable('inpts', 'i4', ('lat','lon')) basicvardef(newvarin, 'inpts', 'Equivalent point from the input source', '-') newvar.setncattr('coordinates', 'lon lat') else: print ' ' + fname + "File '" + ofile + "' exists !!" print ' reading values from file' newnc = NetCDFFile(ofile, 'a') newvar = newnc.variables[varn] newvarin = newnc.variables['inpts'] newvarinpt = newnc.variables['locinpt'] newvarindiff = newnc.variables['locindiff'] amsk = np.arange(3) amsk = ma.masked_equal(amsk, 0) # fraclon = projlon[::Nmaplon*0.1] # fraclat = projlat[::Nmaplat*0.1] # print 'fraclon________________', fraclon.shape # print fraclon # print 'fraclat________________', fraclat.shape # print fraclat # Reducing the searching points newvarinvals = newvarinpt[:] maskpt = np.where(newvarinvals.mask == True, False, True) points = np.arange(Ninpts) mapoints = ma.array(points, mask=maskpt) ptsf = mapoints.compressed() Nptsf = len(ptsf) print Ninpts,'Npoints to find:', len(ptsf), ptsf[0:10], newvarindiff[ptsf[0:10]] # Error at 150024, 150025, 151709, 153421 print ' ' + fname + ': from:', Ninpts,'re-locating:',Nptsf,'points...' if kind == 'Goode': # newvar,newvarin,newvarinpt,newvarindiff = \ # fin.module_forinterpolate.coarseinterpolate(projlon, projlat, lonvs, \ # latvs, percen, mindiff, ivar, dimx, dimy, ninpts) for ir in range(ipoint,Ninpts,fracs): iri = ir ire = ir + fracs + 1 print iri,',',ire # percendone(iri,Ninpts,0.5,'done:') pts = np.arange(iri,ire,1, dtype='int32') lonvss = lonvs[iri:ire].astype('float64') latvss = latvs[iri:ire].astype('float64') ovars = ovar[iri:ire].astype('float64') newvar,newvarin,newvarinpt[pts],newvarindiff[pts] = \ fin.module_forinterpolate.coarseinterpolate(projlon, projlat, \ lonvss, latvss, np.float64(fracd), np.float64(mindiff), ovars) # Slow way # newvar,newvarin,newvarinpt[pts],newvarindiff[pts] = \ # fin.module_forinterpolate.interpolate(projlon, projlat, \ # lonvss, latvss, np.float64(mindiff), ovars) newnc.sync() elif kind == 'lonlat': for iv in range(Ntotvals): difflat = np.abs(projlat - latvs[iv]) mindiffL = np.min(difflat) ilat = index_mat(difflat, mindiffL) difflon = np.abs(projlon - lonvs[iv]) mindiffl = difflon.min() ilon = index_mat(difflon, mindiffl) percendone(iv,Ntotvals,0.5,'done:') if mindiffl > mindiff or mindiffL > mindiff: print errormsg print ' ' + fname + ': for point #', iv,'lon,lat in incomplet map:', \ lonvs[iv], ',', latvs[iv], 'there is not a set of lon,lat in the ' + \ 'completed map closer than: ', mindiff, '!!' print ' minimum difflon:', np.min(difflon), 'difflat:', np.min(difflat) quit() if ilon >= 0 and ilat >= 0: newvar[ilat,ilon] = ovar[iv] newnc.sync() else: print errormsg print ' ' + fname + ': point iv:',iv,'at',lonvs[iv],',',latvs[iv],' not relocated !!' print ' mindiffl:',mindiffl,'mindiffL:',mindiffL,'ilon:',ilon,'ilat:',ilat quit(-1) newnc.sync() elif kind == 'lonlat_dxdyFix': for iv in range(Ntotvals): # for iv in range(15): # print 'Lluis:',iv,'lon lat:',lonvs[iv],',',latvs[iv] difflat = np.abs(projlat - latvs[iv]) mindiffL = np.min(difflat) ilat = index_mat(difflat, mindiffL) # print ' Lluis mindiffL:',mindiffL,'<> ilat:',ilat,'lat_ilat:',projlat[ilat],'Nprojlon:',Nprojlon[ilat],'shape oporjlon:',oprojlon[ilat,:].shape, type(oprojlon[ilat,:]) loncirc = np.zeros((Nprojlon[ilat]), dtype=np.float) loncirc[:] = np.asarray(oprojlon[ilat,0:int(Nprojlon[ilat])].flatten()) difflon = np.abs(loncirc - lonvs[iv]) mindiffl = difflon.min() ilon = index_mat(difflon, mindiffl) # print ' difflon:',type(difflon),'shape difflon',difflon.shape,'shape loncirc:', loncirc.shape # print ' Lluis mindiffl:',mindiffl,'<> ilon:',ilon,'oprojlon:', loncirc[ilon] percendone(iv,Ntotvals,0.5,'done:') if mindiffl > mindiff or mindiffL > mindiff: print errormsg print ' ' + fname + ': for point #', iv,'lon,lat in incomplet map:', \ lonvs[iv], ',', latvs[iv], 'there is not a set of lon,lat in the ' + \ 'completed map closer than: ', mindiff, '!!' print ' minimum difflon:', np.min(difflon), 'difflat:', np.min(difflat) quit() if ilon >= 0 and ilat >= 0: if Lmapvalsshape ==1: newvar[iv] = ovar[iv] newnc.sync() else: print errormsg print ' ' + fname + ': point iv:',iv,'at',lonvs[iv],',',latvs[iv],' not relocated !!' print ' mindiffl:',mindiffl,'mindiffL:',mindiffL,'ilon:',ilon,'ilat:',ilat quit(-1) newnc.sync() # Global attributes ## newnc.setncattr('script', fname) newnc.setncattr('version', '1.0') newnc.setncattr('author', 'L. Fita') newattr = set_attributek(newnc, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') newnc.setncattr('university', 'Pierre et Marie Curie') newnc.setncattr('country', 'France') for attrs in onc.ncattrs(): attrv = onc.getncattr(attrs) attr = set_attribute(newnc, attrs, attrv) newnc.sync() onc.close() newnc.close() print fname + ": Successfull written of file: '" + ofile + "' !!" return def Partialmap_EntiremapForExact(values, filen, varn): """ Function to transform from a partial global map (e.g.: only land points) to an entire one using Fortran code with exact location Coincidence of points is done throughout a first guess from fractions of the total domain of search Using fortran codes: module_ForInterpolate.F90, module_generic.F90 foudre: f2py -m module_ForInterpolate --f90exec=/usr/bin/gfortran-4.7 -c module_ForInterpolate.F90 module_generic.F90 >& run_f2py.log ciclad: f2py --f90flags="-fPIC" -L/opt/canopy-1.3.0/Canopy_64bit/System/lib/ -L/usr/lib64/ -L/opt/canopy-1.3.0/Canopy_64bit/System/lib/ -m module_ForInterpolate -c module_ForInterpolate.F90 module_generic.F90 values= [lonmame],[latname],[fillVal],[resolution],[kind],[lonlatProjfile],[ipoint],[fracd],[fracs],[mindiff] [lonname]: name of the longitude variable [latname]: name of the latitude variable [fillVal]: value for '_FillValue', 'std' for the standard value Float = 1.e20 Character = '-' Integer = -99999 Float64 = 1.e20 Integer32 = -99999 [resolution]: resolution of the map [kind]: kind of target projection 'lonlat': standard lon/lat projection tacking number of points from [dx/dy]res at the Equator 'lonlat_dxdyFix': projection with a fixed grid distance 'Goode': Goode projection [lonlatProjfile]: file with the lon,lat of the desired projection. 'None' to be computed and written on fly [ipoint]: initial point to use for the interpolation (0, the first) [fracd]: Percentage of the fractions within perform the first guess search [fracs]: Number of grid points to perform the syncronization with the file and the computed values [mindiff]: Authorized minium distance between input and final lon,lat point filen= name of the netCDF file varn= name of the variable """ import module_ForInterpolate as fin import numpy.ma as ma import subprocess as sub fname = 'Partialmap_EntiremapFortran' if values == 'h': print fname + '_____________________________________________________________' print Partialmap_Entiremap.__doc__ quit() arguments = '[lonmame],[latname],[fillVal],[resolution],[kind],' + \ '[lonlatProjfile],[ipoint],[fracd],[fracs],[mindiff]' check_arguments(fname, values, arguments, ',') ofile = 'EntireGlobalMap.nc' onc = NetCDFFile(filen, 'r') if not onc.variables.has_key(varn): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "variable '" + varn + "' !!" quit(-1) lonname = values.split(',')[0] latname = values.split(',')[1] fval0 = values.split(',')[2] resolution = np.float(values.split(',')[3]) kind = values.split(',')[4] Projfile = values.split(',')[5] ipoint = int(values.split(',')[6]) fracd = np.float64(values.split(',')[7]) fracs = int(values.split(',')[8]) mindiff = np.float64(values.split(',')[9]) if Projfile == 'None': lonlatProjfile = None else: lonlatProjfile = Projfile if not onc.variables.has_key(lonname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "longitude '" + lonname + "' !!" quit(-1) if not onc.variables.has_key(latname): print errormsg print ' ' + fname + ": file '" + filen + "' does not have " + \ "latitude '" + latname + "' !!" quit(-1) olon = onc.variables[lonname] olat = onc.variables[latname] lonvs = olon[:] latvs = olat[:] Ninpts = len(lonvs) nlat = np.min(latvs) nlon = np.min(lonvs) minlat = np.min(np.abs(latvs)) minlon = np.min(np.abs(lonvs)) print ' ' + fname + ': Closest latitude to Equator:', minlat print ' ' + fname + ': Closest longitude to Greenwich Meridian:', minlon print ' ' + fname + ': Minium distance for point coincidence:', mindiff # Existing longitudes along/closest to the Equator eqlons = [] for i in range(len(latvs)): if latvs[i] == minlat: eqlons.append(lonvs[i]) # Existing latitudes along/closest to the Greenwich Meridian grlats = [] for i in range(len(lonvs)): if lonvs[i] == minlon: grlats.append(latvs[i]) sorteqlons = np.sort(eqlons) sortgrlats = np.sort(grlats) Neqlons = len(sorteqlons) Ngrlats = len(sortgrlats) if Neqlons > 1: diffeqlons = sorteqlons[1:Neqlons-1] - sorteqlons[0:Neqlons-2] mindeqlon = np.min(diffeqlons) print 'N sorteqlons:',Neqlons,'min deqlon:', mindeqlon else: mindeqlon = None if Ngrlats > 1: mindgrlat = np.min(diffgrlats) diffgrlats = sortgrlats[1:Ngrlats-1] - sortgrlats[0:Ngrlats-2] print 'N sortgrlats:',Ngrlats,'min dgrlat:', mindgrlat latmap = np.range(0,360.,360./mindeqlon) else: mindgrlat = None # Fixing in case it has not worked if mindeqlon is not None and mindgrlat is None: mindgrlat = mindeqlon if mindeqlon is None and mindgrlat is not None: mindeqlon = mindgrlat if mindeqlon is None and mindgrlat is None: print errormsg print fname + ': Not enough values along the Equator and Greenwich!!' quit(-1) if lonlatProjfile is None: lonlatProjfile = kind + '.nc' print warnmsg print ' ' + fname + ": no reference map !!" print " creation of '" + lonlatProjfile + "'" print ' creating it via: lonlatProj(', resolution, ',', resolution, \ ', ' + kind + ', True)' lonmap, latmap = lonlatProj(resolution, resolution, kind, True) sub.call(['mv','lonlatProj.nc',lonlatProjfile]) oproj = NetCDFFile(lonlatProjfile, 'r') if kind == 'Goode': olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] projlon = olon[:].astype('float64') projlat = olat[:].astype('float64') omapvals = oproj.variables['ptid'] Ntotvals = omapvals.shape[0] * omapvals.shape[1] Nmaplon = olon.shape[1] Nmaplat = olat.shape[0] elif kind == 'lonlat': olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] projlon = olon[:] projlat = olat[:] omapvals = oproj.variables['points'] Ntotvals = omapvals.shape[0] * omapvals.shape[1] Nmaplon = olon.shape[0] Nmaplat = olat.shape[0] elif kind == 'lonlat_dxdyFix': oprojlon = oproj.variables['loncirc'] oprojlat = oproj.variables['latcirc'] oNprojlon = oproj.variables['Nloncirc'] projlat = oprojlat[:] Nprojlon = oNprojlon[:] Ntotvals = len(oproj.dimensions['Npts']) olon = oproj.variables['lon'] olat = oproj.variables['lat'] lonmap = olon[:] latmap = olat[:] omapvals = oproj.variables['points'] # Ntotvals = len(lonmap) Nmaplon = Ntotvals Nmaplat = Ntotvals else: print errormsg print ' ' + fname + ": projection kind '" + kind + "' not ready!!" quit(-1) # Matrix map creation ## ovar = onc.variables[varn] vartype = type(ovar[0]) varlongname = ovar.long_name varunits = ovar.units fval = fillvalue_kind(type(ovar[0]), fval0) # Final map creation ## if not os.path.isfile(ofile): print ' ' + fname + "File '" + ofile + "' does not exist !!" print ' cration of output file' newnc = NetCDFFile(ofile, 'w') # dimensions newdim = newnc.createDimension('lon',Nmaplon) newdim = newnc.createDimension('lat',Nmaplat) newdim = newnc.createDimension('inpts', Ninpts) newdim = newnc.createDimension('pts', Ntotvals) # Variables if kind == 'Goode': newvar = newnc.createVariable('lon','f8',('lat', 'lon')) else: newvar = newnc.createVariable('lon','f8',('lon')) basicvardef(newvar, 'lon', 'Longitudes','degrees_East') newvar[:] = lonmap newvar.setncattr('axis', 'X') newvar.setncattr('_CoordinateAxisType', 'Lon') if kind == 'Goode': newvar = newnc.createVariable('lat','f8',('lat','lon')) else: newvar = newnc.createVariable('lat','f8',('lat')) basicvardef(newvar, 'lat', 'Latitudes','degrees_North') newvar[:] = latmap newvar.setncattr('axis', 'Y') newvar.setncattr('_CoordinateAxisType', 'Lat') newvarinpt = newnc.createVariable('locinpt','i',('inpts')) basicvardef(newvarinpt, 'locinpt', 'input point located: 0: no, 1: yes','-') newvarindiff = newnc.createVariable('locindiff','f4',('inpts')) basicvardef(newvarindiff, 'locindiff', 'distance between input point and its final location','degree') # map variable Lmapvalsshape = len(omapvals.shape) if Lmapvalsshape == 1: newvar = newnc.createVariable(varn, nctype(vartype), ('pts'), fill_value=fval) else: newvar = newnc.createVariable(varn, nctype(vartype), ('lat','lon'), fill_value=fval) basicvardef(newvar, varn, varlongname, varunits) newvar.setncattr('coordinates', 'lon lat') if Lmapvalsshape == 1: newvarin = newnc.createVariable('inpts', 'i4', ('pts')) else: newvarin = newnc.createVariable('inpts', 'i4', ('lat','lon')) basicvardef(newvarin, 'inpts', 'Equivalent point from the input source', '-') newvar.setncattr('coordinates', 'lon lat') else: print ' ' + fname + "File '" + ofile + "' exists !!" print ' reading values from file' newnc = NetCDFFile(ofile, 'a') newvar = newnc.variables[varn] newvarin = newnc.variables['inpts'] newvarinpt = newnc.variables['locinpt'] newvarindiff = newnc.variables['locindiff'] amsk = np.arange(3) amsk = ma.masked_equal(amsk, 0) # fraclon = projlon[::Nmaplon*0.1] # fraclat = projlat[::Nmaplat*0.1] # print 'fraclon________________', fraclon.shape # print fraclon # print 'fraclat________________', fraclat.shape # print fraclat # Reducing the searching points Ninpts=np.mini([499999, Ninpts-ipoint]) newvarinvals = newvarinpt[ipoint:ipoint+Ninpts] maskpt = np.where(newvarinvals.mask == True, False, True) points = np.arange(ipoint,ipoint+Ninpts,1) mapoints = ma.array(points, mask=maskpt) ptsf = mapoints.compressed() Nptsf = len(ptsf) print Ninpts,'Npoints to find:', len(ptsf), ptsf[0:10], newvarindiff[ptsf[0:10]] print ' ' + fname + ': from:', Ninpts,'re-locating:',Nptsf,'points starting at',\ ipoint,'...' if kind == 'Goode': # newvar,newvarin,newvarinpt,newvarindiff = \ # fin.module_forinterpolate.coarseinterpolate(projlon, projlat, lonvs, \ # latvs, percen, mindiff, ivar, dimx, dimy, ninpts) for ir in range(ipoint,ipoint+Ninpts,fracs): iri = ir ire = ir + fracs print iri,',',ire # percendone(iri,Ninpts,0.5,'done:') lonvss = lonvs[iri:ire+1].astype('float64') latvss = latvs[iri:ire+1].astype('float64') inptss = newvarinpt[iri:ire+1].astype('int32') # newvar,newvarin,newvarinpt[pts],newvarindiff[pts] = \ # fin.module_forinterpolate.coarseinterpolateexact(projlon, projlat, \ # lonvss, latvss, fracd, mindiff, ovars) # Slow way idiff, ilonlatv = fin.module_forinterpolate.interpolate(projlon, projlat,\ lonvss, latvss, np.float64(mindiff), inptss) for i in range(fracs): newvar[ilonlatv[i,0],ilonlatv[i,1]] = ovar[iri + i] newvarin[ilonlatv[i,0],ilonlatv[i,1]] = iri + i newvarinpt[iri+i] = 1 newvarindiff[iri+i] = idiff[i] newnc.sync() # newnc.close() # quit() elif kind == 'lonlat': for iv in range(Ntotvals): difflat = np.abs(projlat - latvs[iv]) mindiffL = np.min(difflat) ilat = index_mat(difflat, mindiffL) difflon = np.abs(projlon - lonvs[iv]) mindiffl = difflon.min() ilon = index_mat(difflon, mindiffl) percendone(iv,Ntotvals,0.5,'done:') if mindiffl > mindiff or mindiffL > mindiff: print errormsg print ' ' + fname + ': for point #', iv,'lon,lat in incomplet map:', \ lonvs[iv], ',', latvs[iv], 'there is not a set of lon,lat in the ' + \ 'completed map closer than: ', mindiff, '!!' print ' minimum difflon:', np.min(difflon), 'difflat:', np.min(difflat) quit() if ilon >= 0 and ilat >= 0: newvar[ilat,ilon] = ovar[iv] newnc.sync() else: print errormsg print ' ' + fname + ': point iv:',iv,'at',lonvs[iv],',',latvs[iv],' not relocated !!' print ' mindiffl:',mindiffl,'mindiffL:',mindiffL,'ilon:',ilon,'ilat:',ilat quit(-1) newnc.sync() elif kind == 'lonlat_dxdyFix': for iv in range(Ntotvals): # for iv in range(15): # print 'Lluis:',iv,'lon lat:',lonvs[iv],',',latvs[iv] difflat = np.abs(projlat - latvs[iv]) mindiffL = np.min(difflat) ilat = index_mat(difflat, mindiffL) # print ' Lluis mindiffL:',mindiffL,'<> ilat:',ilat,'lat_ilat:',projlat[ilat],'Nprojlon:',Nprojlon[ilat],'shape oporjlon:',oprojlon[ilat,:].shape, type(oprojlon[ilat,:]) loncirc = np.zeros((Nprojlon[ilat]), dtype=np.float) loncirc[:] = np.asarray(oprojlon[ilat,0:int(Nprojlon[ilat])].flatten()) difflon = np.abs(loncirc - lonvs[iv]) mindiffl = difflon.min() ilon = index_mat(difflon, mindiffl) # print ' difflon:',type(difflon),'shape difflon',difflon.shape,'shape loncirc:', loncirc.shape # print ' Lluis mindiffl:',mindiffl,'<> ilon:',ilon,'oprojlon:', loncirc[ilon] percendone(iv,Ntotvals,0.5,'done:') if mindiffl > mindiff or mindiffL > mindiff: print errormsg print ' ' + fname + ': for point #', iv,'lon,lat in incomplet map:', \ lonvs[iv], ',', latvs[iv], 'there is not a set of lon,lat in the ' + \ 'completed map closer than: ', mindiff, '!!' print ' minimum difflon:', np.min(difflon), 'difflat:', np.min(difflat) quit() if ilon >= 0 and ilat >= 0: if Lmapvalsshape ==1: newvar[iv] = ovar[iv] newnc.sync() else: print errormsg print ' ' + fname + ': point iv:',iv,'at',lonvs[iv],',',latvs[iv],' not relocated !!' print ' mindiffl:',mindiffl,'mindiffL:',mindiffL,'ilon:',ilon,'ilat:',ilat quit(-1) newnc.sync() # Global attributes ## newnc.setncattr('script', fname) newnc.setncattr('version', '1.0') newnc.setncattr('author', 'L. Fita') newattr = set_attributek(newnc, 'institution', unicode('Laboratoire de M' + \ unichr(233) + 't' + unichr(233) + 'orologie Dynamique'), 'U') newnc.setncattr('university', 'Pierre et Marie Curie') newnc.setncattr('country', 'France') for attrs in onc.ncattrs(): attrv = onc.getncattr(attrs) attr = set_attribute(newnc, attrs, attrv) newnc.sync() onc.close() newnc.close() print fname + ": Successfull written of file: '" + ofile + "' !!" return #Partialmap_Entiremap('longitude,latitude,std,5000.,Goode,Goode_5km.nc', '/home/lluis/etudes/DYNAMICO/ORCHIDEE/interpolation/data/#carteveg5km.nc', 'vegetation_map') #Partialmap_Entiremap('longitude,latitude,std,5000.,lonlat_dxdyFix', '/home/lluis/etudes/DYNAMICO/ORCHIDEE/interpolation/data/carteveg5km.nc', 'vegetation_map') #Partialmap_Entiremap('longitude,latitude,std,5000.,lonlat', '/home/lluis/etudes/DYNAMICO/ORCHIDEE/interpolation/data/carteveg5km.nc', 'vegetation_map') #quit() """operation to make: addfattr, add a attributes to a variable from another file: addfattr -S [file]:[var] addfdim, add a new dimension from another file: addfdim -S [file]:[dimension] addfgattr, add global attribute from another file: addfgattr -S [file] addfvar, add a new variable from another file: addfvar -S [file]:[variable] addgattr, add a new global attribute: addatr -S [attrname]|[attrvalue] addgattrk, add a new global attribute: addatr -S [attrname]|[attrvalue]|[kind(S (!, white spaces),I,R,D)] addref, add a new variable with dimension and attributes from an already existing 'variable ref' in the file: addref -S [variable ref]:[attr name]@[value]:[[attr2]@[value2], ...]:[value/file with values] mname, modify name -S newname addvals, Function to add values to a given variable at a given dimension: addvals -S [dimension]:[position]:[Nvals]:[addval], *[position]: position within the dimension at which the variable wants to be increased 'first': first position of the dimension 'middle': middle position of the dimension 'last': last position of the dimension 'num': before given value *[addval]: value to give to the added values 'val': a given value 'missing': as a missing value addvattr, add a new attribute to any given variable: addvattr -S [attrname]|[attrvalue] addvattrk, add a new attribute to any given variable: addvattrk -S [attrname]|[attrvalue]|[kind(S (!, white spaces),I,R,D)] checkNaNs, checks for NaN values over all variables in a file checkAllValues, check for variables with along all their dimensions with the same value in a file flipdim, flips the value following one dimension: flipdim -S Ndim infgattrs, give the values of all the global attributes of the file: infgattrs infsinggattrs, give the value of a single global attribute of the file: infsinggattrs -S [attribute name] infsingvattrs, give the value of a single attribute of the variable: infsingvattrs -S [attribute name] inftime, give all the information of the variable time: inftime infvars, give the names of all the variables of the file: infvars infvattrs, give the values of all the attributes of a variable: infvattrs mdname, modify dimension name: mdname -S olddname:newdname means, computes the spatial mean of the variable: means meanseas, compute the seasonal mean of a variable: meanseas -S [timename] meant, computes the temporal mean of the variable: meant -S [power](power of the polynomial fit) mname, modify name: mname -S newname out, output values: out -S inidim1,[inidim2,...]:enddim1,[enddim2,...] rgattr, read a global attribute: rgattr -S [attrname] rvattr, read a variable attribute: rvattr -S [attrname] rmgattr, remove a global attribute: rmgattr -S [attrname] rmvariable, remove a variable: rmvariable rmvattr, remove an attribute to any given variable: rmvattr -S [attrname] subsetmns, retrieve a subset of values based on months: subsetmns -S [MM1]:[...:[MMn]] or [MMi]-[MMe] for a period (output as 'subsetmns.nc') subsetyrs, retrieve a subset of values based on years: subsetyrs -S [YYYY1]:[...:[YYYYn]] or [YYYYi]-[YYYYe] for a period (output as 'subsetyrs.nc') timescheck, checks time-steps of a given file: timescheck -S [FirstDate]:[LastDate][freq]:[[auxmonth]] [FirstDate] = first date within file (in [YYYY][MM][DD][HH][MI][SS] format) [LastDate] = last date within file (in [YYYY][MM][DD][HH][MI][SS] format) [freq] = frequency of the data within the file (in time units) or 'month' (monthly values) [auxmonth] = in case of 'month' midmon: dates are given according to the mid time value of the month real: dates are given increassing a month from the previous date timesort, sort a variable time: timesort timestepchg, changes a time-step value from a file, using a given time-step from another one: timestepchg -S [origtime_step]:[newfile]:[newtime_step] valmod, modifiy values of variable -S [modification]: sumc,[constant]: add [constant] to variables values subc,[constant]: substract [constant] to variables values mulc,[constant]: multipy by [constant] to variables values divc,[constant]: divide by [constant] to variables values lowthres,[threshold],[newvalue]: modify all values below [threshold] to [newvalue] upthres,[threshold],[newvalue]: modify all values above [threshold] to [newvalue] valmod_dim(values, ncfile, varn) vartimeshidft, temporaly shift a number of time-steps a given variable inside a netCDF file: vartimeshift -S [nsteps]:[[FillValue]] varmask, mask a variable using a mask: varmask -S [maskfilename]:[maskvar]:[dimsmask]:[timename]. It is assumed that mask[...,dimM,dimJ,dimK] and var[...,dimM,dimJ,dimK] share the last dimensions 'addfattr': fattradd(opts.varname, opts.values, opts.ncfile) 'addfdim': fdimadd(opts.values, opts.ncfile) 'addfgattr': fgaddattr(opts.values, opts.ncfile) 'addfvar': fvaradd(opts.values, opts.ncfile) 'addgattr': gaddattr(opts.values, opts.ncfile) 'addgattrk': gaddattrk(opts.values, opts.ncfile) 'addref': varaddref(opts.values, opts.ncfile, opts.varname) 'addvattr': varaddattr(opts.values, opts.ncfile, opts.varname) 'addvattrk': varaddattrk(opts.values, opts.ncfile, opts.varname) 'DataSetSection': datasetsection(opts.values,opts.ncfile) 'dimflip': flipdim(opts.values,opts.ncfile,opts.varname) 'infgattrs': igattr(opts.ncfile) 'infsinggattrs': isgattrs(opts.values, opts.ncfile) 'infsingvattrs': isvattrs(opts.values, opts.ncfile, opts.varname) 'inftime': timeinf(opts.ncfile, opts.varname) 'infvars': ivars(opts.ncfile) 'infvattrs': ivattr(opts.ncfile, opts.varname) 'mdname': chdimname(opts.values, opts.ncfile, opts.varname) 'means': spacemean(opts.ncfile, opts.varname) 'meanseas': spacemean(opst.values, opts.ncfile, opts.varname) 'meant': timemean(opts.values, opts.ncfile, opts.varname) 'mname': chvarname(opts.values, opts.ncfile, opts.varname) 'out': varout(opts.values, opts.ncfile, opts.varname) 'rgattr': grattr(opts.values, opts.ncfile) 'rvattr': vrattr(opts.values, opts.ncfile, opts.varname) 'rmgattr': grmattr(opts.values, opts.ncfile) 'rmvariable': varrm(opts.ncfile, opts.varname) 'rmvattr': varrmattr(opts.values, opts.ncfile, opts.varname) 'subsetmns': submns(opts.values, opts.ncfile, opts.varname) 'subsetyrs': subyrs(opts.values, opts.ncfile, opts.varname) 'timescheck': check_times_file(opts.values, opts.ncfile, opts.varname) 'timesort': sorttimesmat(opts.ncfile, opts.varname) 'timestepchg': chgtimestep(opts.values, opts.ncfile, opts.varname) 'valsadd': addvals(opts.values, opts.ncfile, opts.varname) 'valmod': valmod(opts.values, opts.ncfile, opts.varname) 'varmask': maskvar(opts.values, opts.ncfile, opts.varname) 'vartimeshift': timeshiftvar(opts.values, opts.ncfile, opts.varname) """