Changeset 2770 in lmdz_wrf for trunk/tools
- Timestamp:
- Nov 18, 2019, 4:41:01 AM (5 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/tools/nc_var_tools.py
r2768 r2770 141 141 # netcdf_concatenation: Function to concatenate netCDF files for a given set of variable 142 142 # netcdf_fold_concatenation: Function to concatenate netCDF files in a given folder for a given set of variables 143 # netcdf_fold_slice_concatenation_HMT: Function to concatenate netCDF files in a given folder for a given slice and 144 # for a given set of variables giving Header, Middle, Tail for the name files 143 145 # netcdf_fold_concatenation_HMT: Function to concatenate netCDF files in a given folder for a given set of variables giving Header, Middle, Tail for the name files 144 146 # operation_alongdims: Function to operate along different dimensions of a variable … … 32025 32027 #CFfile_fixTime(values, 'CFtest.nc', 'time') 32026 32028 32029 def netcdf_fold_slice_concatenation_HMT(values, ncfile, varn): 32030 """ Function to concatenate netCDF files in a given folder for a given slice and 32031 for a given set of variables giving Header, Middle, Tail for the name files 32032 netcdf_fold_concateation(values, ncfile, varn) 32033 [values]= [slice],[fold],[dimname],[vardimname] 32034 [slice]= ':' list of [dimn]|[sec] to use for slicing [varn] 32035 [sec]: section 32036 * [integer]: which value of the dimension 32037 * -1: all along the dimension 32038 * -9: last value of the dimension 32039 * [beg]@[end]@[freq] slice from [beg] to [end] every [freq] 32040 * NOTE: no value provided is for all dimension range 32041 [fold]: folder with the location of the netCDF files 32042 [dimname]: dimension along which files should be concatenated 32043 [vardimname]: name of the dimension variable with the values ('WRFtime' for 32044 CF time conversion from WRF 'Times') 32045 [ncfile]= [header],[middle],[tail] 32046 [header] = header of the name of the files to concatenate [ncfile]*[middle]*[tail] 32047 [middle] = middle of the name of the files to concatenate [ncfile]*[middle]*[tail] 32048 [tail] = tail of the name of the files to concatenate [ncfile]*[middle]*[tail] 32049 [varn]= ',' separated list of variables to concatenate 32050 [var1],[var2],[...[varN]] or 'all' for all variables 32051 """ 32052 import subprocess as sub 32053 fname='netcdf_fold_slice_concatenation_HMT' 32054 32055 ofile = 'netcdf_fold_slice_concatenated_HMT.nc' 32056 32057 if values == 'h': 32058 print fname + '_____________________________________________________________' 32059 print netcdf_fold_slice_concatenation_HMT.__doc__ 32060 quit() 32061 32062 expectargs = '[slice],[fold],[dimname],[vardimname]' 32063 gen.check_arguments(fname,values,expectargs,':') 32064 32065 slicevS = values.split(',')[0] 32066 fold = values.split(',')[1] 32067 condim = values.split(',')[2] 32068 vardimname = values.split(',')[3] 32069 32070 slicedicv = {} 32071 for slcv in slicevS.split(':'): 32072 dimn = slcv.split('|')[0] 32073 slccvv = slcv.split('|')[1] 32074 * [integer]: which value of the dimension 32075 * -1: all along the dimension 32076 * -9: last value of the dimension 32077 * [beg]@[end] slice from [beg] to [end] 32078 if slccvv.count('@') != 0: 32079 slicedicv[dimn] = gen.str_list_k(slccvv,'@','I') 32080 else: 32081 slicedicv[dimn] = int(slccvv) 32082 32083 HMT = ncfile.split(',') 32084 confiles = gen.files_folder_HMT(fold, HMT[0], HMT[1], HMT[2]) 32085 Nfiles = len(confiles) 32086 32087 print ' ' + fname +': concatenating by slice:', slicedicv, ':', Nfiles, 'files' 32088 if Nfiles == 0: 32089 print errormsg 32090 print ' ' +fname+ ": there are no files as '" + fold + '/' + '*'.join(HMT) +\ 32091 "*' !!" 32092 quit(-1) 32093 print confiles 32094 32095 # Opening all files 32096 ncobjs = [] 32097 for filen in confiles: 32098 print 'charging: ',filen 32099 if filen.find(fold) != -1: 32100 ncobjs.append(NetCDFFile(filen, 'r')) 32101 else: 32102 ncobjs.append(NetCDFFile(fold + '/' + filen, 'r')) 32103 32104 # Looking for the new length of the concatenation dimension 32105 if not ncobjs[0].dimensions.has_key(condim): 32106 print errormsg 32107 print ' ' + fname + ": files do not have dimensions '" + condim + "' !!!" 32108 quit(-1) 32109 if vardimname != 'WRFtime' and not ncobjs[0].variables.has_key(vardimname): 32110 print errormsg 32111 print ' ' + fname + ": files do not have variable '" + vardimname + "' !!!" 32112 quit(-1) 32113 32114 totcondim = 0 32115 for ifile in range(Nfiles): 32116 ncobj = ncobjs[ifile] 32117 totcondim = totcondim + len(ncobjs[ifile].dimensions[condim]) 32118 32119 print "total concatenated dimension '" + condim + "': ", totcondim 32120 32121 # concatenated file creation 32122 ## 32123 newnc = NetCDFFile(ofile, 'w') 32124 32125 # dimension creation 32126 ## 32127 if ncobjs[0].dimensions[condim].isunlimited(): 32128 newnc.createDimension(condim, None) 32129 else: 32130 newnc.createDimension(condim, totcondim) 32131 newnc.createDimension('file', Nfiles) 32132 newnc.createDimension('Lstring', 256) 32133 32134 # dim-variables 32135 newvar = newnc.createVariable('file','c',('file','Lstring')) 32136 newvals = writing_str_nc(newvar, confiles, 256) 32137 if confiles[0].find(fold) != -1: 32138 str="concatenated files following dimension '" + condim + "'" 32139 else: 32140 str="concatenated files from folder '" + fold + "' following dimension '" + \ 32141 condim + "'" 32142 vardef = basicvardef(newvar, 'files', str, '-') 32143 32144 # Fake variable for the dimension 32145 print 'condim:',condim 32146 dims = [] 32147 dims.append(condim) 32148 newvar = newnc.createVariable('int' + condim, 'i4', tuple(dims)) 32149 newvar[:] = range(totcondim) 32150 32151 # Dimension variable for the concatenation 32152 32153 urefvals = '-' 32154 if vardimname != 'WRFtime': 32155 odimvar = ncobjs[0].variables[vardimname] 32156 odimvardims = odimvar.dimensions 32157 odimattrs = odimvar.ncattrs() 32158 for dimn in odimvardims: 32159 if not gen.searchInlist(newnc.dimensions,dimn): 32160 idim = gen.index_vec(odimvar.dimensions, dimn) 32161 if ncobjs[0].dimensions[dimn].isunlimited(): 32162 newdim = newnc.createDimension(dimn,None) 32163 else: 32164 Ldim = len(ncobjs[0].dimensions[dimn]) 32165 newdim = newnc.createDimension(dimn,Ldim) 32166 newvar = newnc.createVariable(vardimname, odimvar.dtype, odimvardims) 32167 for attrn in odimattrs: 32168 attrv = odimvar.getncattr(attrn) 32169 newattr = set_attribute(newvar, attrn, attrv) 32170 if attrn == 'units': urefvals = attrv 32171 else: 32172 print infmsg 32173 print ' ' + fname + ": creation of variable 'time' from WRF 'Times' !!" 32174 tvals = [] 32175 for ifile in range(Nfiles): 32176 odimvar = ncobjs[ifile].variables['Times'] 32177 timewrfv = odimvar[:] 32178 tvals0, urefvals = compute_WRFtime(timewrfv, refdate='19491201000000', \ 32179 tunitsval='minutes') 32180 tvals = tvals + list(tvals0) 32181 tvals = np.array(tvals) 32182 dimtwrf = len(tvals) 32183 if not gen.searchInlist(newnc.dimensions,'time'): 32184 newnc.createDimension('time',None) 32185 newvar = newnc.createVariable('time', 'f8', 'time') 32186 newvar[:] = np.zeros((totcondim), dtype=np.float64) 32187 basicvardef(newvar, 'time', 'Time', urefvals) 32188 newvar.setncattr('axis', 'T') 32189 newvar.setncattr('_CoordinateAxisType', 'Time') 32190 set_attribute(newvar, 'calendar', 'standard') 32191 if urefvals == '-': urefvals = '- -' 32192 32193 # Checking units of condimvar 32194 uvalsecs = urefvals.split(' ') 32195 if uvalsecs[1] == 'since': 32196 tunits = True 32197 CFtimeu = gen.get_right_CFtimeunits(urefvals) 32198 uvalsecs = CFtimeu.split(' ') 32199 date=uvalsecs[2] 32200 time=uvalsecs[3] 32201 refdate = gen.datetimeStr_datetime(date+'_'+time) 32202 print warnmsg 32203 print ' ' + fname + ': concatenating dimension-variable has time units!' 32204 print ' shifting temporal-values since the date of the first file', refdate 32205 else: 32206 tunits = False 32207 begslicetot = 0 32208 for ifile in range(Nfiles): 32209 slicevartot = [] 32210 if vardimname != 'WRFtime' and not ncobjs[ifile].variables.has_key(vardimname): 32211 print errormsg 32212 print ' ' + fname + ": file '" + confiles[ifile] + "' does not have " + \ 32213 " dimension-variable '" + vardimname + "' !!" 32214 quit(-1) 32215 if vardimname != 'WRFtime': 32216 objvar = ncobjs[ifile].variables[vardimname] 32217 for dimn in objvar.dimensions: 32218 ldimfile = len(ncobjs[ifile].dimensions[dimn]) 32219 if dimn == condim: 32220 slicevartot.append(slice(begslicetot,begslicetot+ldimfile)) 32221 begslicetot = begslicetot + ldimfile 32222 else: 32223 slicevartot.append(slice(0,ldimfile)) 32224 if tunits: 32225 uvals = objvar.getncattr('units') 32226 #urefvals, objvals = gen.check_timestep(odimvar) 32227 #uvals, objvals = gen.check_timestep(objvar) 32228 newtimes = gen.coincident_CFtimes(objvar[:], urefvals, uvals) 32229 newvar[tuple(slicevartot)] = newtimes[:] 32230 else: 32231 newvar[tuple(slicevartot)] = objvar[:] 32232 newnc.sync() 32233 else: 32234 odimvarf = ncobjs[ifile].variables['Times'] 32235 timewrfvf = odimvarf[:] 32236 tvalsf, utimef = compute_WRFtime(timewrfvf, refdate='19491201000000', tunitsval='minutes') 32237 ldimfile = tvalsf.shape[0] 32238 dimn = 'time' 32239 condimn = 'time' 32240 if dimn == condim or (condim == 'Time' and vardimname == 'WRFtime'): 32241 slicevartot.append(slice(begslicetot,begslicetot+ldimfile)) 32242 begslicetot = begslicetot + ldimfile 32243 else: 32244 slicevartot.append(slice(0,ldimfile)) 32245 if tunits: 32246 uvals = utimef 32247 #urefvals, objvals = gen.check_timestep(odimvar) 32248 #uvals, objvals = gen.check_timestep(objvar) 32249 newtimes = gen.coincident_CFtimes(tvalsf, urefvals, uvals) 32250 newvar[tuple(slicevartot)] = newtimes[:] 32251 newnc.sync() 32252 32253 # Looping variables 32254 ## 32255 if varn == 'all': 32256 desvars = ncobjs[0].variables.keys() 32257 else: 32258 desvars = varn.split(',') 32259 32260 for dvar in desvars: 32261 if not ncobjs[0].variables.has_key(dvar): 32262 print errormsg 32263 print ' ' + fname + ": files do not have variable '" + dvar + "' !!!" 32264 quit(-1) 32265 32266 if gen.searchInlist(desvars, vardimname): desvars.remove(vardimname) 32267 32268 # dictionary with the slice on variable 32269 slicedimv = {} 32270 # list of dimensions to remove due to slice from variable 32271 rmdimv = [] 32272 for dvar in desvars: 32273 print ' ' + fname + ": concatenating '" + dvar + "'..." 32274 objvar = ncobjs[0].variables[dvar] 32275 # creation of dimensions 32276 vardims = objvar.dimensions 32277 lvardims = list(objvar.dimensions) 32278 for ndim in vardims: 32279 if not newnc.dimensions.has_key(ndim): 32280 vardimo = ncobjs[0].dimensions[ndim] 32281 if vardimo.isunlimited(): 32282 newnc.createDimension(ndim, None) 32283 else: 32284 if slicedicv.has_key(ndim): 32285 slcddvv = slicedicv[ndim] 32286 if type(slcddvv) == type([1,2]): 32287 Nvals = np.arange(slcddvv[0], slcddvv[1]+slcddvv[2], \ 32288 slcddvv[2]) 32289 newnc.createDimension(ndim, Nvals) 32290 slicedimv[ndim]= slice(slcddvv[0], slcddvv[1]+slcddvv[2],\ 32291 slcddvv[2]) 32292 else 32293 if slcddv == -1: 32294 newnc.createDimension(ndim, len(vardimo)) 32295 slicedimv[ndim]= slice(0, len(vardimo)) 32296 elif slcddv == -9: 32297 slicedimv[ndim]= len(vardimo)-1 32298 lvardims.remove(ndim) 32299 else: 32300 slicedimv[ndim]= slcddvv 32301 lvardims.remove(ndim) 32302 else: 32303 newnc.createDimension(ndim, len(vardimo)) 32304 # creation of variable 32305 kvar = objvar.dtype 32306 if newnc.variables.has_key(dvar): 32307 print infmsg 32308 print ' ' + fname + ": file already has variable '" + dvar + "' !!" 32309 print " creating a new one called '" + dvar + "2'" 32310 newvar = newnc.createVariable(dvar + '2', kvar, lvardims) 32311 else: 32312 newvar = newnc.createVariable(dvar, kvar, lvardims) 32313 varattrs = objvar.ncattrs() 32314 for atr in varattrs: 32315 attrv = objvar.getncattr(atr) 32316 newattr = set_attribute(newvar, atr, attrv) 32317 32318 # variable values 32319 # vartotv = np.zeros((newvar.shape), dtype=kvar) 32320 objvar = ncobjs[0].variables[dvar] 32321 if gen.searchInlist(list(objvar.dimensions),condim): 32322 begslicetot = 0 32323 for ifile in range(Nfiles): 32324 slicevartot = [] 32325 if ncobjs[ifile].variables.has_key(dvar): 32326 objvar = ncobjs[ifile].variables[dvar] 32327 foundim = False 32328 slcvarif = [] 32329 for dimn in objvar.dimensions: 32330 ldimfile = len(ncobjs[ifile].dimensions[dimn]) 32331 if dimn == condim: 32332 slicevartot.append(slice(begslicetot,begslicetot+ldimfile)) 32333 begslicetot = begslicetot + ldimfile 32334 foundim = True 32335 slicevarif.append(slice(0,ldimfile)) 32336 else: 32337 if slicedimv.has_key(dimn): 32338 slcvvdd = slicedimv[dimn] 32339 if slcvdd != type(int): 32340 Nvals = len(range(slcddvv[0], \ 32341 slcddvv[1]+slcddvv[2], slcddvv[2]) 32342 slicevartot.append(slice(0,Nvals)) 32343 slicevarif.append(slcvvdd) 32344 else: 32345 slicevarif.append(slicedimv[dimn]) 32346 else: 32347 slicevartot.append(slice(0,ldimfile)) 32348 slicevarif.append(slice(0,ldimfile)) 32349 if not foundim: 32350 print 'Dimension ', condim, 'not found!! Assuming 1 value' 32351 slicevartot.append(0) 32352 newvar[tuple(slicevartot)] = objvar[tuple(slicevarif)] 32353 newnc.sync() 32354 else: 32355 print errormsg 32356 print ' ' + fname + ": file '" + fold + '/' + confiles[ifile] + \ 32357 "' does not have variable '" + dvar + "' !!" 32358 quit(-1) 32359 else: 32360 newvar[:] = objvar[:] 32361 newnc.sync() 32362 32363 # newvar[:] = vartotv 32364 32365 gattrs = ncobjs[0].ncattrs() 32366 for attr in gattrs: 32367 attrv = ncobjs[0].getncattr(attr) 32368 newattr = set_attribute(newnc, attr, attrv) 32369 32370 for ifile in range(Nfiles): 32371 ncobjs[ifile].close() 32372 32373 newnc.sync() 32374 add_global_PyNCplot(newnc, main, fname, '1.1') 32375 32376 newnc.close() 32377 32378 print "Successfull creation of concatenated file '" + ofile + "' !!!" 32379 32380 return 32381 32027 32382 #quit() 32028 32383
Note: See TracChangeset
for help on using the changeset viewer.