Context Navigation

source: lmdz_wrf/trunk/tools/model_graphics.py @ 1018

Last change on this file since 1018 was 1015, checked in by lfita, 9 years ago
Working on the changing name of variables and disappearing variables...
File size: 34.5 KB

Line
1	# Python script to generate the files and the plots for a sensitivity study with multiple modle configurations and models
2	# L. Fita
3	# LMD, Jussieu/Palaisseau, France
4	# Script configuration get from ASCII file 'model_graphics.dat'
5
6	import numpy as np
7	import nc_var_tools as ncvar
8	import generic_tools as gen
9	import time as tim
10	# To avoid errors like: '/bin/sh: 1: Syntax error: Bad fd number'
11	# Make sure that /bin/sh directs to bash and not to dash:
12	# http://stackoverflow.com/questions/15809060/sh-syntax-error-bad-fd-number
13	import subprocess as sub
14	import os
15
16	main = 'model_graphics.py'
17
18	errmsg = ncvar.errormsg
19	warnmsg = ncvar.warnmsg
20
21	def scratches(config):
22	""" Function to set-up if it is needed to start from the scratch
23	config: dictionary with the configuration
24	"""
25	fname = 'scratches'
26
27	if config['scratch'] == 'true':
28	scr = True
29	print warnmsg
30	print " " + main + ": starting from the SCRATCH !!"
31	print " 10 seconds left!!"
32	filescr = True
33	figscr = True
34	tim.sleep(10)
35	else:
36	scr = False
37	if config['filescratch'] == 'true':
38	filescr = True
39	print warnmsg
40	print " " + main + ": files starting from the SCRATCH !!"
41	print " 5 seconds left!!"
42	tim.sleep(5)
43	else:
44	filescr = False
45
46	if config['figscratch'] == 'true':
47	figscr = True
48	print warnmsg
49	print " " + main + ": figures starting from the SCRATCH !!"
50	print " 5 seconds left!!"
51	tim.sleep(5)
52	else:
53	figscr = False
54
55	if config['addfiles'] == 'true':
56	addfils = True
57	else:
58	addfils = False
59
60	if config['addfigures'] == 'true':
61	addfigs = True
62	else:
63	addfigs = False
64
65	if config['debug'] == 'true':
66	debug = True
67	else:
68	debug = False
69
70	return scr, filescr, figscr, addfils, addfigs, debug
71
72	def exp_headers(mod,config):
73	""" Function to provide the headers and the experiments of a given model
74	model= model
75	config= configuration of the experiment
76	"""
77	fname = 'exp_headers'
78
79	# No CASE in python!
80	# case ${mod} in ...
81	if mod == 'WRF':
82	expers = config['WRFexps'].split(':')
83	fhs = config['WRFheaders'].split(':')
84	elif mod == 'LMDZ':
85	expers = config['LMDZexps'].split(':')
86	fhs = config['LMDZheaders'].split(':')
87	elif mod == 'WRF_LMDZ':
88	expers = config['WRF_LMDZexps'].split(':')
89	fhs = config['WRF_LMDZheaders'].split(':')
90	else:
91	print errmsg
92	print " " + fname + ": model '" + mod + "' not ready!!"
93	quit(-1)
94
95	return expers, fhs
96
97	class VariableInf(object):
98	""" Class which holds all information of a given variable
99	name= CF name of the variable
100	header= header of the file from which it can be computed
101	model= file-header variable from which it can be directly computed
102	diag= file-header diagnostics from which it can be computed
103	"""
104	def __init__( self, name, fheader, model, diag):
105	self.name= None
106	self.fheader= None
107	self.model= None
108	self.diag= None
109	if name is not None:
110	self.name = name
111	self.fheader= fheader
112	self.model= model
113	self.diag= diag
114
115	def variable_compute(idir,var,ftests,db):
116	""" Function to retrieve the computation way of a given variable using a series of test files
117	iwdir= directory with the test files
118	var= name of the variable to test
119	filtests= dictionary with the test files for each header
120	"""
121	fname='variable_compute'
122
123	cancompute = True
124	for headerf in ftests.keys():
125	filen=ftests[headerf]
126	vmod, vdiag = ncvar.computevar_model(var, idir + '/' + filen)
127
128	if vmod is None and vdiag is None:
129	cancompute = False
130	else:
131	cancompute = True
132	# Should be considered that variable can also be computed by both ways?
133	break
134
135	if not cancompute:
136	print warnmsg
137	print ' ' + fname + ": there is no way to compute '" + var + \
138	"' for model '" + mod
139	# Too extrict!
140	# quit(-1)
141
142	# Getting only the first possibility of model and diagnostic
143	if vmod is not None:
144	modV = vmod[0]
145	else:
146	modV = None
147	if vdiag is not None:
148	diagV = vdiag[0]
149	else:
150	diagV = None
151
152	varcomp = VariableInf(var, headerf, modV, diagV)
153
154	# a ';' list 'varcompute' it is created for each variable giving:
155	# [var]\|[vark]\|[headerf][varmod]\|[vardiag]
156	# This list will be used to compute a new file for each variable
157
158	if db:
159	print 'Variable information _______'
160	gen.printing_class(varcomp)
161
162	return varcomp
163
164	def get_operations_var(vc,db):
165	""" Function to provide the operations to make for each variable from 'VAR_' dictionary
166	vc= 'VAR_' dictionary, dictionary with all the parameters started with 'VAR_'
167	"""
168	fname = 'get_operations_var'
169
170	LH = len('VAR_')
171	iop = 1
172	# list of operations
173	doopers = {}
174	# operations by variable
175	operationsvar = {}
176	# individual operations by variable (not repeated)
177	indivoperationsvar = {}
178
179	# Variables with a calculation whic requires vertical interpolation to all the
180	# file (operations stating by 'VAR_pinterp')
181	LVp = len('VAR_pinterp')
182	varglobalp = []
183
184	for oper in vc.keys():
185	Loper = len(oper)
186	opn = oper[LH:Loper+1]
187	vns = vc[oper].split(':')
188	ops1 = opn.split('+')
189	doopers[opn] = ops1
190	for vn in vns:
191	if not operationsvar.has_key(vn):
192	operationsvar[vn] = [opn]
193	indivoperationsvar[vn] = ops1
194	else:
195	opers = operationsvar[vn]
196	opers.append(opn)
197	operationsvar[vn] = opers
198	opers1 = indivoperationsvar[vn]
199	for op1 in ops1:
200	if not gen.searchInlist(opers1, op1):
201	opers1.append(op1)
202	indivoperationsvar[vn] = opers1
203
204	if oper[0:LVp] == 'VAR_pinterp':
205	for vn in vns:
206	if not gen.searchInlist(varglobalp,vn): varglobalp.append(vn)
207
208	iop = iop + 1
209
210	if db:
211	print ' operations to make _______'
212	gen.printing_dictionary(doopers)
213	print ' operations by variable _______'
214	gen.printing_dictionary(operationsvar)
215	print ' individual operations by variable _______'
216	gen.printing_dictionary(indivoperationsvar)
217	print ' variables with the vertical interpolation for all the file _______'
218	print ' #', varglobalp
219
220	return doopers, operationsvar, indivoperationsvar, varglobalp
221
222	def pinterp_var(oper):
223	""" Function to retrieve characteristics of the vertical interpolation for the operation
224	oper= operation
225	Wether vertical interpolation is:
226	# 'global': for all file
227	# 'local': at a given step of the process
228	# 'none': no vertical interpolation
229	>>> pinterp_var('last+pinterp+xmean')
230	local
231	>>> pinterp_var('pinterp+tmean+xmean')
232	global
233	>>> pinterp_var('xmean')
234	none
235	"""
236	fname = 'pinterp_var'
237
238	pinn = 'pinterp'
239
240	Lpinterp = len(pinn)
241	if oper[0:Lpinterp] == pinn:
242	pkind = 'global'
243	return pkind
244
245	if oper.find(pinn) != -1:
246	pkind = 'local'
247	else:
248	pkind = 'none'
249
250	return pkind
251
252	def compvars_listconstruct(config, minf, Files, TestFiles, idir, odir, debug):
253	""" Function to construct the list of variables to compute
254	config= dictionary with the configuration of the execution
255	variables to compute are get with all values started by `VAR_' in 'module_graphics.dat', and then
256	providing a consecutive number of calculations separated by '+'
257	VAR_[calc1]+[calc2] = tas:wss
258	will compute first [calc1] and then [calc2] for 'tas' and 'wss'
259	minf= class with information about the model
260	Files= dictionary of files for heach header
261	TestFiles= dictionary of files to test how to compute the variable
262	odir= output directory
263	"""
264	fname='compvars_listconstruct'
265
266	varcomp = gen.get_specdictionary_HMT(config,H='VAR_')
267
268	if debug:
269	print ' variables to compute ________'
270	gen.printing_dictionary(varcomp)
271
272	# Getting operations by variable
273	opers, varoper, indivaroper, varglobp = get_operations_var(varcomp,debug)
274
275	strfiles = gen.dictKeysVals_stringList(Files)
276	strtestfiles = gen.dictKeysVals_stringList(TestFiles)
277
278	print 'Lluis' + fname + ': strtestfile:', strtestfiles
279
280	Svarcompute = ''
281
282	# Main dictionary with all the variables and their calculation
283	allvarcomp = {}
284
285	ivop = 0
286	for vn in varoper:
287	vcomp = variable_compute(idir,vn,TestFiles,False)
288	for op in varoper[vn]:
289	# Creation of a String as: [CFname]\|[operation]\|[fheader]\|[vmodel]\|[vdiag]\|[globalP]
290	# [CFname]: CF-name of the variable (must appear in 'variables_values.dat')
291	# [operation]: operation to compute
292	# [fheader]: header of file with the required variables
293	# [vmodel]:
294	# [vdiag]:
295	# [globalP]: Wether vertical interpolation is:
296	# 'global': for all file
297	# 'local': at a given step of the process
298	# 'none': no vertical interpolation
299	globalP = pinterp_var(op)
300	if vcomp.model is not None:
301	Smodel = ':'.join(vcomp.model)
302	else:
303	Smodel = 'None'
304	if vcomp.diag is not None:
305	Sdiag = ':'.join(vcomp.diag)
306	else:
307	Sdiag = 'None'
308
309	Svc = vcomp.name + '\|' + op + '\|' + vcomp.fheader + '\|' + Smodel + '\|' + \
310	Sdiag + '\|' + globalP
311	if ivop == 0:
312	Svarcompute = Svc
313	else:
314	Svarcompute = Svarcompute + ',' + Svc
315
316	allvarcomp[vn + '_' + op] = [vcomp.fheader, vcomp.model, vcomp.diag, globalP]
317	ivop = ivop + 1
318
319	if debug:
320	print ' Variables to compute _______'
321	gen.printing_dictionary(allvarcomp)
322
323	# Outwritting the varcompute to avoid next time (if it is not filescratch!)
324	objf = open(owdir + '/varcompute.inf', 'w')
325	objf.write('files: ' + strfiles + '\n')
326	objf.write('testfiles: ' + strtestfiles + '\n')
327	objf.write('varcompute: ' + Svarcompute + '\n')
328	objf.write('itotv: ' + str(ivop) + '\n')
329	objf.close()
330
331	return allvarcomp, ivop
332
333	def read_varcomp_file(filen):
334	""" Function to read 'varcompute.inf' and reconstruct the dictionaries
335	"""
336	fname = 'read_varcomp_file'
337
338	allvarcomp = {}
339
340	objf = open(filen, 'r')
341
342	for line in objf:
343	vals = line.split(' ')
344	if vals[0] == 'files:':
345	Files = gen.stringList_dictKeysVals(vals[1])
346	elif vals[0] == 'testfiles:':
347	TestFiles = gen.stringList_dictKeysVals(vals[1])
348	elif vals[0] == 'varcompute:':
349	Vvals = vals[1].split('\|')
350	if Vvals[3] == 'None':
351	mod = None
352	else:
353	mod = Vvals[3].split(':')
354	if Vvals[4] == 'None':
355	diag = None
356	else:
357	diag = Vvals[4].split(':')
358
359	allvarcomp[Vvals[0]+'_'+Vvals[1]] = [Vvals[2], mod, diag, Vvals[5]]
360	elif vals[0] == 'itotv:':
361	ivop = int(vals[1])
362
363	objf.close()
364
365	return Files, TestFiles, allvarcomp, ivop
366
367	def compute_variable(minf, idir, usefiles, odir, cvar, gP, scr, pyH, Tref, Tunits, db):
368	""" Function to compute a variable
369	minf= class with the information of the model
370	idir= directory with the input files
371	usefiles= dictionary of files as dict([headerf]) = [file1], ..., [fileN]
372	odir= directory to write the output files
373	cvar= class with the information of the variable: 'name', 'fheader', 'varmod', 'vardiag'
374	gP= kind of vertical interpolation ('global', 'local', 'none')
375	scr= should it be done from the scratch?
376	pyH= location of the python HOME
377	Tref= CF time reference
378	Tunits= CF time units
379	"""
380	fname='compute_variable'
381
382	CFvarn=cvar.name
383	headerf = cvar.fheader
384	modvar = cvar.model
385	diagvar = cvar.diag
386
387	cfiles = usefiles[headerf]
388
389	# dimensions
390	dnx = minf.dimxn
391	dny = minf.dimyn
392	# var-dimensions
393	vdnx = minf.vardxn
394	vdny = minf.vardyn
395
396	# Computing separately and then joinging for all files
397	Ntotfiles = len(cfiles)
398	Nzeros = len(str(Ntotfiles))
399	NStot = str(Ntotfiles).zfill(Nzeros)
400
401	# For that variables which require vertical interpolation 'p' suffix to the
402	# file header is added
403	if gP != 'none':
404	SgP = 'p'
405	else:
406	SgP = ''
407
408	# Getting in working dir
409	os.chdir(odir)
410
411	# File to keep track of all operations
412	otrackf = open( odir + '/all_computevars.inf', 'a')
413
414	# Computing variable
415	ifile=1
416	for cf in cfiles:
417	ifS = str(ifile).zfill(Nzeros)
418	ifilen = odir + '/' + CFvarn + '_' + headerf + SgP + '_' + ifS + '-' + \
419	NStot + '.nc'
420	fileon = odir + '/' + CFvarn + '_' + headerf + SgP + '.nc'
421
422	if scr:
423	sout = sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
424	sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
425
426	if not os.path.isfile(ifilen) and not os.path.isfile(fileon):
427	# Since model direct values are retrieved from `variables_valules.dat' which was
428	# initially coincived as a way to only give variable attributes, range and color
429	# bars, if a variable has a diagnostic way to be computed, the later one will be
430	# preferred
431	if db:
432	print ' ' + fname + ": creation of variable file '" + CFvarn + \
433	"' in file '" + ifilen + "' ..."
434
435	if modvar is not None and diagvar is None:
436	# model variable
437	values = modvar + ',0,-1,-1'
438	vs = modvar + ',' + vdnx + ',' + vdny + ',' + vdnz + ',' + vdnt
439	with gen.Capturing() as output:
440	ncvar.DataSetSection_multivars(values, cf, vs)
441
442	newfile, loc = gen.search_sec_list(output,'succesfull')
443	ofile = newfile[0].split(' ')[7]
444	sub.call('mv ' + ofile + ' ' + ifilen, shell=True)
445
446	# Keeping track of the operations
447	pyins = pyH + "/nc_var.py -f "+cf+" -o DataSetSection_multivars -v "+\
448	vs + " -S '" + values + "'"
449	otrackf.write('\n')
450	otrackf.write('# ' + CFvarn + " " + modvar + '\n')
451	otrackf.write(pyins + '\n')
452
453	# CF renaming of variable
454	ncvar.chvarname(CFvarn,ifilen,modvar)
455	else:
456	# diagnostic variable
457	dims = dnt+'@'+vdnt+','+dnz+'@'+vdnz+','+dny+'@'+vdny+','+dnx+'@'+vdnx
458	diagn = diagvar[0]
459	Ndiagvars = len(diagvar)
460	diagc = '@'.join(diagvar[1:])
461
462	values = '-f ' + cf + " -d '" + dims + "' -v '" + diagn + '\|' + \
463	diagc + "'"
464	sout = sub.call('python ' + pyH + '/diagnostics.py ' + values, \
465	shell=True)
466	sout = sub.call(' mv diagnostics.nc ' + ifilen, shell=True)
467
468	# Keeping track of the operations
469	pyins = 'python ' + pyH + '/diagnostics.py ' + values
470	otrackf.write('\n')
471	otrackf.write('# ' + CFvarn + " " + diagn + '\n')
472	otrackf.write(pyins + '\n')
473
474	# Attaching necessary variables for the pressure interpolation
475	if gP != 'none':
476	requiredinterpvars = ['P', 'PB', 'PSFC', 'PH', 'PHB', 'HGT', 'T', \
477	'QVAPOR', 'XLONG', 'XLAT', 'Times']
478	print " " + fname + ": adding variables:", requiredinterpvars, \
479	' to allow pressure interpolation'
480	for rqv in requiredinterpvars:
481	ncvar.fvaradd(idir+'/'+cf+','+rqv,ifilen)
482
483	# adding CF lon,lat,time in WRF files
484	if minf.name == 'WRF':
485	values = vdnx + ':' + vdny + ':'+ Tref + ':' + Tunits
486	ncvar.WRF_toCF(values, ifilen)
487
488	ifile = ifile + 1
489
490	otrackf.close()
491
492	# Joining variable files
493	if not os.path.isfile(fileon):
494	if db:
495	print ' ' + fname + ": concatenating all variable files to create '" + \
496	fileon + "..."
497	ncvar.netcdf_fold_concatenation_HMT('./,time', CFvarn+'_'+headerf+SgP+ \
498	'_,-,.nc', 'all')
499	sout = sub.call('mv netcdf_fold_concatenated_HMT.nc ' + fileon, shell=True)
500	if os.path.isfile(fileon):
501	sout = sub.call('rm ' + CFvarn + '_' + headerf + '_-.nc', shell=True)
502
503	return
504
505	def compute_statistics(minf, config, idir, usefiles, odir, cvar, gP, Opers, scr, db):
506	""" Function to compute different statistics it will take previous steps if they
507	are availale
508	minf= class with the information of the model
509	config= dictionary with the configuration of the experiment
510	idir= directory with the input files
511	usefiles= ',' list of files to use [file1],...,[fileN]
512	odir= directory to write the output files
513	cvar= class with the information of the variable: 'name', 'fheader', 'varmod', 'vardiag'
514	gP= kind of vertical interpolation ('global', 'local', 'none')
515	Opers= kind of operation: (as possible multiple consecutive combination of operations separated by '+'
516	[calc1]+[calc2] will compute first [calc1] and then [calc2]
517	acc: temporal accumulated values
518	diff: differences between models
519	direct: no statistics
520	last: last temporal value
521	Lmean: latitudinal mean values
522	Lsec: latitudinal section (latitudinal value must be given, [var]@[lat])
523	lmean: longitudinal mean values
524	lsec: longitudinal section (longitudinal value must be given, [var]@[lat])
525	pinterp: pressure interpolation (to the given $plevels)
526	tmean: temporal mean values
527	turb: Taylor's turbulence decomposition value
528	xmean: x-axis mean values
529	ymean: y-axis mean values
530	zsum: vertical aggregated values
531	scr= should it be done from the scratch?
532	"""
533	fname='compute_statistics'
534
535	# Getting a previous file name to continue with(for combinations)
536	CFvarn=cvar.name
537	headerf = cvar.fheader
538	modvar = cvar.model
539	diagvar = cvar.diag
540
541	cfiles = usefiles[headerf]
542
543	# dimensions
544	dnx = minf.dimxn
545	dny = minf.dimyn
546	# var-dimensions
547	vdnx = minf.vardxn
548	vdny = minf.vardyn
549
550	# Some experiment configuration values
551	# plevels= ':' separated list of pressures (in Pa) to use for the vertical interpolation
552	plevels = config['plevels']
553	# pyH= location of the python HOME
554	pyH = config['pyHOME']
555	# Tref= CF time reference
556	Tref = config['CFreftime']
557	# Tunits= CF time units
558	Tunits = config['CFunitstime']
559	# opsur = operation surnames
560	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
561	Opsurs = {}
562	for opk in opsur:
563	opn = opk.split('_')[1]
564	vals = opsur[opk]
565	Opsurs[opn] = vals.split(':')
566	if db:
567	print ' ' + fname + ' operation surnames _______'
568	gen.printing_dictionary(Opsurs)
569
570	# Getting in working dir
571	os.chdir(odir)
572
573	# File to keep track of all operations
574	otrackf = open( odir + '/all_statsvars.inf', 'a')
575
576	# For that variables which require vertical interpolation 'p' suffix to the file
577	# header is added
578	if gP != 'none':
579	SgP = 'p'
580	varnCFs = ['lon', 'lat', 'pres', 'time']
581	else:
582	SgP = ''
583	varnCFs = ['lon', 'lat', 'time']
584
585	# Mandatory CF variables to be in all files
586	SvarnCFs = ',' + ','.join(varnCFs)
587
588	# Input file
589	ifilen = odir + '/' + CFvarn + '_' + headerf + SgP + '.nc'
590
591	# Computing variable statisitcs
592	istats=0
593
594	# List of not computed statistics
595	wrongstats = []
596
597	# Variables to be kept in the final file
598	varkeep = []
599
600	opers = Opers.split('+')
601	Fopers = ''
602	if db: print ' computing statistics of variable:', CFvarn, ' ...'
603	for op in opers:
604
605	# File name from previous operations
606	if op == opers[0]:
607	Fopers = op
608	prevfile = ifilen
609	vninF = CFvarn
610	else:
611	Fopers = Fopers + '_' + op
612	prevfile = fileon
613	fileon = odir + '/' + CFvarn + '_' + headerf + SgP + '_' + Fopers + '.nc'
614
615	# Adding required variables for the vertical interpolation in all operations
616	if gP != 'none':
617	if gP == 'local':
618	CFvarnp = vninF + ',P,PB,PSFC,PH,PHB,HGT,T,QVAPOR,XLONG,XLAT,Times' + SvarnCFs
619	else:
620	CFvarnp = vninF + SvarnCFs
621	else:
622	CFvarnp = vninF + SvarnCFs
623
624	if scr:
625	sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
626
627	hprevfile = prevfile[0:len(prevfile)-3]
628	if not os.path.isfile(fileon):
629	if db: print ' ', op, "using '" + prevfile + "' "
630
631	if op == 'acc':
632	# temporal accumulated values
633	print " " + fname + ": kind '" + op + "' not ready !!"
634	wrongstats.appen(CFvarn + '_' + opers)
635	break
636	elif op == 'direct':
637	# no statistics
638	sout = sub.call('mv ' + prevfile + ' ' + fileon, shell=True)
639	elif op == 'last':
640	# last temporal value
641	vals='time,-9,0,0'
642	with gen.Capturing() as output:
643	ncvar.DataSetSection(vals,prevfile)
644
645	sout = sub.call('mv ' + hprevfile + '_time_B-9-E0-I0.nc ' + fileon, \
646	shell=True)
647
648	# Keeping the operations
649	pyins=pyH + "/nc_var.py -o DataSetSection -S '" + vals + "' -f " + \
650	prevfile
651	otrackf.write("\n")
652	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
653	otrackf.write(pyins + "\n")
654	elif op =='Lmean':
655	# latitudinal mean values
656	print " " + fname + ": kind '" + op + "' not ready !!"
657	wrongstats.appen(CFvarn + '_' + opers)
658	break
659	elif op =='Lsec':
660	# latitudinal section (latitudinal value must be given, [var]@[lat])
661	print " " + fname + ": kind '" + op + "' not ready !!"
662	wrongstats.appen(CFvarn + '_' + opers)
663	break
664	elif op =='lmean':
665	# longitudinal mean values
666	print " " + fname + ": kind '" + op + "' not ready !!"
667	wrongstats.appen(CFvarn + '_' + opers)
668	break
669	elif op =='lsec':
670	# longitudinal section (longitudinal value must be given, [var]@[lon])
671	print " " + fname + ": kind '" + op + "' not ready !!"
672	wrongstats.appen(CFvarn + '_' + opers)
673	break
674	elif op == 'pinterp':
675	# pinterp: pressure interpolation (to the given $plevels)
676	vals=plevels + ',1,1'
677	ncvar.pinterp(vals,prevfile,vninF)
678	sout = sub.call('mv pinterp.nc ' + fileon, shell=True)
679
680	# Keeping the operations
681	pyins=pyH + "/nc_var.py -o pinterp -S '" + vals + "' -f " + \
682	prevfile + "-v " + vninF
683	otrackf.write("\n")
684	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
685	otrackf.write(pyins + "\n")
686
687	# adding CF lon,lat,time in WRF files
688	if minf.name == 'WRF':
689	values = vdnx + ':' + vdny + ':'+ Tref + ':' + Tunits
690	ncvar.WRF_toCF(values, fileon)
691
692	# vertical interpolation variables are no more needed
693	CFvarnp = vninF
694	gP = 'none'
695
696	elif op == 'tmean':
697	# temporal mean values
698	vals='time\|-1,time,mean,lon:lat:' + vdnz + ':time:pres'
699	dims='time@time,' + dnz + '@' + vdnz + ',lat@lat,lon@lon'
700	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
701	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
702
703	# Keeping the operations
704	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
705	"' -f " + prevfile + " -v " + CFvarnp
706	otrackf.write("\n")
707	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
708	otrackf.write(pyins + "\n")
709
710	# lon, lat CF-variables do not have time, thus they are not kept in
711	# output
712	for rqv in varnCFs:
713	if rqv != 'time':
714	ncvar.fvaradd(prevfile+','+rqv,fileon)
715	varkeep.append('timestats')
716
717	# elif op == 'turb':
718	# # turbulence values
719	# vals='turbulence\|'${CFvarn}
720	# dims='time@time,'${dnz}'@'${vdnz}',lat@lat,lon@lon,pres@pres'
721	# pyout=`python ${pyHOME}/diagnostics.py -d ${dims} -v ${vals} -f ${cfiles}`
722
723	# # Keeping the operations
724	# pyins="python "${pyHOME}"/diagnostics.py -d "${dims}" -v '"${vals}
725	# pyins=${pyins}"' -f ${cfiles}"
726	# echo " " >> ${odir}/all_statsvars.inf
727	# echo "# ${CFvarn}" "${vark}" >> ${odir}/all_statsvars.inf
728	# echo ${pyins} >> ${odir}/all_statsvars.inf
729
730	# varkeep=':'${CFvarn}'turb'
731
732	elif op == 'xmean':
733	# x-axis mean values
734	vals='lon\|-1,lon,mean,lon:lat:' + vdnz + ':time:pres'
735	dims='time@time,' + dnz + '@' + vdnz + ',lat@lat,lon@lon'
736	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
737	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
738
739	# Keeping the operations
740	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
741	"' -f " + prevfile + " -v " + CFvarnp
742	otrackf.write("\n")
743	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
744	otrackf.write(pyins + "\n")
745
746	# time CF-variable do not have 'x', thus they are not kept in
747	# output
748	rqv = 'time'
749	ncvar.fvaradd(prevfile+','+rqv,fileon)
750	varkeep.append('lonstats')
751
752	elif op == 'ymean':
753	# y-axis mean values
754	vals='lat\|-1,lat,mean,lon:lat:' + vdnz + ':time:pres'
755	dims='time@time,' + dnz + '@' + vdnz + ',lat@lat,lon@lon'
756	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
757	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
758
759	# Keeping the operations
760	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
761	"' -f " + prevfile + " -v " + CFvarnp
762	otrackf.write("\n")
763	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
764	otrackf.write(pyins + "\n")
765
766	# time CF-variable do not have 'y', thus they are not kept in
767	# output
768	rqv = 'time'
769	ncvar.fvaradd(prevfile+','+rqv,fileon)
770	varkeep.append('latstats')
771	elif op == 'zsum':
772	# vertical aggregated values
773	print " " + fname + ": kind '" + op + "' not ready !!"
774	wrongstats.append(CFvarn + '_' + opers)
775	break
776	else:
777	print errmsg
778	print ' ' + fname + ": operation '" + op + "' not ready !!"
779	quit(-1)
780
781	# End of kind of operation
782
783	# Variable name in file (vninF) changed due to operation
784	# but only if previous operation does not the same 'statistic'
785	chvn = gen.dictionary_key_list(Opsurs, op)
786	if chvn is not None:
787	oldvninF = vninF
788	if vninF.find(chvn) != -1:
789	Lchvn = len(chvn)
790	LvninF = len(vninF)
791	if vninF[LvninF-Lchvn:LvninF-Lchvn+1] != chvn:
792	vninF = vninF + chvn
793	else:
794	vninF = vninF + chvn
795	CFvarnp = CFvarnp.replace(oldvninF,vninF)
796
797	if len(varkeep) > 0:
798	varkeepS = ',' + ','.join(varkeep)
799	else:
800	varkeepS = ''
801
802	totalvarkeeps = CFvarnp + ',' + ','.join(varnCFs) + varkeepS
803	oclean = ncvar.cleaning_varsfile(totalvarkeeps,fileon)
804
805	# End of operations
806	if len(wrongstats) > 1:
807	print warnmsg
808	print ' ' + fname + ": statisitcs not possible to compute:", wrongstats
809
810	return
811
812	def compute_vars(config, modinf, idir, odir, Files, allvarcomp, fscr, debug):
813	""" Function to compute the variables
814	config= Configuration of the experiment
815	modinf= class with information about the model
816	idir= input experiment folder
817	odir= output experiment folder
818	Files= dictionary with the header and the files' correspondence
819	allvarcomp= dictionary with all the variables to compute and their information
820	fscr= whether files should be done from the scratch or not
821	"""
822	fname = 'compute_vars'
823
824	for vopn in allvarcomp:
825	# variable & operation
826	vn = vopn.split('_')[0]
827	op = vopn.split('_')[1]
828	vopnV = allvarcomp[vopn]
829	fheader = vopnV[0]
830	model = vopnV[1]
831	diag = vopnV[2]
832	globalP = vopnV[3]
833
834	if debug:
835	print ' ' + vn + ': ' + op
836
837	# Computing CF variable
838	if model is not None or diag is not None:
839	vinf = VariableInf(vn,fheader,model,diag)
840	# Comppute variable
841	compute_variable(modinf, idir, Files, odir, vinf, globalP, fscr, \
842	config['pyHOME'], config['CFreftime'], config['CFunitstime'], debug)
843
844	# Compute variable statistics
845	compute_statistics(modinf, cnf, iwdir, Files, owdir, vinf, globalP, op, \
846	fscr, debug)
847
848	else:
849	print errmsg
850	print ' ' + fname + ": neither 'model' or 'diag' variables for '" + vn \
851	+ "' !!"
852	quit(-1)
853
854	return
855
856	# Files with information about the configuration of the script
857	inffiles = ['varcompute.inf', 'all_computevars.inf', 'all_statsvars.inf']
858
859	####### #######
860	## MAIN
861	#######
862
863	# Getting configuration from external ASCII file 'model_graphics.dat'
864	cnf = gen.get_configuration('model_graphics.dat', False)
865
866	# scratches
867	scratch, filescratch, figscratch, addfiles, addfigures, dbg = scratches(cnf)
868
869	# Getting models
870	mods = cnf['models'].split(':')
871
872	# Models loop
873	##
874	for mod in mods:
875	print mod
876	# Get experiments and headers of model
877	exps, fheaders = exp_headers(mod,cnf)
878
879	# Characteristics of the model
880	Modinf = ncvar.model_characteristics(mod,'None','False')
881	dnx = Modinf.dimxn
882	dny = Modinf.dimyn
883	dnz = Modinf.dimzn
884	dnt = Modinf.dimtn
885	vdnx = Modinf.vardxn
886	vdny = Modinf.vardyn
887	vdnz = Modinf.vardzn
888	vdnt = Modinf.vardtn
889
890	if dbg:
891	print ' model characteristics _______'
892	print " dims:", dnx, dny, dnz, dnt
893	print " var dims:", vdnx, vdny, vdnz, vdnt
894
895	moddims = dnx + ',' + dny + ',' + dnz + ',' + dnt
896	modvdims = vdnx + ',' + vdny + ',' + vdnz + ',' + vdnt
897
898	# Experiments loop
899	##
900	for exp in exps:
901	print ' ' + exp + '...'
902
903	# input folder
904	iwdir = cnf['ifold'] + '/' + mod + '/' + exp
905
906	# Does input folder exist?
907	if not os.path.isdir(iwdir):
908	print errmsg
909	print " " + main + ": folder '" + iwdir + "' does not exist !!"
910	quit(-1)
911
912	owdir = cnf['ofold'] + '/' + mod + '/' + exp
913	sout = sub.call('mkdir -p ' + owdir, shell=True)
914
915	# Need to pass to analyze all the data?
916	if filescratch:
917	for inff in inffiles:
918	if dbg: print " removing information file '" + inff + "' ..."
919	ins = 'rm ' + owdir + '/' + inff + ' >& /dev/null'
920	sout = sub.call(ins, shell=True)
921
922	objf = open(owdir+'/all_computevars.inf','w')
923	objf.write("## Computation of variables \n")
924	objf.close()
925	objf = open(owdir+'/all_statsvars.inf','w')
926	objf.write("## Computation of statistics \n")
927	objf.close()
928
929	if addfiles:
930	sub.call('rm ' + owdir +'/varcompute.inf >& /dev/null', shell=True)
931
932	varcompf = owdir + '/varcompute.inf'
933	if not os.path.isfile(varcompf):
934	# Does input folder has header files?
935	ih=1
936	# Dictionary with the list of files for each headers
937	files = {}
938	# Dictionary with a file fromor each headers
939	testfiles = {}
940
941	for fh in fheaders:
942	if filescratch:
943	ins = 'rm '+ owdir+'/_' + fh + '.nc >& /dev/null'
944	sout = sub.call(ins, shell=True)
945	files1h = gen.files_folder(iwdir,fh)
946	if len(files1h) < 1:
947	print errmsg
948	print ' ' + main + ": folder '" + iwdir + "' does not " + \
949	"contain files '" + fh + "*' !!"
950	quit(-1)
951	files[fh] = files1h
952	testfiles[fh] = files1h[0]
953
954	if dbg:
955	print ' Dictionary of files _______'
956	gen.printing_dictionary(files)
957
958	print ' Dictionary of test-files _______'
959	gen.printing_dictionary(testfiles)
960
961	allcompvar, Nvar = compvars_listconstruct(cnf, Modinf, files, testfiles, \
962	iwdir, owdir, dbg)
963	else:
964	print warnmsg
965	print ' ' + main + ": getting variables to compute already from file !!"
966	files, testfiles, allcompvar, Nvar = read_varcomp_file(owdir + \
967	'/varcompute.inf')
968	# End of avoiding to repeat all the experiment search
969
970	print " For experiment '"+exp+"' is required to compute:", Nvar, "variables"
971
972	###
973	# Computing variables
974	###
975	print " Computing variables ..."
976	compute_vars(cnf, Modinf, iwdir, owdir, files, allcompvar, filescratch, dbg)
977
978	quit()
979
980	# end of experiments loop
981	# end of mods loop
982

Note: See TracBrowser for help on using the repository browser.

Download in other formats: