Context Navigation

source: lmdz_wrf/trunk/tools/model_graphics.py @ 1214

Last change on this file since 1214 was 1194, checked in by lfita, 9 years ago
Fixing typo
File size: 181.8 KB

Line
1	# Python script to generate the files and the plots for a sensitivity study with multiple modle configurations and models
2	# L. Fita
3	# LMD, Jussieu/Palaisseau, France
4	# Script configuration get from ASCII file 'model_graphics.dat'
5
6	import numpy as np
7	import nc_var_tools as ncvar
8	import generic_tools as gen
9	import drawing_tools as drw
10	import time as tim
11	# To avoid errors like: '/bin/sh: 1: Syntax error: Bad fd number'
12	# Make sure that /bin/sh directs to bash and not to dash:
13	# http://stackoverflow.com/questions/15809060/sh-syntax-error-bad-fd-number
14	import subprocess as sub
15	import os
16
17	main = 'model_graphics.py'
18
19	errmsg = ncvar.errormsg
20	warnmsg = ncvar.warnmsg
21
22	def verify_configuration(ExpConf, debug):
23	""" Function to verify mandatory keys from experiment configuration
24	ExpConf= dictionary with the configuration
25	"""
26	fname = 'verify_configuration'
27
28	# List with the mandatory keys
29	mandatorykeys = ['pyHOME', 'cdoHOME', 'scratch', 'filescratch', 'figscratch', \
30	'diffscratch', 'figdiffscratch', 'figallmodexpscratch', 'addfiles', \
31	'addfigures', 'adddiffs', 'adddifffigures', 'addallmodexpfigures', 'debug', \
32	'models', 'modgraphchar', 'expgraphchar', 'ifold', 'ofold', 'warnmsg', \
33	'errmsg', 'titleoperations', 'kindfig', 'CFreftime', 'CFunitstime', 'mapval', \
34	'timekind', 'timefmt', 'timelabel' ]
35
36	# List with the optional keys
37	optionalkeys = ['specificvarplot', 'specificdiffopplot', 'specificdiffvarplot', \
38	'RefProj']
39
40	# Dictionary with the mandatory keys which are required as function of a given
41	# key from the dictionary
42	# if [key] must exist [keyB] = [values]
43	ifkeys = {'WRFexps': ['models','WRF'], 'LMDZexps': ['models','LMDZ'], \
44	'WRF_LMDZexps': ['models','WRF_LMDZ'], 'WRFheaders': ['models','WRF'], \
45	'WRF_LMDZheaders': ['models','WRF_LMDZ'], 'LMDZheaders': ['models','LMDZ']}
46
47	# Dictionary with the optional keys (at least one) which are required as function
48	# of a given key from the dictionary
49	# if [key] must exist at least one of the [keyA, keyB, keyC, ...] with values
50	ifopkeys = {'RefProj': ['reprojectvar_remapbil', 'reprojectvar_remapbic', \
51	'reprojectvar_remapdis', 'reprojectvar_remapnn', 'reprojectvar_remapcon', \
52	'reprojectvar_remapcon2', 'reprojectvar_remaplaf', 'reprojectvar_dis', \
53	'reprojecvar_dis']}
54
55	for kdict in mandatorykeys:
56	if not ExpConf.has_key(kdict):
57	print errmsg
58	print ' ' + fname + "' configuration without required '" + kdict + "' !!"
59	quit(-1)
60	if debug:
61	print ExpConf[kdict]
62
63	for kdict in ifkeys.keys():
64	vals = ifkeys[kdict]
65	keyn = vals[0]
66	keyv = vals[1]
67	if type(ExpConf[keyn]) == type('A'):
68	if not ExpConf.has_key(kdict) and ExpConf[keyn] == keyv:
69	print errmsg
70	print ' ' + fname + "' configuration without '" + kdict + \
71	"' when it is required with configuration '" + keyn + '=' + keyv + \
72	"' !!"
73	quit(-1)
74	elif type(ExpConf[keyn]) == type(['A', 'B']):
75	keyvals = ExpConf[keyn]
76	if not ExpConf.has_key(kdict) and gen.searchInlist(keyvals, keyv):
77	print errmsg
78	print ' ' + fname + "' configuration without '" + kdict + \
79	"' when it is required with configuration '" + keyn + '= [..., ' + \
80	keyv + ", ...]' !!"
81	quit(-1)
82	else:
83	print errmsg
84	print ' ' +fname+ 'dictionary type ', type(ExpConf[keyn]), ' not ready!!'
85	quit(-1)
86	if debug:
87	print ExpConf[kdict]
88
89	for kdict in ifopkeys.keys():
90	if ExpConf.has_key(kdict):
91	opkeys = ifopkeys[kdict]
92	hasopkey = False
93	opdictvals = {}
94	for opkv in opkeys:
95	if ExpConf.has_key(opkv):
96	hasopkey = True
97	opdictvals[opkv] = ExpConf[opkv]
98
99	if not hasopkey:
100	print errmsg
101	print ' ' + fname + "' configuration wit '" + kdict + "' but " + \
102	"without any key value:", opkeys, "' when it is required !!"
103	quit(-1)
104
105	if debug:
106	print " Optional key '" + kdict + "' with values ________"
107	gen.printing_dictionary(opdictvals)
108
109	print fname + ': configuration seems to be fine'
110
111	return
112
113	def scratches(config):
114	""" Function to set-up if it is needed to start from the scratch
115	config: dictionary with the configuration
116	"""
117	fname = 'scratches'
118
119	if config['scratch'] == 'true':
120	scr = True
121	print warnmsg
122	print " " + fname + ": starting from the SCRATCH !!"
123	print " 10 seconds left!!"
124	filescr = True
125	figscr = True
126	difscr = True
127	figdifscr = True
128	moddifscr = True
129	figmoddifscr = True
130	figallmodexpscr = True
131	tim.sleep(10)
132	else:
133	scr = False
134	if config['filescratch'] == 'true':
135	filescr = True
136	print warnmsg
137	print " " + main + ": files starting from the SCRATCH !!"
138	print " 5 seconds left!!"
139	tim.sleep(5)
140	else:
141	filescr = False
142
143	if config['figscratch'] == 'true':
144	figscr = True
145	print warnmsg
146	print " " + main + ": figures starting from the SCRATCH !!"
147	print " 5 seconds left!!"
148	tim.sleep(5)
149	else:
150	figscr = False
151
152	if config['diffscratch'] == 'true':
153	difscr = True
154	print warnmsg
155	print " " + main + ": experiment differences starting from the SCRATCH !!"
156	print " 5 seconds left!!"
157	tim.sleep(5)
158	else:
159	difscr = False
160
161	if config['figdiffscratch'] == 'true':
162	figdifscr = True
163	print warnmsg
164	print " " + main + ": figures experiment differences starting from the SCRATCH !!"
165	print " 5 seconds left!!"
166	tim.sleep(5)
167	else:
168	figdifscr = False
169
170	if config['moddiffscratch'] == 'true':
171	moddifscr = True
172	print warnmsg
173	print " " + main + ": model differences starting from the SCRATCH !!"
174	print " 5 seconds left!!"
175	tim.sleep(5)
176	else:
177	moddifscr = False
178
179	if config['figmoddiffscratch'] == 'true':
180	figmoddifscr = True
181	print warnmsg
182	print " " + main + ": figures model differences starting from the SCRATCH !!"
183	print " 5 seconds left!!"
184	tim.sleep(5)
185	else:
186	figmoddifscr = False
187
188	if config['figallmodexpscratch'] == 'true':
189	figallmodexpscr = True
190	print warnmsg
191	print " " + main + ": figures all model-experiment starting from the SCRATCH !!"
192	print " 5 seconds left!!"
193	tim.sleep(5)
194	else:
195	figallmodexpscr = False
196
197	if config['addall'] == 'true':
198	print warnmsg
199	print " " + fname + ": adding new values everywhere !!"
200	addfils = True
201	addfigs = True
202	adddiffs = True
203	adddifffigs = True
204	addmoddiffs = True
205	addmoddifffigs = True
206	addallmodexpfigs = True
207	else:
208	if config['addfiles'] == 'true':
209	addfils = True
210	else:
211	addfils = False
212
213	if config['addfigures'] == 'true':
214	addfigs = True
215	else:
216	addfigs = False
217
218	if config['adddiffs'] == 'true':
219	adddiffs = True
220	else:
221	adddiffs = False
222
223	if config['adddifffigures'] == 'true':
224	adddifffigs = True
225	else:
226	adddifffigs = False
227
228	if config['addmoddiffs'] == 'true':
229	addmoddiffs = True
230	else:
231	addmoddiffs = False
232
233	if config['addmoddifffigures'] == 'true':
234	addmoddifffigs = True
235	else:
236	addmoddifffigs = False
237
238	if config['addallmodexpfigures'] == 'true':
239	addallmodexpfigs = True
240	else:
241	addallmodexpfigs = False
242
243	if config['debug'] == 'true':
244	debug = True
245	else:
246	debug = False
247
248	return scr, filescr, figscr, difscr, figdifscr, moddifscr, figmoddifscr, \
249	figallmodexpscr, addfils, addfigs, adddiffs, adddifffigs, addmoddiffs, \
250	addmoddifffigs, addallmodexpfigs, debug
251
252
253	class gc(object):
254	""" Class which holds all the graphical characteristics
255	[label]: label as it should appear in the graphics
256	[color]: specific color of the model
257	[linetype]: specific type of line of the model
258	[marker]: specific marker of the model
259	[sizes]: line width and point size for the model
260	[tmod]: time modification to apply to the model files (None for nothing)
261	'setorigin',[YYYYMMDDHHMISS]: re-set origin of times at [YYYYMMDDHHMISS]
262	"""
263	def __init__( self, label, color, linetype, marker, sizes, Tmod):
264	self.label = None
265	self.color = None
266	self.linetype = None
267	self.marker = None
268	self.sizes = None
269	self.tmod = None
270	if label is not None:
271	self.label = label
272	self.color = color
273	self.linetype = linetype
274	self.marker = marker
275	self.sizes = sizes
276	if Tmod != 'None':
277	self.tmod = Tmod
278
279	def graphical_characteristics(config, debug):
280	""" Function to provide the graphical characteristics of models and experiments
281	config= configuration of the experiment
282	"""
283	fname = 'graphical_characteristics'
284
285	modgraphchar = {}
286	modvals = config['modgraphchar'].split(':')
287	for modval in modvals:
288	modv = modval.split('\|')
289	modgraphchar[modv[0]] = gc(modv[1], modv[2], modv[3], modv[4], modv[5], modv[6])
290
291	expgraphchar = {}
292	expvals = config['expgraphchar'].split(':')
293	for expval in expvals:
294	expvs = expval.split('\|')
295	modexp = expvs[0] + '/' + expvs[1]
296	if not modgraphchar.has_key(expvs[0]):
297	print errmsg
298	print ' ' + fname + ": a model called '" +expvs[0]+ "' does not exist !!"
299	print ' existing models with graphic characteristics:', \
300	modgraphchar.keys()
301	quit(-1)
302
303	modvs = modgraphchar[expvs[0]]
304	expv = expvs[2:7]
305	if expv[1] == 'asmodel': expv[1] = modvs.color
306	if expv[2] == 'asmodel': expv[2] = modvs.linetype
307	if expv[3] == 'asmodel': expv[3] = modvs.marker
308	if expv[4] == 'asmodel': expv[4] = modvs.sizes
309
310	expgraphchar[modexp] = gc(expv[0], expv[1], expv[2], expv[3], expv[4], 'None')
311
312	# Running some tests
313	mods = config['models'].split(':')
314	for mod in mods:
315	if not modgraphchar.has_key(mod):
316	print errmsg
317	print ' ' + fname + ": model called '" + mod + "' does not have " + \
318	"graphical characteristics !!"
319	print ' existing models with graphic characteristics:', \
320	modgraphchar.keys()
321	quit(-1)
322
323	exps = config[mod + 'exps'].split(':')
324	for exp in exps:
325	modexp = mod + '/' + exp
326	if not expgraphchar.has_key(modexp):
327	print errmsg
328	print ' ' + fname + ": model/experiment called '" + modexp + \
329	"' does not have graphical characteristics !!"
330	print ' existing model/experiments with graphic ' + \
331	'characteristics:', expgraphchar.keys()
332	quit(-1)
333
334	if debug:
335	print ' ' + fname + ': model graphical characteristics _______'
336	for modn in modgraphchar.keys():
337	with gen.Capturing() as output:
338	gen.printing_class(modgraphchar[modn])
339	print "'" + modn + "'; " + ', '.join(output)
340	print ' ' + fname + ': experiment graphical characteristics _______'
341	for modexpn in expgraphchar.keys():
342	with gen.Capturing() as output:
343	gen.printing_class(expgraphchar[modexpn])
344	print "'" + modexpn + "'; " + ', '.join(output)
345
346	return modgraphchar, expgraphchar
347
348	def exp_headers(mod,config):
349	""" Function to provide the headers and the experiments of a given model
350	model= model
351	config= configuration of the experiment
352	"""
353	fname = 'exp_headers'
354
355	# No CASE in python!
356	# case ${mod} in ...
357	if mod == 'WRF':
358	expers = config['WRFexps'].split(':')
359	fhs = config['WRFheaders'].split(':')
360	elif mod == 'LMDZ':
361	expers = config['LMDZexps'].split(':')
362	fhs = config['LMDZheaders'].split(':')
363	elif mod == 'WRF_LMDZ':
364	expers = config['WRF_LMDZexps'].split(':')
365	fhs = config['WRF_LMDZheaders'].split(':')
366	else:
367	print errmsg
368	print " " + fname + ": model '" + mod + "' not ready!!"
369	quit(-1)
370
371	return expers, fhs
372
373	class VariableInf(object):
374	""" Class which holds all information of a given variable
375	name= CF name of the variable
376	header= header of the file from which it can be computed
377	model= file-header variable from which it can be directly computed
378	diag= file-header diagnostics from which it can be computed
379	"""
380	def __init__( self, name, fheader, model, diag):
381	self.name= None
382	self.fheader= None
383	self.model= None
384	self.diag= None
385	if name is not None:
386	self.name = name
387	self.fheader= fheader
388	self.model= model
389	self.diag= diag
390
391	class ModelInf(object):
392	""" Class which holds all information from a given model
393	name= Acronym of the model
394	model= Long description of the model
395	dim[x/y/z/t/s]n= name of the dimensions of the model
396	vdim[x/y/z/t/s]n= name of the variable-dimensions of the model
397	dimensions= list of the dimensions
398	vardimensions= list of the variable-dimensions
399	timemodif= time modification to apply to the model files ('None' for nothing)
400	'setorigin',[YYYYMMDDHHMISS]: re-set origin of times at [YYYYMMDDHHMISS]
401	"""
402	def __init__( self, name, model, dx, dy, dz, dt, ds, vdx, vdy, vdz, vdt, vds, \
403	tmod):
404	self.name= None
405	self.model= None
406	self.dimxn= None
407	self.dimyn= None
408	self.dimzn= None
409	self.dimtn= None
410	self.dimsn= None
411	self.vardxn= None
412	self.vardyn= None
413	self.vardzn= None
414	self.vardtn= None
415	self.vardsn= None
416	self.timemodif = None
417	if name is not None:
418	self.name = name
419	self.model= model
420	self.dimxn= dx
421	self.dimyn= dy
422	self.dimzn= dz
423	self.dimtn= dt
424	self.dimsn= ds
425	self.vardxn= vdx
426	self.vardyn= vdy
427	self.vardzn= vdz
428	self.vardtn= vdt
429	self.vardsn= vds
430	if tmod is not None:
431	self.timemodif = tmod
432	self.dimensions = [dx, dy, dz, dt, ds]
433	self.vardimensions = [vdx, vdy, vdz, vdt, vds]
434
435	def variable_compute(idir,var,ftests,db):
436	""" Function to retrieve the computation way of a given variable using a series of test files
437	iwdir= directory with the test files
438	var= name of the variable to test
439	filtests= dictionary with the test files for each header
440	"""
441	fname='variable_compute'
442
443	cancompute = True
444	for headerf in ftests.keys():
445	filen=ftests[headerf]
446	try:
447	with gen.Capturing() as output:
448	vmod, vdiag = ncvar.computevar_model(var, idir + '/' + filen)
449	except:
450	print errmsg
451	print 'ncvar.computevar_model(' + var + ', ' + idir + '/' + filen + ')'
452	for s1out in output: print s1out
453	quit(-1)
454
455	if vmod is None and vdiag is None:
456	cancompute = False
457	else:
458	cancompute = True
459	# Should be considered that variable can also be computed by both ways?
460	break
461
462	if not cancompute:
463	print warnmsg
464	print ' ' + fname + ": there is no way to compute '" + var + \
465	"' for model '" + mod
466	# Too extrict!
467	# quit(-1)
468
469	# Getting only the first possibility of model and diagnostic
470	if vmod is not None:
471	modV = vmod[0]
472	else:
473	modV = None
474	if vdiag is not None:
475	diagV = vdiag[0]
476	else:
477	diagV = None
478
479	varcomp = VariableInf(var, headerf, modV, diagV)
480
481	# a ';' list 'varcompute' it is created for each variable giving:
482	# [var]\|[vark]\|[headerf][varmod]\|[vardiag]
483	# This list will be used to compute a new file for each variable
484
485	if db:
486	print 'Variable information _______'
487	gen.printing_class(varcomp)
488
489	return varcomp
490
491	def get_operations_var(vc,db):
492	""" Function to provide the operations to make for each variable from 'VAR_' dictionary
493	vc= 'VAR_' dictionary, dictionary with all the parameters started with 'VAR_'
494	"""
495	fname = 'get_operations_var'
496
497	LH = len('VAR_')
498	iop = 1
499	# list of operations
500	doopers = {}
501	# operations by variable
502	operationsvar = {}
503	# individual operations by variable (not repeated)
504	indivoperationsvar = {}
505
506	# Variables with a calculation whic requires vertical interpolation to all the
507	# file (operations stating by 'VAR_pinterp')
508	LVp = len('VAR_pinterp')
509	varglobalp = []
510
511	for oper in vc.keys():
512	Loper = len(oper)
513	opn = oper[LH:Loper+1]
514	vns = vc[oper].split(':')
515	ops1 = opn.split('+')
516	doopers[opn] = ops1
517	for vn in vns:
518	if not operationsvar.has_key(vn):
519	operationsvar[vn] = [opn]
520	indivoperationsvar[vn] = ops1
521	else:
522	opers = operationsvar[vn]
523	opers.append(opn)
524	operationsvar[vn] = opers
525	opers1 = indivoperationsvar[vn]
526	for op1 in ops1:
527	if not gen.searchInlist(opers1, op1):
528	opers1.append(op1)
529	indivoperationsvar[vn] = opers1
530
531	if oper[0:LVp] == 'VAR_pinterp':
532	for vn in vns:
533	if not gen.searchInlist(varglobalp,vn): varglobalp.append(vn)
534
535	iop = iop + 1
536
537	if db:
538	print ' operations to make _______'
539	gen.printing_dictionary(doopers)
540	print ' operations by variable _______'
541	gen.printing_dictionary(operationsvar)
542	print ' individual operations by variable _______'
543	gen.printing_dictionary(indivoperationsvar)
544	print ' variables with the vertical interpolation for all the file _______'
545	print ' #', varglobalp
546
547	return doopers, operationsvar, indivoperationsvar, varglobalp
548
549	def pinterp_var(oper):
550	""" Function to retrieve characteristics of the vertical interpolation for the operation
551	oper= operation
552	Wether vertical interpolation is:
553	# 'global': for all file
554	# 'local': at a given step of the process
555	# 'none': no vertical interpolation
556	>>> pinterp_var('last+pinterp+xmean')
557	local
558	>>> pinterp_var('pinterp+tmean+xmean')
559	global
560	>>> pinterp_var('xmean')
561	none
562	"""
563	fname = 'pinterp_var'
564
565	pinn = 'pinterp'
566
567	Lpinterp = len(pinn)
568	if oper[0:Lpinterp] == pinn:
569	pkind = 'global'
570	return pkind
571
572	if oper.find(pinn) != -1:
573	pkind = 'local'
574	else:
575	pkind = 'none'
576
577	return pkind
578
579	def compvars_listconstruct(config, minf, Files, TestFiles, idir, odir, debug):
580	""" Function to construct the list of variables to compute
581	config= dictionary with the configuration of the execution
582	variables to compute are get with all values started by `VAR_' in 'module_graphics.dat', and then
583	providing a consecutive number of calculations separated by '+'
584	VAR_[calc1]+[calc2] = tas:wss
585	will compute first [calc1] and then [calc2] for 'tas' and 'wss'
586	minf= class with information about the model
587	Files= dictionary of files for heach header
588	TestFiles= dictionary of files to test how to compute the variable
589	odir= output directory
590	"""
591	fname='compvars_listconstruct'
592
593	varcomp = gen.get_specdictionary_HMT(config,H='VAR_')
594
595	if debug:
596	print ' variables to compute ________'
597	gen.printing_dictionary(varcomp)
598
599	# Getting operations by variable
600	opers, varoper, indivaroper, varglobp = get_operations_var(varcomp,debug)
601
602	strfiles = gen.dictKeysVals_stringList(Files)
603	strtestfiles = gen.dictKeysVals_stringList(TestFiles)
604
605	Svarcompute = ''
606
607	# Main dictionary with all the variables and their calculation
608	allvarcomp = {}
609
610	ivop = 0
611	for vn in varoper:
612	vcomp = variable_compute(idir,vn,TestFiles,False)
613	for op in varoper[vn]:
614	# Creation of a String as: [CFname]\|[operation]\|[fheader]\|[vmodel]\|[vdiag]\|[globalP]
615	# [CFname]: CF-name of the variable (must appear in 'variables_values.dat')
616	# [operation]: operation to compute
617	# [fheader]: header of file with the required variables
618	# [vmodel]: Variable from model output
619	# [vdiag]: Varible as diagnostic from different variables of the model output
620	# [globalP]: Wether vertical interpolation is:
621	# 'global': for all file
622	# 'local': at a given step of the process
623	# 'none': no vertical interpolation
624	globalP = pinterp_var(op)
625	if vcomp.model is not None:
626	if type(vcomp.model) == type(list([1, 2])):
627	Smodel = ':'.join(vcomp.model)
628	elif type(vcomp.model) == type('A'):
629	Smodel = vcomp.model
630	else:
631	Smodel = 'None'
632	if vcomp.diag is not None:
633	if type(vcomp.diag) == type(list([1, 2])):
634	Sdiag = ':'.join(vcomp.diag)
635	elif type(vcomp.diag) == type('A'):
636	Sdiag = vcomp.diag
637	else:
638	Sdiag = 'None'
639
640	# To get a combination of operations, all the precedent steps are kept
641	# thus, they are also available !
642	seqops = op.split('+')
643	for seqop in seqops:
644	if seqop == seqops[0]: seqopS=seqop
645	else: seqopS = seqopS + '+' + seqop
646	allvarcomp[vn+'_'+seqopS] = [vcomp.fheader, vcomp.model, vcomp.diag, \
647	globalP]
648
649	Svc = vcomp.name + '\|' + seqopS + '\|' + vcomp.fheader + '\|' + \
650	Smodel + '\|' + Sdiag + '\|' + globalP
651	if ivop == 0:
652	Svarcompute = Svc
653	else:
654	Svarcompute = Svarcompute + ',' + Svc
655
656	ivop = ivop + 1
657
658	if debug:
659	print ' Variables to compute _______'
660	gen.printing_dictionary(allvarcomp)
661
662	# Outwritting the varcompute to avoid next time (if it is not filescratch!)
663	objf = open(odir + '/varcompute.inf', 'w')
664	objf.write('files: ' + strfiles + '\n')
665	objf.write('testfiles: ' + strtestfiles + '\n')
666	objf.write('varcompute: ' + Svarcompute + '\n')
667	objf.write('itotv: ' + str(ivop) + '\n')
668	objf.close()
669
670	return allvarcomp, ivop
671
672	def read_varcomp_file(filen):
673	""" Function to read 'varcompute.inf' and reconstruct the dictionaries
674	"""
675	fname = 'read_varcomp_file'
676
677	allvarcomp = {}
678
679	objf = open(filen, 'r')
680
681	for line in objf:
682	vals = line.replace('\n','').split(' ')
683	if vals[0] == 'files:':
684	Files = gen.stringList_dictKeysVals(vals[1])
685	elif vals[0] == 'testfiles:':
686	TestFiles = gen.stringList_dictKeysVals(vals[1])
687	elif vals[0] == 'varcompute:':
688	for VvalsS in vals[1].split(','):
689
690	Vvals = VvalsS.split('\|')
691	if Vvals[3] == 'None':
692	mod = None
693	else:
694	mod = Vvals[3].split(':')
695	if Vvals[4] == 'None':
696	diag = None
697	else:
698	diag = Vvals[4].split(':')
699
700	allvarcomp[Vvals[0]+'_'+Vvals[1]] = [Vvals[2], mod, diag, Vvals[5]]
701
702	# # To get a combination of operations, all the precedent steps are kept
703	# # thus, they are also available !
704	# seqops = Vvals[1].split('+')
705	# for seqop in seqops:
706	# if seqop == seqops[0]: seqopS=seqop
707	# else: seqopS = seqopS + '+' + seqop
708	# allvarcomp[Vvals[0]+'_'+seqopS] = [Vvals[2], mod, diag, Vvals[5]]
709	elif vals[0] == 'itotv:':
710	ivop = int(vals[1])
711
712	objf.close()
713
714	return Files, TestFiles, allvarcomp, ivop
715
716	def pinterpS(gP):
717	""" For that variables which require vertical interpolation 'p' suffix to the file header is added
718	gP= kind of pinterp: 'global', 'local', 'none'
719	"""
720	fname = 'pinterpS'
721
722	if gP != 'none':
723	gPS = 'p'
724	else:
725	gPS = ''
726
727	return gPS
728
729	def compute_variable(minf, idir, usefiles, odir, cvar, gP, scr, pyH, Tref, Tunits, db):
730	""" Function to compute a variable
731	minf= class with the information of the model
732	idir= directory with the input files
733	usefiles= dictionary of files as dict([headerf]) = [file1], ..., [fileN]
734	odir= directory to write the output files
735	cvar= class with the information of the variable: 'name', 'fheader', 'varmod', 'vardiag'
736	gP= kind of vertical interpolation ('global', 'local', 'none')
737	scr= should it be done from the scratch?
738	pyH= location of the python HOME
739	Tref= CF time reference
740	Tunits= CF time units
741	"""
742	fname='compute_variable'
743
744	CFvarn=cvar.name
745	headerf = cvar.fheader
746	modvar = cvar.model
747	diagvar = cvar.diag
748
749	cfiles = usefiles[headerf]
750
751	# dimensions
752	dnx = minf.dimxn
753	dny = minf.dimyn
754	# var-dimensions
755	vdnx = minf.vardxn
756	vdny = minf.vardyn
757
758	# Computing separately and then joinging for all files
759	Ntotfiles = len(cfiles)
760	Nzeros = len(str(Ntotfiles))
761	NStot = str(Ntotfiles).zfill(Nzeros)
762
763	# For that variables which require vertical interpolation 'p' suffix to the
764	# file header is added
765	SgP = pinterpS(gP)
766
767	# Getting in working dir
768	os.chdir(odir)
769
770	# File to keep track of all operations
771	otrackf = open( odir + '/all_computevars.inf', 'a')
772
773	# Computing variable
774	ifile=1
775	for cf in cfiles:
776	ifS = str(ifile).zfill(Nzeros)
777	ifilen = odir + '/' + CFvarn + '_' + headerf + SgP + '_' + ifS + '-' + \
778	NStot + '.nc'
779	fileon = odir + '/' + CFvarn + '_' + headerf + SgP + '.nc'
780
781	if scr:
782	sout = sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
783	sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
784
785	if not os.path.isfile(ifilen) and not os.path.isfile(fileon):
786	# Since model direct values are retrieved from `variables_valules.dat' which was
787	# initially coincived as a way to only give variable attributes, range and color
788	# bars, if a variable has a diagnostic way to be computed, the later one will be
789	# preferred
790	if db:
791	print ' ' + fname + ": creation of variable file '" + CFvarn + \
792	"' in file '" + ifilen + "' ..."
793	print ' model variable:', modvar
794	print ' diagostic variable:', diagvar
795
796	if diagvar is None:
797	# model variable
798	if db: print ' '+fname+ ": variable direct from model '" +modvar+ "'"
799	values = modvar + ',0,-1,-1'
800	vs = modvar + ',' + vdnx + ',' + vdny + ',' + vdnz + ',' + vdnt
801
802	try:
803	with gen.Capturing() as output:
804	ncvar.DataSetSection_multivars(values, idir + '/' + cf, vs)
805	except:
806	print errmsg
807	print 'ncvar.DataSetSection_multivars('+values+', '+idir+'/'+cf+ \
808	', '+vs+')'
809	for s1out in output: print s1out
810	quit(-1)
811
812	newfile, loc = gen.search_sec_list(output,'succesfull')
813	ofile = newfile[0].split(' ')[7]
814	sub.call('mv ' + ofile + ' ' + ifilen, shell=True)
815
816	# Keeping track of the operations
817	pyins = pyH + "/nc_var.py -f " + idir + '/' + cf + \
818	" -o DataSetSection_multivars -v " + vs + " -S '" + values + "'"
819	otrackf.write('\n')
820	otrackf.write('# ' + CFvarn + " " + modvar + '\n')
821	otrackf.write('python ' + pyins + '\n')
822
823	# CF renaming of variable
824	ncvar.chvarname(CFvarn,ifilen,modvar)
825	else:
826	# diagnostic variable
827	if db: print ' '+fname+ ": variable as diagnostic '" +diagvar[0]+ "'"
828	dims = dnt+'@'+vdnt+','+dnz+'@'+vdnz+','+dny+'@'+vdny+','+dnx+'@'+vdnx
829	diagn = diagvar[0]
830	Ndiagvars = len(diagvar)
831	diagc = '@'.join(diagvar[1:])
832
833	values = '-f ' + idir + '/' + cf + " -d '" + dims + "' -v '" + \
834	diagn + '\|' + diagc + "'"
835	try:
836	with gen.Capturing() as output:
837	sout = sub.call('python ' +pyH+ '/diagnostics.py ' + values, \
838	shell=True)
839	except:
840	print errmsg
841	print 'python ' + pyH + '/diagnostics.py ' + values
842	for s1out in output: print s1out
843	quit(-1)
844	if db:
845	for s1out in output: print s1out
846
847	sout = sub.call(' mv diagnostics.nc ' + ifilen, shell=True)
848
849	# Keeping track of the operations
850	pyins = 'python ' + pyH + '/diagnostics.py ' + values
851	otrackf.write('\n')
852	otrackf.write('# ' + CFvarn + " " + diagn + '\n')
853	otrackf.write(pyins + '\n')
854
855	# Attaching necessary variables for the pressure interpolation
856	if gP != 'none':
857	if minf.name == 'WRF' or minf.name == 'WRF_LMDZ':
858	requiredinterpvars = ['P', 'PB', 'PSFC', 'PH', 'PHB', 'HGT', 'T',\
859	'QVAPOR', 'XLONG', 'XLAT', 'Times']
860	elif minf.name == 'LMDZ':
861	requiredinterpvars = ['pres', 'psol', 'geop', 'phis', 'temp', \
862	'ovap', 'lon', 'lat', 'time_counter']
863	else:
864	print errmsg
865	print fname + ": for model '" + minf.name + "' required " + \
866	" variables for vertical interpolation are not known !!"
867	quit(-1)
868
869	print " " + fname + ": adding variables:", requiredinterpvars, \
870	' to allow pressure interpolation'
871	for rqv in requiredinterpvars:
872	try:
873	with gen.Capturing() as output:
874	ncvar.fvaradd(idir+'/'+cf+','+rqv,ifilen)
875	except:
876	print errmsg
877	print 'fvaradd('+idir+'/'+cf+','+rqv+', '+ifilen+')'
878	for s1out in output: print s1out
879	quit(-1)
880	if db:
881	for s1out in output: print s1out
882
883	# CFification of files
884	if minf.name == 'WRF' or minf.name == 'WRF_LMDZ':
885	values = vdnx + ':' + vdny + ':'+ Tref + ':' + Tunits
886	try:
887	with gen.Capturing() as output:
888	ncvar.WRF_toCF(values, ifilen)
889	except:
890	print errmsg
891	print 'WRF_toCF('+ values +', ' + ifilen + ')'
892	for s1out in output: print s1out
893	quit(-1)
894	if db:
895	for s1out in output: print s1out
896	elif minf.name == 'LMDZ':
897	try:
898	with gen.Capturing() as output:
899	ncvar.LMDZ_toCF(Tunits + '!since!' + Tref, ifilen)
900	except:
901	print errmsg
902	print 'LMDZ_toCF('+ Tunits +'!since!'+ Tref + ', ' + ifilen + ')'
903	for s1out in output: print s1out
904	quit(-1)
905	if db:
906	for s1out in output: print s1out
907	else:
908	print errmsg
909	print ' ' + fname + ": no CFification for model '" +minf.name+ "' !!"
910	print " available ones: 'WRF', 'WRF_LMDZ', 'LMDZ'"
911	quit(-1)
912
913	ifile = ifile + 1
914
915	otrackf.close()
916
917	# Joining variable files
918	if not os.path.isfile(fileon):
919	if db:
920	print ' ' + fname + ": concatenating all variable files to create '" + \
921	fileon + "..."
922	try:
923	with gen.Capturing() as output:
924	ncvar.netcdf_fold_concatenation_HMT('./,time', CFvarn+'_'+headerf+ \
925	SgP+'_,-,.nc', 'all')
926	except:
927	print errmsg
928	print 'netcdf_fold_concatenation_HMT(./,time, '+CFvarn+'_'+headerf+SgP+ \
929	'_,-,.nc, all)'
930	for s1out in output: print s1out
931	quit(-1)
932	if db:
933	for s1out in output: print s1out
934
935	sout = sub.call('mv netcdf_fold_concatenated_HMT.nc ' + fileon, shell=True)
936	if os.path.isfile(fileon):
937	sout = sub.call('rm '+CFvarn +'_'+ headerf+SgP + '_-.nc >& /dev/null',\
938	shell=True)
939
940	# Re-setting time of final concatenated file
941	if minf.timemodif is not None:
942	if db:
943	print ' ' + fname + ": Modifying times of the file by '" + \
944	minf.timemodif + "' !!"
945	try:
946	with gen.Capturing() as output:
947	ncvar.time_reset(minf.timemodif, fileon,'time')
948	except:
949	print errmsg
950	print 'time_reset(' + minf.timemodif + ', ' + fileon + ', time)'
951	for s1out in output: print s1out
952	quit(-1)
953	if db:
954	for s1out in output: print s1out
955
956	return
957
958	def compute_statistics(minf, config, idir, usefiles, odir, cvar, gP, Opers, scr, db):
959	""" Function to compute different statistics it will take previous steps if they
960	are availale
961	minf= class with the information of the model
962	config= dictionary with the configuration of the experiment
963	idir= directory with the input files
964	usefiles= ',' list of files to use [file1],...,[fileN]
965	odir= directory to write the output files
966	cvar= class with the information of the variable: 'name', 'fheader', 'varmod', 'vardiag'
967	gP= kind of vertical interpolation ('global', 'local', 'none')
968	Opers= kind of operation: (as possible multiple consecutive combination of operations separated by '+'
969	[calc1]+[calc2] will compute first [calc1] and then [calc2]
970	acc: temporal accumulated values
971	diff: differences between models
972	direct: no statistics
973	last: last temporal value
974	Lmean: latitudinal mean values
975	Lsec: latitudinal section (latitudinal value must be given, [var]@[lat])
976	lmean: longitudinal mean values
977	lsec: longitudinal section (longitudinal value must be given, [var]@[lat])
978	pinterp: pressure interpolation (to the given $plevels)
979	tmean: temporal mean values
980	tstd: temporal standard deviation values
981	tturb: Taylor's turbulence decomposition value (x - <x>) for time
982	tvar: temporal variance values
983	xmean: x-axis mean values
984	xvar: x-axis variance values
985	ymean: y-axis mean values
986	zsum: vertical aggregated values
987	scr= should it be done from the scratch?
988	"""
989	fname='compute_statistics'
990
991	# Getting a previous file name to continue with(for combinations)
992	CFvarn=cvar.name
993	headerf = cvar.fheader
994	modvar = cvar.model
995	diagvar = cvar.diag
996
997	cfiles = usefiles[headerf]
998
999	# dimensions
1000	dnx = minf.dimxn
1001	dny = minf.dimyn
1002	# var-dimensions
1003	vdnx = minf.vardxn
1004	vdny = minf.vardyn
1005	# Model dimension-variable dictionary
1006	Modeldimvardict = {dnx: vdnx, dny: vdny, dnz: vdnz, dnt: vdnt}
1007
1008	# Some experiment configuration values
1009	# plevels= ':' separated list of pressures (in Pa) to use for the vertical interpolation
1010	plevels = config['plevels']
1011	# pyH= location of the python HOME
1012	pyH = config['pyHOME']
1013	# Tref= CF time reference
1014	Tref = config['CFreftime']
1015	# Tunits= CF time units
1016	Tunits = config['CFunitstime']
1017	# opsur = operation surnames
1018	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
1019	Opsurs = {}
1020	for opk in opsur:
1021	opn = opk.split('_')[1]
1022	vals = opsur[opk]
1023	Opsurs[opn] = vals.split(':')
1024	# if db:
1025	# print ' ' + fname + ' operation surnames _______'
1026	# gen.printing_dictionary(Opsurs)
1027
1028	# Getting in working dir
1029	os.chdir(odir)
1030
1031	# File to keep track of all operations
1032	otrackf = open( odir + '/all_statsvars.inf', 'a')
1033
1034	# For that variables which require vertical interpolation 'p' suffix to the file
1035	# header is added.
1036	# After some operations some CF dimension-variables might not be kept
1037
1038	if gP != 'none':
1039	SgP = 'p'
1040	varnCFs = ['lon', 'lat', 'pres', 'time']
1041	else:
1042	SgP = ''
1043	varnCFs = ['lon', 'lat', 'time']
1044
1045	# CF dimension-variable dictionary
1046	CFdimvardict = {'lon': 'lon', 'lat': 'lat', 'pres': 'pres', 'time': 'time'}
1047
1048	# Input file
1049	ifilen = odir + '/' + CFvarn + '_' + headerf + SgP + '.nc'
1050
1051	# Computing variable statisitcs
1052	istats=0
1053
1054	# List of not computed statistics
1055	wrongstats = []
1056
1057	opers = Opers.split('+')
1058	Fopers = ''
1059	if db: print " computing statistics of variable:'", CFvarn, "' operation '" + \
1060	Opers + "' using file '" + ifilen + "...'"
1061	for op in opers:
1062
1063	# File name from previous operations, and name of the variable within the
1064	# file (some operations change the name of the variable)
1065	if op == opers[0]:
1066	Fopers = op
1067	prevfile = ifilen
1068	vninF = CFvarn
1069	else:
1070	Fopers = Fopers + '_' + op
1071	prevfile = fileon
1072	fileon = odir + '/' + CFvarn + '_' + headerf + SgP + '_' + Fopers + '.nc'
1073
1074	# Adding required variables for the vertical interpolation in all operations
1075	SvarnCFs = ',' + ','.join(varnCFs)
1076
1077	if gP != 'none':
1078	if gP == 'local':
1079	CFvarnp = vninF + ',P,PB,PSFC,PH,PHB,HGT,T,QVAPOR,XLONG,XLAT,Times'+ \
1080	SvarnCFs
1081	else:
1082	CFvarnp = vninF + SvarnCFs
1083	else:
1084	CFvarnp = vninF + SvarnCFs
1085
1086	if scr:
1087	sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1088
1089	hprevfile = prevfile[0:len(prevfile)-3]
1090
1091	if db: print ' ', op, "using '" + prevfile + "' "
1092	# Just produce file in case output file does not exist
1093	if not os.path.isfile(fileon):
1094	if db:
1095	print ' ' + fname + ": creation of file '" + fileon + "' ..."
1096	dofile = True
1097	else:
1098	dofile = False
1099
1100	# Variables to be kept in the final file
1101	varkeep = []
1102
1103	if op == 'acc':
1104	# temporal accumulated values
1105	print " " + fname + ": kind '" + op + "' not ready !!"
1106	wrongstats.append(CFvarn + '_' + opers)
1107	break
1108	elif op == 'direct':
1109	# no statistics
1110	if dofile:
1111	sout = sub.call('mv ' + prevfile + ' ' + fileon, shell=True)
1112	elif op == 'last':
1113	# last temporal value
1114	vals='time,-9,0,0'
1115	if dofile:
1116	try:
1117	with gen.Capturing() as output:
1118	ncvar.DataSetSection(vals,prevfile)
1119	except:
1120	print errmsg
1121	print 'DataSetSection('+vals+',', prevfile, ')'
1122	for s1out in output: print s1out
1123	quit(-1)
1124
1125	if db:
1126	for s1out in output: print s1out
1127
1128	sout = sub.call('mv ' + hprevfile + '_time_B-9-E0-I0.nc ' + fileon, \
1129	shell=True)
1130
1131	# Keeping the operations
1132	pyins=pyH + "/nc_var.py -o DataSetSection -S '" + vals + "' -f " + \
1133	prevfile
1134	otrackf.write("\n")
1135	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1136	otrackf.write(pyins + "\n")
1137
1138	elif op =='Lmean':
1139	# latitudinal mean values
1140	print " " + fname + ": kind '" + op + "' not ready !!"
1141	wrongstats.append(CFvarn + '_' + opers)
1142	break
1143	elif op =='Lsec':
1144	# latitudinal section (latitudinal value must be given, [var]@[lat])
1145	print " " + fname + ": kind '" + op + "' not ready !!"
1146	wrongstats.append(CFvarn + '_' + opers)
1147	break
1148	elif op =='lmean':
1149	# longitudinal mean values
1150	print " " + fname + ": kind '" + op + "' not ready !!"
1151	wrongstats.append(CFvarn + '_' + opers)
1152	break
1153	elif op =='lsec':
1154	# longitudinal section (longitudinal value must be given, [var]@[lon])
1155	print " " + fname + ": kind '" + op + "' not ready !!"
1156	wrongstats.append(CFvarn + '_' + opers)
1157	break
1158	elif op == 'pinterp':
1159	# pinterp: pressure interpolation (to the given $plevels)
1160	vals=plevels + ',1,1'
1161	if dofile:
1162	try:
1163	with gen.Capturing() as output:
1164	ncvar.pinterp(vals,prevfile,vninF)
1165	except:
1166	print errmsg
1167	print 'pinterp('+vals+',', prevfile, ','+vninF+')'
1168	for s1out in output: print s1out
1169	quit(-1)
1170
1171	if db:
1172	for s1out in output: print s1out
1173
1174	sout = sub.call('mv pinterp.nc ' + fileon, shell=True)
1175
1176	# Keeping the operations
1177	pyins=pyH + "/nc_var.py -o pinterp -S '" + vals + "' -f " + \
1178	prevfile + "-v " + vninF
1179	otrackf.write("\n")
1180	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1181	otrackf.write(pyins + "\n")
1182
1183	# adding CF lon,lat,time in WRF files
1184	if minf.name == 'WRF':
1185	values = vdnx + ':' + vdny + ':'+ Tref + ':' + Tunits
1186	ncvar.WRF_toCF(values, fileon)
1187
1188	# vertical interpolation variables are no more needed
1189	CFvarnp = vninF
1190	gP = 'none'
1191
1192	elif op == 'tmean':
1193	# temporal mean values
1194	vals='time\|-1,time,mean,' + ':'.join(varnCFs) + ':' + vdnz
1195	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1196	if dofile:
1197	try:
1198	with gen.Capturing() as output:
1199	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1200	except:
1201	print errmsg
1202	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1203	for s1out in output: print s1out
1204	quit(-1)
1205
1206	if db:
1207	for s1out in output: print s1out
1208
1209	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
1210
1211	# Keeping the operations
1212	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1213	"' -f " + prevfile + " -v " + CFvarnp
1214	otrackf.write("\n")
1215	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1216	otrackf.write(pyins + "\n")
1217
1218	# removing dimension variable-dimension 'time'
1219	varnCFs.remove('time')
1220
1221	varkeep.append('timestats')
1222
1223	elif op == 'tstd':
1224	# temporal standard deviation values
1225	vals='time\|-1,time,std,' + ':'.join(varnCFs) + ':' + vdnz
1226	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1227	if dofile:
1228	try:
1229	with gen.Capturing() as output:
1230	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1231	except:
1232	print errmsg
1233	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1234	for s1out in output: print s1out
1235	quit(-1)
1236
1237	if db:
1238	for s1out in output: print s1out
1239
1240	sout = sub.call('mv file_oper_alongdims_std.nc '+fileon, shell=True)
1241
1242	# Keeping the operations
1243	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1244	"' -f " + prevfile + " -v " + CFvarnp
1245	otrackf.write("\n")
1246	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1247	otrackf.write(pyins + "\n")
1248
1249	# removing dimension variable-dimension 'time'
1250	varnCFs.remove('time')
1251
1252	varkeep.append('timestats')
1253
1254	elif op == 'tturb':
1255	# temporal turbulent values
1256	vals='time\|-1,time,turb,' + ':'.join(varnCFs) + ':' + vdnz
1257	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1258	if dofile:
1259	try:
1260	with gen.Capturing() as output:
1261	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1262	except:
1263	print errmsg
1264	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1265	for s1out in output: print s1out
1266	quit(-1)
1267
1268	if db:
1269	for s1out in output: print s1out
1270
1271	sout = sub.call('mv file_oper_alongdims_turb.nc '+fileon, shell=True)
1272
1273	# Keeping the operations
1274	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1275	"' -f " + prevfile + " -v " + CFvarnp
1276	otrackf.write("\n")
1277	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1278	otrackf.write(pyins + "\n")
1279
1280	# removing dimension variable-dimension 'time'
1281	varkeep.append('timestats')
1282
1283	elif op == 'tvar':
1284	# temporal variance values (Taylor's turbulence)
1285	vals='time\|-1,time,var,' + ':'.join(varnCFs) + ':' + vdnz
1286	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1287	if dofile:
1288	try:
1289	with gen.Capturing() as output:
1290	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1291	except:
1292	print errmsg
1293	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1294	for s1out in output: print s1out
1295	quit(-1)
1296
1297	if db:
1298	for s1out in output: print s1out
1299
1300	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
1301
1302	# Keeping the operations
1303	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1304	"' -f " + prevfile + " -v " + CFvarnp
1305	otrackf.write("\n")
1306	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1307	otrackf.write(pyins + "\n")
1308
1309	# removing dimension variable-dimension 'time'
1310	varnCFs.remove('time')
1311
1312	varkeep.append('timestats')
1313
1314	elif op == 'xmean':
1315	# x-axis mean values
1316	vals='lon\|-1,lon,mean,' + ':'.join(varnCFs) + ':' + vdnz
1317	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1318	if dofile:
1319	try:
1320	with gen.Capturing() as output:
1321	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1322	except:
1323	print errmsg
1324	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1325	for s1out in output: print s1out
1326	quit(-1)
1327
1328	if db:
1329	for s1out in output: print s1out
1330
1331	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
1332
1333	# Keeping the operations
1334	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1335	"' -f " + prevfile + " -v " + CFvarnp
1336	otrackf.write("\n")
1337	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1338	otrackf.write(pyins + "\n")
1339
1340	# removing dimension variable-dimension 'lon'
1341	varnCFs.remove('lon')
1342
1343	varkeep.append('lonstats')
1344
1345	elif op == 'xvar':
1346	# x-axis variance values
1347	vals='lon\|-1,lon,var,' + ':'.join(varnCFs) + ':' + vdnz
1348	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1349	if dofile:
1350	try:
1351	with gen.Capturing() as output:
1352	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1353	except:
1354	print errmsg
1355	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1356	for s1out in output: print s1out
1357	quit(-1)
1358
1359	if db:
1360	for s1out in output: print s1out
1361
1362	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
1363
1364	# Keeping the operations
1365	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1366	"' -f " + prevfile + " -v " + CFvarnp
1367	otrackf.write("\n")
1368	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1369	otrackf.write(pyins + "\n")
1370
1371	# removing dimension variable-dimension 'lon'
1372	varnCFs.remove('lon')
1373
1374	varkeep.append('lonstats')
1375
1376	elif op == 'ymean':
1377	# y-axis mean values
1378	vals='lat\|-1,lat,mean,' + ':'.join(varnCFs) + ':' + vdnz
1379	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1380	if dofile:
1381	try:
1382	with gen.Capturing() as output:
1383	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1384	except:
1385	print errmsg
1386	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1387	for s1out in output: print s1out
1388	quit(-1)
1389
1390	if db:
1391	for s1out in output: print s1out
1392
1393	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
1394
1395	# Keeping the operations
1396	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1397	"' -f " + prevfile + " -v " + CFvarnp
1398	otrackf.write("\n")
1399	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1400	otrackf.write(pyins + "\n")
1401
1402	# removing dimension variable-dimension 'lat'
1403	varnCFs.remove('lat')
1404
1405	varkeep.append('latstats')
1406
1407	elif op == 'zsum':
1408	# vertical aggregated values
1409	print " " + fname + ": kind '" + op + "' not ready !!"
1410	wrongstats.append(CFvarn + '_' + opers)
1411	break
1412	else:
1413	print errmsg
1414	print ' ' + fname + ": operation '" + op + "' not ready !!"
1415	quit(-1)
1416
1417	# End of kind of operation
1418
1419	# Variable name in file (vninF) changed due to operation
1420	# but only if previous operation does not the same 'statistic'
1421	chvn = gen.dictionary_key_list(Opsurs, op)
1422	if chvn is not None:
1423	oldvninF = vninF
1424	vninF = vninF + chvn
1425	CFvarnp = CFvarnp.replace(oldvninF,vninF)
1426
1427	if dofile:
1428	if len(varkeep) > 0:
1429	varkeepS = ',' + ','.join(varkeep)
1430	else:
1431	varkeepS = ''
1432
1433	# Adding such CF dimension variables which might not be in the
1434	# post-operation file
1435	try:
1436	with gen.Capturing() as output:
1437	varinfile = ncvar.ivars(fileon)
1438	except:
1439	print errmsg
1440	print 'ivars('+fileon+')'
1441	for s1out in output: print s1out
1442	quit(-1)
1443
1444	for CFvn in varnCFs:
1445	if not gen.searchInlist(varinfile, CFvn):
1446	if db:
1447	print ' ' + fname + ": recupering CF variable '" + CFvn + "'"
1448	try:
1449	with gen.Capturing() as output:
1450	ncvar.fvaradd(prevfile+','+CFvn, fileon)
1451	except:
1452	print errmsg
1453	print 'fvaradd(' + prevfile + ',' + CFvn +', '+fileon+')'
1454	for s1out in output: print s1out
1455	quit(-1)
1456
1457	if db:
1458	for s1out in output: print s1out
1459
1460	totalvarkeeps = CFvarnp + ',' + ','.join(varnCFs) + varkeepS
1461	try:
1462	with gen.Capturing() as output:
1463	oclean = ncvar.cleaning_varsfile(totalvarkeeps,fileon)
1464	except:
1465	print errmsg
1466	print 'cleaning_varsfile('+totalvarkeeps+','+fileon+')'
1467	for s1out in output: print s1out
1468	quit(-1)
1469
1470	if db:
1471	for s1out in output: print s1out
1472
1473	# End of operations
1474	if db:
1475	print fname + ": successful calculation of '" + vninF + "' from '" + \
1476	CFvarn + "' '" + Opers + "' !!"
1477
1478	if len(wrongstats) > 1:
1479	print warnmsg
1480	print ' ' + fname + ": statisitcs not possible to compute:", wrongstats
1481
1482	return
1483
1484	def compute_vars(config, modinf, idir, odir, Files, allvarcomp, fscr, debug):
1485	""" Function to compute the variables
1486	config= Configuration of the experiment
1487	modinf= class with information about the model
1488	idir= input experiment folder
1489	odir= output experiment folder
1490	Files= dictionary with the header and the files' correspondence
1491	allvarcomp= dictionary with all the variables to compute and their information
1492	fscr= whether files should be done from the scratch or not
1493	"""
1494	fname = 'compute_vars'
1495
1496	for vopn in allvarcomp.keys():
1497	# variable & operation
1498	vn = vopn.split('_')[0]
1499	op = vopn.split('_')[1]
1500	vopnV = allvarcomp[vopn]
1501	fheader = vopnV[0]
1502	if type(vopnV[1]) == type([1, 2]) and len(vopnV[1]) == 1:
1503	vals = vopnV[1]
1504	model = vals[0]
1505	else:
1506	model = vopnV[1]
1507	diag = vopnV[2]
1508	globalP = vopnV[3]
1509
1510	if debug:
1511	print ' ' + vn + ': ' + op
1512
1513	# Computing CF variable
1514	if model is not None or diag is not None:
1515	vinf = VariableInf(vn,fheader,model,diag)
1516	if op.find('pinterp') != -1:
1517	SP = 'p'
1518	else:
1519	SP = ''
1520
1521	# Comppute variable
1522	finalfilen = odir + '/' + vn + '_' + fheader + SP + '.nc'
1523	if fscr:
1524	sout = sub.call('rm ' + finalfilen + ' >& /dev/null', shell=True)
1525
1526	if not os.path.isfile(finalfilen):
1527	compute_variable(modinf, idir, Files, odir, vinf, globalP, fscr, \
1528	config['pyHOME'], config['CFreftime'], config['CFunitstime'], debug)
1529
1530	# Compute variable statistics
1531	finalfilen = odir + '/' + vn + '_' + fheader + SP + '_' + \
1532	op.replace('+','_') + '.nc'
1533	if fscr:
1534	sout = sub.call('rm ' + finalfilen + ' >& /dev/null', shell=True)
1535
1536	if not os.path.isfile(finalfilen):
1537	compute_statistics(modinf, cnf, iwdir, Files, owdir, vinf, globalP, \
1538	op, fscr, debug)
1539
1540	else:
1541	print errmsg
1542	print ' ' + fname + ": neither 'model' or 'diag' variables for '" + vn \
1543	+ "' !!"
1544	quit(-1)
1545
1546	return
1547
1548	def get_plots_var(vc,pltk,db):
1549	""" Function to provide the plots to make for each variable from pltk ('DIRPLT_', or 'DIFFPLT_') dictionary
1550	vc= pltk dictionary, dictionary with all the parameters started with pltk
1551	pltk= kind of plots. Values started by 'DIRPLT_', or 'DIFFPLT_' in model configuration
1552	"""
1553	fname = 'get_plots_var'
1554
1555	LH = len(pltk)
1556	# list of plots
1557	doplots = {}
1558	# plots by variable
1559	plotsvar = {}
1560	# individual plots by variable (not repeated)
1561	indivplotsvar = {}
1562
1563	# Variables with a plot which requires vertical interpolation
1564	varplotp = []
1565
1566	ipl = 0
1567	for plot in vc.keys():
1568	Lplot = len(plot)
1569	pln = plot[LH:Lplot+1]
1570	varops = vc[plot].split(':')
1571	for vn in varops:
1572	VOn = vn.split('#')
1573	# Variables and operations in plot
1574	vplt = []
1575	oplt = []
1576	voplt = []
1577	for vop in VOn:
1578	vv = vop.split('\|')[0]
1579	oo = vop.split('\|')[1]
1580	vplt.append(vv)
1581	oplt.append(oo)
1582	if not gen.searchInlist(voplt,vop): voplt.append(vop)
1583	vpltS = '#'.join(vplt)
1584	opltS = '#'.join(oplt)
1585
1586	if not doplots.has_key(pln):
1587	doplots[pln] = [vn]
1588	else:
1589	plots = doplots[pln]
1590	plots.append(vn)
1591	doplots[pln] = plots
1592
1593	if not plotsvar.has_key(vn):
1594	plotsvar[vn] = [pln]
1595	else:
1596	plots = plotsvar[vn]
1597	plots.append(pln)
1598	plotsvar[vn] = plots
1599
1600	for VOv in voplt:
1601	if not indivplotsvar.has_key(VOv):
1602	indivplotsvar[VOv] = [pln]
1603	else:
1604	plots = indivplotsvar[VOv]
1605	plots.append(pln)
1606	indivplotsvar[VOv] = plots
1607
1608	if vc[plot].find('pinterp') != -1:
1609	if not gen.searchInlist(varplotp,vn): varplotp.append(pln)
1610
1611	ipl = ipl + 1
1612
1613	if db:
1614	print ' plots to make _______'
1615	gen.printing_dictionary(doplots)
1616	print ' plots by variable\|operation _______'
1617	gen.printing_dictionary(plotsvar)
1618	print ' individual plots by variable\|operation _______'
1619	gen.printing_dictionary(indivplotsvar)
1620	print ' plots with the vertical interpolation _______'
1621	print ' #', varplotp
1622
1623	return doplots, plotsvar, indivplotsvar, varplotp
1624
1625	def plots_listconstruct(config, pkind, finf, odir, debug):
1626	""" Function to create the list of plots to draw
1627	config= Configuration of the experiment
1628	pkind= kind of plots ('DIRPLT', 'DIFFPLT')
1629	finf= file with the instructions of the plots
1630	odir= output experiment folder
1631	"""
1632	fname='plots_listconstruct'
1633
1634	dirplot = gen.get_specdictionary_HMT(config, H=pkind+'_')
1635
1636	# No plots because there are not plots of this kind
1637	if dirplot is None:
1638	print warnmsg
1639	print ' ' + fname + ": no plots for type '" + pkind + "' !!"
1640	return {}, 0
1641
1642	if debug:
1643	print " '" + pkind + "' plots to draw ________"
1644	gen.printing_dictionary(dirplot)
1645
1646	# Getting plots by variable
1647	plots, varplot, indivarplot, plotp = get_plots_var(dirplot, pkind+'_', debug)
1648
1649	Svarplot = gen.dictKeysVals_stringList(plots,cV=':')
1650
1651	Nplots = 0
1652	for pl in Svarplot.split(','):
1653	Nplots = Nplots + len(pl.split(':'))
1654
1655	# Outwritting the varcompute to avoid next time (if it is not filescratch!)
1656	objf = open(finf, 'w')
1657	objf.write('plots: ' + Svarplot + '\n')
1658	objf.write('itotp: ' + str(Nplots) + '\n')
1659	objf.close()
1660
1661	return plots, Nplots
1662
1663	def read_plot_file(figfile):
1664	""" Function to read the file with the information about the plots to draw
1665	figfile= file with the information
1666	"""
1667	fname = 'read_plot_file'
1668
1669	if not os.path.isfile(figfile):
1670	print errormsg
1671	print ' ' + fname + ": figures file '" + figfile + "' does not exist !!"
1672	quit(-1)
1673
1674	objf = open(figfile, 'r')
1675	for line in objf:
1676	if line[0:1] != '#' and len(line) > 1:
1677	values = line.replace('\\n','').split(' ')
1678	if values[0] == 'plots:':
1679	if values[1] != 'none':
1680	plots = gen.stringList_dictKeysVals(values[1],cV=':')
1681	else:
1682	pots = {}
1683	elif values[0] == 'itotp:':
1684	Nplots = int(values[1])
1685
1686	objf.close()
1687
1688	return plots, Nplots
1689
1690	def varnoper(vn, oper, opsurdict):
1691	""" Function to provide name of the variable after operation as result of addition of the 'surnames'
1692	vn= variable name
1693	oper= '+' separated list of operations
1694	opsurdict= dictionary with the surname of operations
1695	>>> OpSurDict = {'mean': ['tmean', 'xmean', 'ymean']}
1696	>>> varnoper('tas', 'pinterp+tmean+xmean', OpSurDict)
1697	tasmeanmean
1698	"""
1699	fname = 'varnoper'
1700
1701	newvn = vn
1702
1703	for op in oper.split('+'):
1704	surname = gen.dictionary_key_list(opsurdict,op)
1705	if surname is not None:
1706	newvn = newvn + surname
1707
1708	return newvn
1709
1710	def draw_plot(kplot, CFvplot, fplot, vplot, dplot, pplot, finame, tfig, kfig, mapval,\
1711	tvals, expgraphc, od, pyH, fscr, db):
1712	""" Function to draw a plot
1713	kplot= kind of plot
1714	CFvplot= list of the CFnames of the variables
1715	fplot= list with the files in the plot
1716	vplot= list with the name of the variables in each file
1717	dplot= list of the dimensions in each file
1718	pplot= list of pictoric characteristics for each variable
1719	finame= name of the figure
1720	tfig= title of the figure
1721	kfig= kind of the figure
1722	mapval= value of the map
1723	tvals= list with time values
1724	tunits: units of time ('!' for spaces)
1725	timekind: kind of time ticks in the plot
1726	timefmt: format of time ticks in the plot
1727	timelabel: label of time-axis in the plot
1728	expgraphc= dictionary with expriment graphical characteristics
1729	od= output directory
1730	pyH= python HOME
1731	fscr= whether figure should be done from scratch
1732	"""
1733	fname = 'draw_plot'
1734
1735	if fscr:
1736	sout = sub.call('rm ' + finame + ' >& /dev/null', shell=True)
1737
1738	os.chdir(od)
1739
1740	# CF variable-dimensions
1741	CFvardims = {'lon':'lon', 'lat': 'lat', 'pres': 'pres', 'time': 'time'}
1742
1743	# Tracking file with all the figures
1744	trkobjf = open('all_figures.inf', 'a')
1745
1746	# Time-characteristics
1747	tunits = tvals[0]
1748	timekind = tvals[1]
1749	timefmt = tvals[2]
1750	timelabel = tvals[3]
1751
1752	# Graphical labels and configuration
1753
1754
1755	if not os.path.isfile(finame):
1756	if db:
1757	print " drawing '" + finame + "' ..."
1758	print ' plot kind:', kplot
1759	print ' CFvars:', CFvplot
1760	print ' files:', fplot
1761	print ' in file vars:', vplot
1762	print ' dims:', dplot
1763	print ' pictoric values:', pplot
1764	print ' figure name:', finame
1765	print ' title:', tfig.replace('!',' ')
1766	print ' kind:', kfig
1767	print ' map:', mapval
1768	print ' time units:', tunits
1769
1770	if kplot == 'diffmap2Dsfc':
1771	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1772	quit(-1)
1773	elif kplot == 'diffmap2Dz':
1774	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1775	quit(-1)
1776	elif kplot == 'map2Dsfc':
1777	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1778	quit(-1)
1779	elif kplot == 'map3D':
1780	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1781	quit(-1)
1782	elif kplot == '2lines':
1783	figtit = tfig.replace('!','\|')
1784
1785	lineAstdn = CFvplot[0]
1786	lineBstdn = CFvplot[1]
1787	figfs = ','.join(fplot)
1788
1789	lineA = pplot[0]
1790	Arange = str(lineA[0]) + ',' + str(lineA[1])
1791	# Values are changed from specific values in `model_graphics.dat'
1792	if lineA[3].find('%') == -1:
1793	Aline = lineA[2]
1794	Akind = lineA[3]
1795	Amark = lineA[4]
1796	Asize = lineA[5]
1797	else:
1798	Aline = 'red'
1799	Akind = '-'
1800	Amark = ','
1801	Asize = 2.
1802
1803	lineB = pplot[0]
1804	Brange = str(lineB[0]) + ',' + str(lineB[1])
1805	# Values are changed from specific values in `model_graphics.dat'
1806	if lineB[3].find('%') == -1:
1807	Bline = lineB[2]
1808	Bkind = lineB[3]
1809	Bmark = lineB[4]
1810	Bsize = lineB[5]
1811	else:
1812	Bline = 'blue'
1813	Bkind = '-'
1814	Bmark = ','
1815	Bsize = '2.'
1816
1817	# It is assumed that if the space variable is 'lon': is desired a
1818	# (lon, vals) plot if it is 'lat': then (vals, lat) plot
1819	if gen.searchInlist(dplot[0],'lon'):
1820	spacedim ='lon'
1821	axisvals = 'y'
1822	figmid = 'longitudinal'
1823	elif gen.searchInlist(dplot[0],'lat'):
1824	spacedim = 'lat'
1825	axisvals = 'x'
1826	figmid = 'meridional'
1827	elif gen.searchInlist(dplot[0],'pres'):
1828	spacedim = 'pres'
1829	axisvals = 'x'
1830	figmid = 'vertical'
1831	else:
1832	print errmsg
1833	print ' ' + fname + ": in '2lines' only ready for: 'lon', 'lat'," + \
1834	" 'pres' as common dimension !!"
1835	print ' dimensions in plot:', dplot[0]
1836	quit(-1)
1837
1838	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
1839	'!'.join(tfig.split('!')[2:])
1840
1841	graphvals = spacedim + ':' + Arange + ':' + Brange + ':Extrs:' + \
1842	axisvals + ':' + ','.join(CFvplot) + ':' + Aline + ',' + Bline + \
1843	':2.,2.:' + Akind + ',' + Bkind + ':2.,2.:,:' + figtit + ':' + \
1844	spacedim + ':0\|auto:' + finame.replace('.'+kfig, '') + ':' + kfig + \
1845	':True'
1846
1847	fvarS = ','.join(vplot)
1848
1849	drwins = 'draw_2lines'
1850	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
1851	" -S '" + graphvals + "' -v " + fvarS
1852
1853	try:
1854	with gen.Capturing() as output:
1855	sout = sub.call(plotins, shell=True)
1856	except:
1857	print errmsg
1858	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
1859	print sout
1860	for s1out in output: print s1out
1861	quit(-1)
1862
1863	# keeping all figures
1864	trkobjf.write('\n')
1865	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
1866	trkobjf.write(plotins + '\n')
1867
1868	elif kplot == '2linesTime':
1869	figtit = tfig.replace('!','\|')
1870
1871	lineAstdn = CFvplot[0]
1872	lineBstdn = CFvplot[1]
1873	figfs = ','.join(fplot)
1874
1875	lineA = pplot[0]
1876	Arange = str(lineA[0]) + ',' + str(lineA[1])
1877	# Values are changed from specific values in `model_graphics.dat'
1878	if lineA[3].find('%') == -1:
1879	Aline = lineA[2]
1880	Akind = lineA[3]
1881	Amark = lineA[4]
1882	Asize = lineA[5]
1883	else:
1884	Aline = 'red'
1885	Akind = '-'
1886	Amark = ','
1887	Asize = '2.'
1888
1889	lineB = pplot[1]
1890	Brange = str(lineB[0]) + ',' + str(lineB[1])
1891	# Values are changed from specific values in `model_graphics.dat'
1892	if lineB[3].find('%') == -1:
1893	Bline = lineB[2]
1894	Bkind = lineB[3]
1895	Bmark = lineB[4]
1896	Bsize = lineB[5]
1897	else:
1898	Bline = 'blue'
1899	Bkind = '-'
1900	Bmark = ','
1901	Bsize = '2.'
1902
1903	timeaxis = 'x'
1904	figmid = 'evolution'
1905
1906	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
1907	'!'.join(tfig.split('!')[2:])
1908
1909	graphvals = 'time:' + Arange + ':' + Brange + ':' + timekind + ';' + \
1910	timefmt + ':' + timeaxis + ':' + ','.join(CFvplot) + ':' + Aline + ',' \
1911	+ Bline + ':2.,2.:' + Akind + ',' + Bkind + ':2.,2.:,:' + figtit + ':' \
1912	+ timelabel + ':0\|auto:' + finame.replace('.'+kfig, '') + ':' + \
1913	kfig + ':True'
1914
1915	fvarS = ','.join(vplot)
1916
1917	drwins = 'draw_2lines_time'
1918	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
1919	" -S '" + graphvals + "' -v " + fvarS
1920
1921	try:
1922	with gen.Capturing() as output:
1923	sout = sub.call(plotins, shell=True)
1924	except:
1925	print errmsg
1926	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
1927	print sout
1928	for s1out in output: print s1out
1929	quit(-1)
1930
1931	# keeping all figures
1932	trkobjf.write('\n')
1933	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
1934	trkobjf.write(plotins + '\n')
1935
1936	elif kplot == 'Nlines':
1937	figtit = tfig.replace('!','\|')
1938
1939	linestdn = CFvplot[0]
1940	figfs = ','.join(fplot)
1941
1942	Nlines = len(fplot)
1943
1944	lablines = []
1945	for il in range(Nlines):
1946	linev = pplot[il]
1947	# Values are changed from specific values in `model_graphics.dat'
1948	if linev[3].find('%') == -1:
1949	line = linev[2]
1950	kind = linev[3]
1951	mark = linev[4]
1952	size = linev[5]
1953	if il == 0:
1954	vrange = str(linev[0]) + ',' + str(linev[1])
1955	kinds = kind
1956	lines = line
1957	marks = mark
1958	sizes = str(size)
1959	else:
1960	kinds = kinds + ',' + kind
1961	lines = lines + ',' + line
1962	marks = marks + '@' + mark
1963	sizes = sizes + ',' + str(size)
1964	else:
1965	lines = 'None'
1966	kinds = '-'
1967	marks = ','
1968	sizes = '2.'
1969
1970	secsf = fplot[il].split('/')
1971	Nsecsf = len(secsf)
1972	expvs = expgraphc[secsf[Nsecsf-3] + '/' + secsf[Nsecsf-2]]
1973	lablines.append(expvs.label)
1974
1975	# It is assumed that if the space variable is 'lon': is desired a
1976	# (lon, vals) plot if it is 'lat': then (vals, lat) plot
1977	if gen.searchInlist(dplot[0],'lon'):
1978	spacedim ='lon'
1979	sdlab ='lon ($degrees\ East$)'
1980	axisvals = 'y'
1981	figmid = 'longitudinal'
1982	elif gen.searchInlist(dplot[0],'lat'):
1983	spacedim = 'lat'
1984	sdlab = 'lat ($degrees\ North$)'
1985	axisvals = 'x'
1986	figmid = 'meridional'
1987	elif gen.searchInlist(dplot[0],'pres'):
1988	spacedim = 'pres'
1989	sdlab = 'pres ($Pa$)'
1990	axisvals = 'x'
1991	figmid = 'vertical'
1992	else:
1993	print errmsg
1994	print ' ' + fname + ": in 'Nlines' only ready for: 'lon', 'lat'," + \
1995	" 'pres' as common dimension !!"
1996	print ' dimensions in plot:', dplot[0]
1997	quit(-1)
1998
1999	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
2000	'!'.join(tfig.split('!')[2:])
2001	figtit = figtit.replace('!',' ')
2002	leglabels = ','.join(lablines)
2003
2004	graphvals = spacedim + ':' + axisvals + ':' + sdlab + ':' + leglabels + \
2005	':' + CFvplot[0] +':'+ figtit + ':0\|auto:' + lines +':'+ kinds + ':' + \
2006	marks +':'+ sizes +':'+ sizes +':all:'+ finame.replace('.'+kfig, '') + \
2007	':' + kfig + ':True'
2008
2009	fvarS = vplot[0]
2010
2011	drwins = 'draw_lines'
2012	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2013	" -S '" + graphvals + "' -v " + fvarS
2014
2015	try:
2016	with gen.Capturing() as output:
2017	sout = sub.call(plotins, shell=True)
2018	except:
2019	print errmsg
2020	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2021	print sout
2022	for s1out in output: print s1out
2023	quit(-1)
2024
2025	# keeping all figures
2026	trkobjf.write('\n')
2027	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2028	trkobjf.write(plotins + '\n')
2029
2030	elif kplot == 'Nlines_time':
2031	figtit = tfig.replace('!','\|')
2032
2033	linestdn = CFvplot[0]
2034	figfs = ','.join(fplot)
2035
2036	Nlines = len(fplot)
2037
2038	lablines = []
2039	for il in range(Nlines):
2040	linev = pplot[il]
2041	# Values are changed from specific values in `model_graphics.dat'
2042	if linev[3].find('%') == -1:
2043	line = linev[2]
2044	kind = linev[3]
2045	mark = linev[4]
2046	size = linev[5]
2047	if il == 0:
2048	vrange = str(linev[0]) + ',' + str(linev[1])
2049	kinds = kind
2050	lines = line
2051	marks = mark
2052	sizes = str(size)
2053	else:
2054	kinds = kinds + ',' + kind
2055	lines = lines + ',' + line
2056	marks = marks + '@' + mark
2057	sizes = sizes + ',' + str(size)
2058	else:
2059	lines = 'None'
2060	kinds = '-'
2061	marks = ','
2062	sizes = '2.'
2063
2064	secsf = fplot[il].split('/')
2065	Nsecsf = len(secsf)
2066	expvs = expgraphc[secsf[Nsecsf-3] + '/' + secsf[Nsecsf-2]]
2067	lablines.append(expvs.label)
2068
2069	valsaxis = 'y'
2070	figmid = 'evolution'
2071
2072	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
2073	'!'.join(tfig.split('!')[2:])
2074	figtit = figtit.replace('!',' ')
2075	leglabels = ','.join(lablines)
2076
2077	graphvals = 'time;' + valsaxis + ';time;' + leglabels + ';' + \
2078	CFvplot[0] + ';' + figtit + ';None;time\|' + '\|'.join(tvals) + \
2079	';0\|auto;' + kfig + ';' + kinds + ';' + lines + ';' + marks + ';' + \
2080	sizes + ';' + sizes + ';all;-1;True'
2081
2082	fvarS = vplot[0]
2083
2084	drwins = 'draw_lines_time'
2085	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2086	" -S '" + graphvals + "' -v " + fvarS
2087
2088	try:
2089	with gen.Capturing() as output:
2090	sout = sub.call(plotins, shell=True)
2091	except:
2092	print errmsg
2093	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2094	print sout
2095	for s1out in output: print s1out
2096	quit(-1)
2097
2098	sout = sub.call('mv lines_time.' + kfig + ' ' + finame, shell=True)
2099
2100	# keeping all figures
2101	trkobjf.write('\n')
2102	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2103	trkobjf.write(plotins + '\n')
2104
2105	elif kplot == 'shadcont2Dsfc':
2106	figtit = tfig.replace('!','\|')
2107	shdstdn = CFvplot[0]
2108	cntstdn = CFvplot[1]
2109	figfs = ','.join(fplot)
2110
2111	shad = pplot[0]
2112	srange = str(shad[0]) + ',' + str(shad[1])
2113	cbar = shad[2]
2114	cnt = pplot[1]
2115	crange = str(cnt[0]) + ',' + str(cnt[1])
2116	cfmt = cnt[3]
2117	ckind = cnt[4]
2118	cline = cnt[5]
2119
2120	graphvals = ','.join(CFvplot)
2121	graphvals = graphvals + ':lon\|-1,lat\|-1:lon\|-1,lat\|-1:lon:lat:auto:' + \
2122	cbar + ',auto,auto:' + ckind + ',' + cline + ':' + cfmt + ':' + \
2123	srange + ':' + crange + ',9:' + figtit + ':' + kfig + ':None:' + \
2124	mapval + ':True'
2125
2126	fvarS = ','.join(vplot)
2127
2128	drwins = 'draw_2D_shad_cont'
2129	plotins = "python " + pyH + "/drawing.py -f " + figfs + " -o " + drwins +\
2130	" -S '" + graphvals + "' -v " + fvarS
2131	try:
2132	with gen.Capturing() as output:
2133	sout = sub.call(plotins, shell=True)
2134	except:
2135	print errmsg
2136	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2137	print sout
2138	for s1out in output: print s1out
2139	quit(-1)
2140
2141	sout = sub.call('mv 2Dfields_shadow-contour.'+kfig+' '+finame, shell=True)
2142
2143	# keeping all figures
2144	trkobjf.write('\n')
2145	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2146	trkobjf.write(plotins + '\n')
2147
2148	elif kplot == 'shadconthovmsfc':
2149	figtit = tfig.replace('!','\|')
2150	shdstdn = CFvplot[0]
2151	cntstdn = CFvplot[1]
2152	figfs = ','.join(fplot)
2153
2154	shad = pplot[0]
2155	srange = str(shad[0]) + ',' + str(shad[1])
2156	cbar = shad[2]
2157	cnt = pplot[1]
2158	crange = str(cnt[0]) + ',' + str(cnt[1])
2159	cfmt = cnt[3]
2160	ckind = cnt[4]
2161	cline = cnt[5]
2162	# It is assumed that if the space variable is 'lon': is desired a
2163	# (lon, time) plot if it is 'lat': then (time, lat) plot
2164	if gen.searchInlist(dplot,'lon'):
2165	spacedim ='lon'
2166	figmid = 'longitudinal\|evolution\|of'
2167	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2168	'\|'.join(tfig.split('!')[2:])
2169	reverse= 'None'
2170	dims= spacedim+'\|-1,time\|-1;'+spacedim+'\|-1,time\|-1;'+spacedim+';time'
2171	else:
2172	spacedim='lat'
2173	figmid = 'meridional\|evolution\|of'
2174	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2175	'\|'.join(tfig.split('!')[2:])
2176	reverse='None'
2177	dims= spacedim+'\|-1,time\|-1;'+spacedim+'\|-1,time\|-1;time;'+spacedim
2178
2179	graphvals = ','.join(CFvplot)
2180	graphvals = graphvals + ';' + dims +';auto;' + cbar + ',auto,auto' + ';' \
2181	+ ckind + ',' + cline + ';' + cfmt + ';' + srange + ';' + crange + \
2182	',9;' + figtit + ';' + kfig + ';' + reverse + ';' + 'time\|' + tunits + \
2183	'\|'+ timekind + '\|' + timefmt + '\|' + timelabel + ';True'
2184
2185	fvarS = ','.join(vplot)
2186
2187	drwins = 'draw_2D_shad_cont_time'
2188	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2189	" -S '" + graphvals + "' -v " + fvarS
2190
2191	try:
2192	with gen.Capturing() as output:
2193	sout = sub.call(plotins, shell=True)
2194	except:
2195	print errmsg
2196	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2197	print sout
2198	for s1out in output: print s1out
2199	quit(-1)
2200
2201	sout = sub.call('mv 2Dfields_shadow-contour.'+kfig+' '+finame, shell=True)
2202
2203	# keeping all figures
2204	trkobjf.write('\n')
2205	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2206	trkobjf.write(plotins + '\n')
2207
2208	elif kplot == 'shadcont2Dzsec':
2209	figtit = tfig.replace('!','\|')
2210	shdstdn = CFvplot[0]
2211	cntstdn = CFvplot[1]
2212	figfs = ','.join(fplot)
2213
2214	shad = pplot[0]
2215	srange = str(shad[0]) + ',' + str(shad[1])
2216	cbar = shad[2]
2217	cnt = pplot[1]
2218	crange = str(cnt[0]) + ',' + str(cnt[1])
2219	cfmt = cnt[3]
2220	ckind = cnt[4]
2221	cline = cnt[5]
2222
2223	# It is assumed that if the space variable is 'lon': is desired a
2224	# (lon, pres) plot it it is 'lat': then (pres, lat) plot
2225	if gen.searchInlist(dplot, 'lon'):
2226	spacedim='lon'
2227	figmid = 'long.\|vert.\|cross\|sec.\|of'
2228	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2229	'\|'.join(tfig.split('!')[2:])
2230	dims= spacedim+'\|-1,pres\|-1:'+spacedim+'\|-1,pres\|-1:'+spacedim+':pres'
2231	else:
2232	spacedim='lat'
2233	figmid = 'mer.\|vert.\|cross\|sec.\|of'
2234	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2235	'\|'.join(tfig.split('!')[2:])
2236	# NOT WORKING? dims= spacedim+'\|-1,pres\|-1:'+spacedim+'\|-1,pres\|-1:'+'pres:'+spacedim
2237	dims= spacedim+'\|-1,pres\|-1:'+spacedim+'\|-1,pres\|-1:'+spacedim+':pres'
2238
2239	reverse = 'flip@y'
2240
2241	graphvals = ','.join(CFvplot)
2242	graphvals = graphvals + ':' + dims + ':auto:' + cbar + ',auto,auto:' + \
2243	ckind + ',' + cline + ':' + cfmt + ':' + srange + ':' + crange + ',9:' \
2244	+ figtit + ':' + kfig + ':' + reverse + ':None:True'
2245
2246	fvarS = ','.join(vplot)
2247
2248	drwins = 'draw_2D_shad_cont'
2249	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs + ' -o ' + drwins + \
2250	" -S '" + graphvals + "' -v " + fvarS
2251	try:
2252	with gen.Capturing() as output:
2253	sout0 = sub.call(plotins, shell=True)
2254	except:
2255	print errmsg
2256	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2257	for s1out in output: print s1out
2258	quit(-1)
2259
2260	sout = sub.call('mv 2Dfields_shadow-contour.'+kfig+' '+finame, shell=True)
2261
2262	# keeping all figures
2263	trkobjf.write('\n')
2264	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2265	trkobjf.write(plotins + '\n')
2266
2267	else:
2268	print errmsg
2269	print " " + fname + ": plot kind '" + kplot + "' not ready !!"
2270	quit(-1)
2271
2272	trkobjf.close()
2273
2274	return
2275
2276	def optitle(op,opexp):
2277	""" Function to creat the operation section in the title of a figure ('!' for spaces)
2278	op= '+' separated list of operations
2279	opexp= dictionary with the `explanation'(text in title) to appear for each operation
2280	>>> optitle('pinterp+xmean+tmean',{'pinterp':'pinterp', 'xmean':'xmean', 'tmean':'tmean'})
2281	$_{[pinterp\!xmean\!tmean]}$
2282	"""
2283	fname = 'optitle'
2284
2285	opvals = op.split('+')
2286	for op1 in opvals:
2287	if not opexp.has_key(op1):
2288	print errmsg
2289	print ' '+fname+": no explanation for operation '"+op1+"' !!"
2290	print ' provided:', opexp.keys()
2291	quit(-1)
2292	if op1 == opvals[0]:
2293	titop = '$_{[' + opexp[op1].replace(' ','!')
2294	if len(opvals) == 1: titop = titop + ']}$'
2295	elif op1 == opvals[len(opvals)-1]:
2296	titop = titop + '\!' + opexp[op1].replace(' ','!') + ']}$'
2297	else:
2298	titop = titop + '\!' + opexp[op1].replace(' ','!')
2299
2300	return titop
2301
2302	def create_figure_title(mod,exp,varops,explop):
2303	""" Function to create the title of a figure ('!' for spaces)
2304	mod: model name
2305	exp: experiment name
2306	varops: list with [var]\|[op] values from wich create the title
2307	explop: dictionary with the `explanation'(text in title) to appear for each operation
2308	>>> expops = {'pinterp':'pinterp', 'xmean':'xmean', 'tmean':'tmean', 'last':'last'}
2309	>>> create_figure_title('WRF','current',['ua\|pinterp+xmean+tmean', 'va\|pinterp+xmean+tmean'], expops)
2310	WRF!current!ua!&!va$_{[pinterp\!xmean\!tmean]}$
2311	>>> create_figure_title('WRF','current',['ua\|pinterp+xmean+tmean', 'ua\|pinterp+xmean+last'], expops)
2312	WRF!current!ua$_{[pinterp\!xmean\!tmean]}$!&!!$_{[pinterp\!xmean\!last]}$
2313	>>> create_figure_title('WRF','current',['ua\|pinterp+xmean+tmean', 'va\|pinterp+xmean+last'], expops)
2314	WRF!current!ua$_{[pinterp\!xmean\!tmean]}$!&!va$_{[pinterp\!xmean\!last]}$
2315	"""
2316	fname = 'create_figure_title'
2317
2318	titfig = mod + '!' + exp + '!'
2319
2320	varns = []
2321	opns = []
2322	for varop in varops:
2323	vn = varop.split('\|')[0]
2324	op = varop.split('\|')[1]
2325	varns.append(vn)
2326	opns.append(op)
2327	if varop == varops[0]:
2328	allvarequal = True
2329	allopequal = True
2330	varinfig = vn
2331	opinfig = op
2332	else:
2333	if vn != varinfig: allvarequal = False
2334	if op != opinfig: allopequal = False
2335
2336	Nvars = len(varns)
2337	Nops = len(opns)
2338
2339	if allvarequal:
2340	titfig = titfig + '!' + varns[0]
2341	if allopequal:
2342	opS = optitle(op,explop)
2343	titfig = titfig + opS
2344	else:
2345	for opn in opns:
2346	opS = optitle(opn,explop)
2347	if opn == opns[0]:
2348	titfig = titfig + opS
2349	elif opn == opns[Nops-1]:
2350	titfig = titfig + '!&!' + opS
2351	else:
2352	titfig = titfig + ',!' + opS
2353	else:
2354	if allopequal:
2355	opS = optitle(op,explop)
2356	for vn in varns:
2357	if vn == varns[0]:
2358	titfig = titfig + '!' + vn
2359	elif vn == varns[Nvars-1]:
2360	titfig = titfig + '!&!' + vn
2361	else:
2362	titfig = titfig + ',!' + vn
2363	titfig = titfig + opS
2364	else:
2365	for ivop in range(Nvars):
2366	vn = varns[ivop]
2367	op = opns[ivop]
2368	vnop = vn + optitle(op,explop)
2369
2370	if vn == varns[0]:
2371	titfig = titfig + '!' + vnop
2372	elif vn == varns[Nvars-1]:
2373	titfig = titfig + '!&!' + vnop
2374	else:
2375	titfig = titfig + ',!' + vnop
2376
2377	return titfig
2378
2379	def draw_plots(config, plots, mod, exp, odir, allvarcomp, figscr, debug):
2380	""" Function to draw all plots
2381	config= Configuration of the experiment
2382	plots= dictionary with the plots
2383	mod= model of the plots
2384	exp= experiment of the plots
2385	odir= output experiment folder
2386	allvarcomp= dictionary with all the variables to compute and their information
2387	figscr= whether figures should be done from the scratch or not
2388
2389	* Plot as
2390	{[kplot]} = [varn1]\|[op1]#[varn2]\|[op2]#[...[varnN]\|[opN]], ...
2391	[kplot] ___
2392	diffmap2Dsfc: 2D map of surface differences values of 1 variable
2393	diffmap2Dz: 2D map of 3D differences values of 1 variable
2394	map2Dsfc: 2D map of surface values of 1 variable
2395	map3D: 2D map of 3D values of 1 variable
2396	shadconthovmsfc: Hovmoeller diagrams of 2 variable at the surface in shadow and the other in contourn
2397	shadcont2Dsfc: 2D map of shadow (1st variable) and countour (2nd variable) [stvar1]#[stvar2]
2398	shadcont2Dzsec: 2D map of vertical section of 2 variables one in shadow and the other in contourn
2399	[varn]
2400	variable
2401	[op]
2402	'+' separated list of operations
2403	in figures with more than 1 variable, use '#' to separate the [varn]\|[op]
2404	"""
2405	fname = 'draw_plots'
2406
2407	os.chdir(odir)
2408
2409	# Dictionary with the operations with surnames for the operated variable
2410	opersurnames = {}
2411	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
2412	for opsr in opsur.keys():
2413	opn = opsr.split('_')[1]
2414	vls = opsur[opsr].split(':')
2415	opersurnames[opn] = vls
2416
2417	# time values
2418	# Units time for the plots
2419	rd = config['CFreftime']
2420	tunits = config['CFunitstime'] + '!since!' + rd[0:4] + '-' + rd[4:6] + '-' + \
2421	rd[6:8] + '!' + rd[8:10] + ':' + rd[10:12] + ':' + rd[12:14]
2422	# time ticks kind
2423	tkind = config['timekind']
2424	# time ticks format
2425	tfmt = config['timefmt']
2426	# time axis label
2427	tlab = config['timelabel']
2428	timevals = [tunits, tkind, tfmt, tlab]
2429
2430	# Dictionary of plot specificities
2431	specplotkeyn = 'specificvarplot'
2432	plotspecifics = {}
2433	if config.has_key(specplotkeyn):
2434	# [minval]: minimum value
2435	# [maxval]: minimum value
2436	# [colorbar]: name of the colorbar (from matplotlib) to use
2437	# [cntformat]: format of the contour labels
2438	# [colorcnt]: color for the countor lines
2439	plotspecs = config[specplotkeyn].split(':')
2440	for pltspc in plotspecs:
2441	pltvls = pltspc.split('\|')
2442	vn = pltvls[0]
2443	op = pltvls[1]
2444	fn = pltvls[2]
2445	plotspecifics[fn + '_' + vn + '_' + op] = pltvls[3:]
2446	if debug:
2447	print 'Specific values for plots _______'
2448	gen.printing_dictionary(plotspecifics)
2449
2450	# Kind of figures
2451	kindfigure = config['kindfig']
2452
2453	# Graphical labels and configuration
2454	modgc, expgc = graphical_characteristics(config, debug)
2455	modv = modgc[mod]
2456	modlab = modv.label
2457	expv = expgc[mod+'/'+exp]
2458	explab = expv.label
2459
2460	# Map value
2461	mapvalue = config['mapval']
2462
2463	# pythone scripts HOME
2464	pyHOME = config['pyHOME']
2465
2466	# Title-text of operations
2467	opexplained = {}
2468	optits = config['titleoperations'].split(':')
2469	for optit in optits:
2470	opn = optit.split('\|')[0]
2471	opt = optit.split('\|')[1]
2472	opexplained[opn] = opt
2473	if debug:
2474	print 'Titles for operations _______'
2475	gen.printing_dictionary(opexplained)
2476
2477	for kplot in plots.keys():
2478	varsplt = plots[kplot]
2479	for varplt in varsplt:
2480	if debug:
2481	print " printing '" + kplot + "' var ':" + varplt + "'..."
2482	varops = varplt.split('#')
2483
2484	# CF variables in plot
2485	CFvarsplot = []
2486	# Files in plot
2487	filesplot = []
2488	# Variables in plot within the files
2489	varsplot = []
2490	# Dims in figure
2491	dimsplot = []
2492	# pictoric values in figure
2493	pictplot = []
2494	# Name of the figure
2495	figname = ''
2496	# Title of the figure
2497	titfigure = ''
2498
2499	ivp = 0
2500	for varop in varops:
2501	vn = varop.split('\|')[0]
2502	op = varop.split('\|')[1]
2503
2504	# CF variables in plot
2505	CFvarsplot.append(vn)
2506
2507	vnopS = vn + '_' + op
2508	if not allvarcomp.has_key(vnopS):
2509	print errmsg
2510	print ' ' + fname + ": no file for variable-operation '" + \
2511	vnopS + "' !!"
2512	vopvals = allvarcomp[vnopS]
2513	headf = vopvals[0]
2514	globalP = vopvals[3]
2515	gP = pinterpS(globalP)
2516
2517	filen = vn + '_' + headf + gP + '_' + op.replace('+','_') + '.nc'
2518	filesplot.append(filen)
2519	# Do we have processed the given variable?
2520	if not os.path.isfile(filen):
2521	print warnmsg
2522	print " " + fname + ": there is no file for variable '" + varop \
2523	+ "' skiping it !!"
2524	break
2525
2526	# Name of the variable inside the file
2527	vnsur = varnoper(vn, op, opersurnames)
2528	varsplot.append(vnsur)
2529
2530	# Dimensions in file
2531	try:
2532	with gen.Capturing() as output:
2533	dims = ncvar.idims(filen)
2534	except:
2535	print errmsg
2536	print 'ncvar.idims('+filen+')'
2537	for s1out in output: print s1out
2538	quit(-1)
2539
2540	dimsplot.append(dims)
2541
2542	# pictoric values for the figure
2543	Sfivaop = kplot + '_' + vn + '_' + op
2544	if plotspecifics.has_key(Sfivaop):
2545	pictvals = plotspecifics[Sfivaop]
2546	else:
2547	Vvals = gen.variables_values(vn)
2548	pictvals = [Vvals[2], Vvals[3], Vvals[6], '%g', 'fixc', 'black']
2549
2550	pictplot.append(pictvals)
2551
2552	# Header of the name of the figure
2553	if ivp == 0:
2554	figname = kplot +'_'+ vn + '_' + headf + '_' + op.replace('+','-')
2555	else:
2556	figname = figname + '-'+vn+'_'+headf+'_'+op.replace('+','-')
2557
2558	ivp = ivp + 1
2559	# End of variable-operation
2560	figname = figname + '.' + kindfigure
2561
2562	# Title of figure
2563	titfigure = create_figure_title(modlab, explab, varops, opexplained)
2564
2565	draw_plot(kplot, CFvarsplot, filesplot, varsplot, dimsplot, pictplot, \
2566	figname, titfigure, kindfigure, mapvalue, timevals, expgc, odir, \
2567	pyHOME, figscr, debug)
2568
2569	# End of variables-operations
2570
2571	# End of kind of plots
2572
2573	return
2574
2575	def get_differences_var(vc,kdiff,db):
2576	""" Function to provide the operations to make for each variable from kdiff ('DIFFOP_' or 'DIFFVAR_') dictionary
2577	vc= kdiff dictionary, dictionary with all the parameters started with kdiff
2578	kdiff= kind of differences 'op', 'var'
2579	"""
2580	fname = 'get_differences_var'
2581
2582	if kdiff == 'op':
2583	diffn = 'DIFFOP'
2584	elif kdiff == 'var':
2585	diffn = 'DIFFVAR'
2586	else:
2587	print errmsg
2588	print ' ' + fname + ": differences '" + kdiff + "' not ready !!"
2589	quit(-1)
2590	LD = len(diffn + '_')
2591
2592	iop = 1
2593	# list of operations by difference
2594	doopers = {}
2595	# operations by variable
2596	operationsvar = {}
2597	# individual operations by variable (not repeated)
2598	indivoperationsvar = {}
2599
2600	# Variables with a calculation whic requires vertical interpolation to all the
2601	# file (operations stating by [kdiff]'_pinterp')
2602	LVp = len(diffn + '_pinterp')
2603	varglobalp = []
2604
2605	for oper in vc.keys():
2606	Loper = len(oper)
2607	opn = oper[LD:Loper+1]
2608	vns = vc[oper].split(':')
2609	ops1 = opn.split('+')
2610	doopers[opn] = ops1
2611	for vn in vns:
2612	if not operationsvar.has_key(vn):
2613	operationsvar[vn] = [opn]
2614	indivoperationsvar[vn] = ops1
2615	else:
2616	opers = operationsvar[vn]
2617	opers.append(opn)
2618	operationsvar[vn] = opers
2619	opers1 = indivoperationsvar[vn]
2620	for op1 in ops1:
2621	if not gen.searchInlist(opers1, op1):
2622	opers1.append(op1)
2623	indivoperationsvar[vn] = opers1
2624
2625	if oper[0:LVp] == diffn + '_pinterp':
2626	for vn in vns:
2627	if not gen.searchInlist(varglobalp,vn): varglobalp.append(vn)
2628
2629	iop = iop + 1
2630
2631	if db:
2632	print ' operations to make _______'
2633	gen.printing_dictionary(doopers)
2634	print ' operations by variable _______'
2635	gen.printing_dictionary(operationsvar)
2636	print ' individual operations by variable _______'
2637	gen.printing_dictionary(indivoperationsvar)
2638	print ' variables with the vertical interpolation for all the file _______'
2639	print ' #', varglobalp
2640
2641	return doopers, operationsvar, indivoperationsvar, varglobalp
2642
2643	def diffvarop_listconstruct(config, mods, exps, reprj, odir, debug):
2644	""" Function to create the list of differences to compute and draw
2645	config= Configuration of the experiment
2646	mods= list with the models to use
2647	exps= list with the experiments to use
2648	reprj= list with whether re-rpojection needs to be done
2649	odir= output differences folder
2650	"""
2651	fname='diffvarop_listconstruct'
2652
2653	diffop = gen.get_specdictionary_HMT(config, H='DIFFOP_')
2654	diffvar = gen.get_specdictionary_HMT(config, H='DIFFVAR_')
2655
2656	if debug:
2657	if diffop is not None:
2658	print " 'op' differences ________"
2659	gen.printing_dictionary(diffop)
2660	if diffvar is not None:
2661	print " 'var' differences ________"
2662	gen.printing_dictionary(diffvar)
2663
2664	# Getting variable characteristics from each model/experiment
2665	idir1 = config['ofold'] + '/' + mods[0] + '/' + exps[0]
2666	idir2 = config['ofold'] + '/' + mods[1] + '/' + exps[1]
2667
2668	fvarcomp = 'varcompute.inf'
2669	if reprj[0]: fvc1 = 'reproj_' + fvarcomp
2670	else: fvc1 = fvarcomp
2671
2672	if reprj[1]: fvc2 = 'reproj_' + fvarcomp
2673	else: fvc2 = fvarcomp
2674
2675	files1, testfiles1, allcompvar1, Nvar1= read_varcomp_file(idir1 + '/' + fvc1)
2676	files2, testfiles2, allcompvar2, Nvar2= read_varcomp_file(idir2 + '/' + fvc2)
2677
2678	diffops = {}
2679	if diffop is not None:
2680	# Getting 'op' differences characteristics
2681	doops, opvar, indivopvar, varglobalp = get_differences_var(diffop,'op',debug)
2682
2683	Ndiffop = 0
2684	for vn in opvar.keys():
2685	vnops = opvar[vn]
2686	for op in vnops:
2687	vnop = vn + '_' + op
2688	if not allcompvar1.has_key(vnop):
2689	print warnmsg
2690	print ' ' + fname + ": DIFFOP variable+operation '" + vnop + \
2691	"' in '" + mods[0] + '/' + exps[0] + "' was not computed !!"
2692	print ' skipping it!'
2693	print ' variables computed:', allcompvar1
2694	break
2695	if not allcompvar2.has_key(vnop):
2696	print warnmsg
2697	print ' ' + fname + ": DIFFOP variable+operation '" + vnop + \
2698	"' in '" + mods[1] + '/' + exps[1] + "' was not computed !!"
2699	print ' skipping it!'
2700	print ' variables computed:', allcompvar2
2701	break
2702	vals1 = allcompvar1[vnop]
2703	vals2 = allcompvar2[vnop]
2704
2705	headerf1 = vals1[0]
2706	headerf2 = vals2[0]
2707
2708	diffops[vnop] = [mods[0],mods[1],exps[0],exps[1],headerf1,headerf2]
2709
2710	Ndiffop = Ndiffop + 1
2711	else:
2712	Ndiffop = 0
2713
2714	diffvars = {}
2715	if diffvar is not None:
2716	# Getting 'var' differences characteristics
2717	doops,opvar,indivopvar,varglobalp = get_differences_var(diffvar,'var',debug)
2718
2719	Ndiffvar = 0
2720	for vn in opvar.keys():
2721	vnops = opvar[vn]
2722	for op in vnops:
2723	vnop = vn + '_' + op
2724	if not allcompvar1.has_key(vnop):
2725	print warnmsg
2726	print ' ' + fname + ": DIFFVAR variable+operation '" + vnop + \
2727	"' in '" + mods[0] + '/' + exps[0] + "' was not computed !!"
2728	print ' skipping it!'
2729	print ' variables computed:', allcompvar1
2730	break
2731	if not allcompvar2.has_key(vnop):
2732	print warnmsg
2733	print ' ' + fname + ": DIFFVAR variable+operation '" + vnop + \
2734	"' in '" + mods[1] + '/' + exps[1] + "' was not computed !!"
2735	print ' skipping it!'
2736	print ' variables computed:', allcompvar2
2737	break
2738	vals1 = allcompvar1[vnop]
2739	vals2 = allcompvar2[vnop]
2740
2741	headerf1 = vals1[0]
2742	headerf2 = vals2[0]
2743
2744	diffvars[vnop] = [mods[0],mods[1],exps[0],exps[1],headerf1,headerf2]
2745
2746	Ndiffvar = Ndiffvar + 1
2747	else:
2748	Ndiffvar = 0
2749
2750	if diffop is not None:
2751	Sopdiffs = gen.dictKeysVals_stringList(diffops)
2752	else:
2753	Sopdiffs = 'none'
2754	# Outwritting the op diffferences file to avoid next time
2755	objf = open(odir + '/diffop.inf', 'w')
2756	objf.write('differences: ' + Sopdiffs + '\n')
2757	objf.write('Ndiff: ' + str(Ndiffop) + '\n')
2758	objf.close()
2759
2760	if diffvar is not None:
2761	Svardiffs = gen.dictKeysVals_stringList(diffvars)
2762	else:
2763	Svardiffs = 'none'
2764	# Outwritting the var diffferences file to avoid next time
2765	objf = open(odir + '/diffvar.inf', 'w')
2766	objf.write('differences: ' + Svardiffs + '\n')
2767	objf.write('Ndiff: ' + str(Ndiffvar) + '\n')
2768	objf.close()
2769
2770	return diffops, diffvars, Ndiffop, Ndiffvar
2771
2772	def read_diff_file(difffile):
2773	""" Function to read the file with the information about the differences
2774	difffile= file with the information
2775	"""
2776	fname = 'read_diff_file'
2777
2778	if not os.path.isfile(difffile):
2779	print errormsg
2780	print ' ' + fname + ": differences file '" + difffile + "' does not exist !!"
2781	quit(-1)
2782
2783	objf = open(difffile, 'r')
2784	for line in objf:
2785	if line[0:1] != '#' and len(line) > 1:
2786	values = line.replace('\n','').split(' ')
2787	if values[0] == 'differences:':
2788	if values[1] != 'none':
2789	diffs = gen.stringList_dictKeysVals(values[1])
2790	else:
2791	diffs = {}
2792	elif values[0] == 'Ndiff:':
2793	Ndiffs = int(values[1])
2794
2795	objf.close()
2796
2797	return diffs, Ndiffs
2798
2799	def compute_op_diffs(config, diffallop, odir, diffscr, debug):
2800	""" Function to compute operation differences
2801	config= experiment configuration
2802	diffallop= dictonary with the differences to compute
2803	odir= output folder
2804	diffscr= wether it should be done from the scratch
2805	"""
2806	fname = 'compute_op_diffs'
2807
2808	# output folder
2809	ofold = config['ofold']
2810
2811	# Home of the python scripts
2812	pyH = config['pyHOME']
2813
2814	# opsur = operation surnames
2815	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
2816	Opsurs = {}
2817	for opk in opsur:
2818	opn = opk.split('_')[1]
2819	vals = opsur[opk]
2820	Opsurs[opn] = vals.split(':')
2821
2822	# Getting in working dir
2823	os.chdir(odir)
2824
2825	# CF dimensions
2826	CFdims = ['lon', 'lat', 'pres', 'time']
2827
2828	# Trakcing file
2829	trkobjf = open(odir + '/all_diffop.inf', 'a')
2830
2831	for vnop in diffallop.keys():
2832	vn = vnop.split('_')[0]
2833	op = vnop.split('_')[1]
2834	opS = op.replace('+','_')
2835
2836	# `p' header add related to 'pinterp'
2837	if opS.find('pinterp') != -1: SgP = 'p'
2838	else: SgP = ''
2839
2840	# Variable name within the file
2841	vninF = varnoper(vn, op, Opsurs)
2842
2843	vals = diffallop[vnop]
2844	mod1 = vals[0]
2845	mod2 = vals[1]
2846	exp1 = vals[2]
2847	exp2 = vals[3]
2848	headerf1 = vals[4]
2849	headerf2 = vals[5]
2850	modexpdiff = mod2 + '/' + exp2 + ' - ' + mod1 + '/' + exp1
2851	modexpdiffS = mod2 + '-' + exp2 + '_' + mod1 + '-' + exp1
2852
2853	ifile1 = ofold + '/' + mod1 + '/' + exp1 + '/' + vn + '_' + headerf1 + SgP + \
2854	'_' + opS + '.nc'
2855	ifile2 = ofold + '/' + mod2 + '/' + exp2 + '/' + vn + '_' + headerf2 + SgP + \
2856	'_' + opS + '.nc'
2857	difffilen = odir + '/' + vn + '_diffop_' + modexpdiffS + '_' + opS + '.nc'
2858
2859	if diffscr:
2860	sout = sub.call('rm ' + difffilen + ' >& /dev/null', shell=True)
2861
2862	if not os.path.isfile(difffilen):
2863	if debug:
2864	print " Computing operation difference '" + op + "' of '" + vn + \
2865	"' for: '" + modexpdiff
2866	print " var in file: '" + vninF + "'"
2867
2868	dfiles = 'add\|' + ifile2 + '\|' + vninF + ',sub\|' + ifile1 + '\|' + vninF
2869	values = '\|'.join(CFdims) + '@' + dfiles
2870	pyins = 'python ' + pyH + "/nc_var.py -o compute_opersvarsfiles -S '" + \
2871	values + "' -v " + vninF
2872
2873	try:
2874	with gen.Capturing() as output:
2875	ncvar.compute_opersvarsfiles(values,vninF)
2876	except:
2877	print errmsg
2878	print 'ncvar.compute_opersvarsfiles(' + values + ',' + vninF + ')'
2879	for s1out in output: print s1out
2880	quit(-1)
2881
2882	sout = sub.call('mv opersvarsfiles_'+vninF+'.nc ' + difffilen, shell=True)
2883
2884	# keeping all differences
2885	trkobjf.write('\n')
2886	trkobjf.write("#" + vn + ' ' + op + '\n')
2887	trkobjf.write( pyins + '\n')
2888
2889	trkobjf.close()
2890
2891	return
2892
2893	def compute_diff_statistics(Ecnf, filen, ofold, varn, Opers, scr, db):
2894	""" Function to compute statistics from var diff files
2895	Ecnf= dictionary with the configuration of the experiment
2896	filen= variable difference file to use
2897	ofold= directory to write the output files
2898	varn= name of the variable
2899	Opers= kind of operation: (as possible multiple consecutive combination of operations separated by '+'
2900	[calc1]+[calc2] will compute first [calc1] and then [calc2]
2901	acc: temporal accumulated values
2902	diff: differences between models
2903	direct: no statistics
2904	last: last temporal value
2905	Lmean: latitudinal mean values
2906	Lsec: latitudinal section (latitudinal value must be given, [var]@[lat])
2907	lmean: longitudinal mean values
2908	lsec: longitudinal section (longitudinal value must be given, [var]@[lat])
2909	pinterp: pressure interpolation (to the given $plevels)
2910	tmean: temporal mean values
2911	tstd: temporal standard deviation values
2912	tturb: Taylor's turbulence decomposition value (x - <x>) for time
2913	tvar: temporal variance values
2914	xmean: x-axis mean values
2915	xvar: x-axis variance values
2916	ymean: y-axis mean values
2917	zsum: vertical aggregated values
2918	scr= should it be done from the scratch?
2919	"""
2920	fname='compute_diff_statistics'
2921
2922	# Full header of file
2923	Lfilen = len(filen)
2924	Hfile = filen[0:Lfilen-3]
2925
2926	# CF dimension-variables name
2927	OpersS = '+'.join(Opers)
2928	if OpersS.find('pinterp') != -1 or filen.find('pinterp') != -1:
2929	varnCFs = ['lon', 'lat', 'pres', 'time']
2930	else:
2931	varnCFs = ['lon', 'lat', 'time']
2932
2933	# Variable-dimension CF dictionary
2934	CFdimvardict = {'lon': 'lon', 'lat': 'lat', 'pres': 'pres', 'time': 'time'}
2935
2936	# Python scripts HOME
2937	pyH = Ecnf['pyHOME']
2938
2939	# opsur = operation surnames
2940	opsur = gen.get_specdictionary_HMT(Ecnf, H='opsur_',M='',T='')
2941	Opsurs = {}
2942	for opk in opsur:
2943	opn = opk.split('_')[1]
2944	vals = opsur[opk]
2945	Opsurs[opn] = vals.split(':')
2946
2947	# Getting in working dir
2948	os.chdir(ofold)
2949
2950	# File to keep track of all the statistics of var differences
2951	otrackf = open(ofold + '/all_vardiffstatistics.inf', 'a')
2952
2953	if db:
2954	print " computing statistics of diff variable: '", varn ,"' operation '"+ \
2955	OpersS + "' using file '" + filen + " ...'"
2956
2957	for op in Opers:
2958	# File name from previous operations, and name of the variable within the
2959	# file (some operations change the name of the variable)
2960	if op == Opers[0]:
2961	Fopers = op
2962	prevfile = filen
2963	vninF = varn
2964	else:
2965	Fopers = Fopers + '_' + op
2966	prevfile = fileon
2967	fileon = Hfile + '_' + Fopers + '.nc'
2968
2969	SvarnCFs = ',' + ','.join(varnCFs)
2970	CFvarnp = vninF + SvarnCFs
2971
2972	if scr:
2973	sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
2974
2975	# Just produce file in case output file does not exist
2976	if not os.path.isfile(fileon):
2977	if db:
2978	print ' ' + fname + ": creation of file '" + fileon + "' ..."
2979	dofile = True
2980	else:
2981	dofile = False
2982
2983	# Variables to be kept in the final file
2984	varkeep = []
2985
2986	if op == 'acc':
2987	# temporal accumulated values
2988	print " " + fname + ": kind '" + op + "' not ready !!"
2989	wrongstats.append(varn + '_' + opers)
2990	break
2991	elif op == 'direct':
2992	# no statistics
2993	if dofile:
2994	sout = sub.call('mv ' + prevfile + ' ' + fileon, shell=True)
2995	elif op == 'last':
2996	# last temporal value
2997	vals='time,-9,0,0'
2998	if dofile:
2999	try:
3000	with gen.Capturing() as output:
3001	ncvar.DataSetSection(vals,prevfile)
3002	except:
3003	print errmsg
3004	print 'DataSetSection('+vals+',', prevfile, ')'
3005	for s1out in output: print s1out
3006	quit(-1)
3007
3008	if db:
3009	for s1out in output: print s1out
3010
3011	hprevfile = prevfile[0:len(prevfile)-3]
3012
3013	sout = sub.call('mv ' + hprevfile + '_time_B-9-E0-I0.nc ' + fileon, \
3014	shell=True)
3015
3016	# Keeping the operations
3017	pyins=pyH + "/nc_var.py -o DataSetSection -S '" + vals + "' -f " + \
3018	prevfile
3019	otrackf.write("\n")
3020	otrackf.write("# " + varn + " " + Fopers + "\n")
3021	otrackf.write(pyins + "\n")
3022
3023	elif op =='Lmean':
3024	# latitudinal mean values
3025	print " " + fname + ": kind '" + op + "' not ready !!"
3026	wrongstats.append(varn + '_' + opers)
3027	break
3028	elif op =='Lsec':
3029	# latitudinal section (latitudinal value must be given, [var]@[lat])
3030	print " " + fname + ": kind '" + op + "' not ready !!"
3031	wrongstats.append(varn + '_' + opers)
3032	break
3033	elif op =='lmean':
3034	# longitudinal mean values
3035	print " " + fname + ": kind '" + op + "' not ready !!"
3036	wrongstats.append(varn + '_' + opers)
3037	break
3038	elif op =='lsec':
3039	# longitudinal section (longitudinal value must be given, [var]@[lon])
3040	print " " + fname + ": kind '" + op + "' not ready !!"
3041	wrongstats.append(varn + '_' + opers)
3042	break
3043	elif op == 'tmean':
3044	# temporal mean values
3045	vals='time\|-1,time,mean,' + ':'.join(varnCFs)
3046	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3047	if dofile:
3048	try:
3049	with gen.Capturing() as output:
3050	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3051	except:
3052	print errmsg
3053	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3054	for s1out in output: print s1out
3055	quit(-1)
3056
3057	if db:
3058	for s1out in output: print s1out
3059
3060	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
3061
3062	# Keeping the operations
3063	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3064	"' -f " + prevfile + " -v " + CFvarnp
3065	otrackf.write("\n")
3066	otrackf.write("# " + varn + " " + Fopers + "\n")
3067	otrackf.write(pyins + "\n")
3068
3069	# removing dimension variable-dimension 'time'
3070	varnCFs.remove('time')
3071
3072	varkeep.append('timestats')
3073
3074	elif op == 'tstd':
3075	# temporal standard deviation values
3076	vals='time\|-1,time,std,' + ':'.join(varnCFs) + ':' + vdnz
3077	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
3078	if dofile:
3079	try:
3080	with gen.Capturing() as output:
3081	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3082	except:
3083	print errmsg
3084	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3085	for s1out in output: print s1out
3086	quit(-1)
3087
3088	if db:
3089	for s1out in output: print s1out
3090
3091	sout = sub.call('mv file_oper_alongdims_std.nc '+fileon, shell=True)
3092
3093	# Keeping the operations
3094	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3095	"' -f " + prevfile + " -v " + CFvarnp
3096	otrackf.write("\n")
3097	otrackf.write("# " + varn + " " + Fopers + "\n")
3098	otrackf.write(pyins + "\n")
3099
3100	# removing dimension variable-dimension 'time'
3101	varnCFs.remove('time')
3102
3103	varkeep.append('timestats')
3104
3105	elif op == 'tturb':
3106	# temporal turbulent values
3107	vals='time\|-1,time,turb,' + ':'.join(varnCFs)
3108	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3109	if dofile:
3110	try:
3111	with gen.Capturing() as output:
3112	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3113	except:
3114	print errmsg
3115	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3116	for s1out in output: print s1out
3117	quit(-1)
3118
3119	if db:
3120	for s1out in output: print s1out
3121
3122	sout = sub.call('mv file_oper_alongdims_turb.nc '+fileon, shell=True)
3123
3124	# Keeping the operations
3125	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3126	"' -f " + prevfile + " -v " + CFvarnp
3127	otrackf.write("\n")
3128	otrackf.write("# " + varn + " " + Fopers + "\n")
3129	otrackf.write(pyins + "\n")
3130
3131	varkeep.append('timestats')
3132
3133	elif op == 'tvar':
3134	# temporal variance values (Taylor's turbulence)
3135	vals='time\|-1,time,var,' + ':'.join(varnCFs) + ':' + vdnz
3136	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
3137	if dofile:
3138	try:
3139	with gen.Capturing() as output:
3140	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3141	except:
3142	print errmsg
3143	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3144	for s1out in output: print s1out
3145	quit(-1)
3146
3147	if db:
3148	for s1out in output: print s1out
3149
3150	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
3151
3152	# Keeping the operations
3153	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3154	"' -f " + prevfile + " -v " + CFvarnp
3155	otrackf.write("\n")
3156	otrackf.write("# " + varn + " " + Fopers + "\n")
3157	otrackf.write(pyins + "\n")
3158
3159	# removing dimension variable-dimension 'time'
3160	varnCFs.remove('time')
3161
3162	varkeep.append('timestats')
3163
3164	elif op == 'xmean':
3165	# x-axis mean values
3166	vals='lon\|-1,lon,mean,' + ':'.join(varnCFs)
3167	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3168	if dofile:
3169	try:
3170	with gen.Capturing() as output:
3171	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3172	except:
3173	print errmsg
3174	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3175	for s1out in output: print s1out
3176	quit(-1)
3177
3178	if db:
3179	for s1out in output: print s1out
3180
3181	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
3182
3183	# Keeping the operations
3184	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3185	"' -f " + prevfile + " -v " + CFvarnp
3186	otrackf.write("\n")
3187	otrackf.write("# " + varn + " " + Fopers + "\n")
3188	otrackf.write(pyins + "\n")
3189
3190	# removing dimension variable-dimension 'lon'
3191	varnCFs.remove('lon')
3192
3193	varkeep.append('lonstats')
3194
3195	elif op == 'xvar':
3196	# x-axis variance values
3197	vals='lon\|-1,lon,var,' + ':'.join(varnCFs)
3198	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3199	if dofile:
3200	try:
3201	with gen.Capturing() as output:
3202	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3203	except:
3204	print errmsg
3205	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3206	for s1out in output: print s1out
3207	quit(-1)
3208
3209	if db:
3210	for s1out in output: print s1out
3211
3212	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
3213
3214	# Keeping the operations
3215	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3216	"' -f " + prevfile + " -v " + CFvarnp
3217	otrackf.write("\n")
3218	otrackf.write("# " + varn + " " + Fopers + "\n")
3219	otrackf.write(pyins + "\n")
3220
3221	# removing dimension variable-dimension 'lon'
3222	varnCFs.remove('lon')
3223
3224	varkeep.append('lonstats')
3225
3226	elif op == 'ymean':
3227	# y-axis mean values
3228	vals='lat\|-1,lat,mean,' + ':'.join(varnCFs)
3229	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3230	if dofile:
3231	try:
3232	with gen.Capturing() as output:
3233	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3234	except:
3235	print errmsg
3236	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3237	for s1out in output: print s1out
3238	quit(-1)
3239
3240	if db:
3241	for s1out in output: print s1out
3242
3243	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
3244
3245	# Keeping the operations
3246	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3247	"' -f " + prevfile + " -v " + CFvarnp
3248	otrackf.write("\n")
3249	otrackf.write("# " + varn + " " + Fopers + "\n")
3250	otrackf.write(pyins + "\n")
3251
3252	# removing dimension variable-dimension 'lat'
3253	varnCFs.remove('lat')
3254
3255	varkeep.append('latstats')
3256
3257	elif op == 'zsum':
3258	# vertical aggregated values
3259	print " " + fname + ": kind '" + op + "' not ready !!"
3260	wrongstats.append(varn + '_' + opers)
3261	break
3262	else:
3263	print errmsg
3264	print ' ' + fname + ": operation '" + op + "' not ready !!"
3265	quit(-1)
3266
3267	# End of kind of operation
3268
3269	# Variable name in file (vninF) changed due to operation
3270	# but only if previous operation does not the same 'statistic'
3271	chvn = gen.dictionary_key_list(Opsurs, op)
3272	if chvn is not None:
3273	oldvninF = vninF
3274	vninF = vninF + chvn
3275	CFvarnp = CFvarnp.replace(oldvninF,vninF)
3276
3277	if dofile:
3278	if len(varkeep) > 0:
3279	varkeepS = ',' + ','.join(varkeep)
3280	else:
3281	varkeepS = ''
3282
3283	# Adding such CF dimension variables which might not be in the
3284	# post-operation file
3285	try:
3286	with gen.Capturing() as output:
3287	varinfile = ncvar.ivars(fileon)
3288	except:
3289	print errmsg
3290	print 'ivars('+fileon+')'
3291	for s1out in output: print s1out
3292	quit(-1)
3293
3294	for CFvn in varnCFs:
3295	if not gen.searchInlist(varinfile, CFvn):
3296	if db:
3297	print ' ' + fname + ": recupering CF variable '" + CFvn + "'"
3298	try:
3299	with gen.Capturing() as output:
3300	ncvar.fvaradd(prevfile+','+CFvn, fileon)
3301	except:
3302	print errmsg
3303	print 'fvaradd(' + prevfile + ',' + CFvn +', ' + fileon + ')'
3304	for s1out in output: print s1out
3305	quit(-1)
3306
3307	if db:
3308	for s1out in output: print s1out
3309
3310	totalvarkeeps = CFvarnp + ',' + ','.join(varnCFs) + varkeepS
3311	try:
3312	with gen.Capturing() as output:
3313	oclean = ncvar.cleaning_varsfile(totalvarkeeps,fileon)
3314	except:
3315	print errmsg
3316	print 'cleaning_varsfile('+totalvarkeeps+','+fileon+')'
3317	for s1out in output: print s1out
3318	quit(-1)
3319
3320	if db:
3321	for s1out in output: print s1out
3322
3323	# End of operations
3324	otrackf.close()
3325
3326	return
3327
3328	def compute_var_diffs(config, diffallvar, odir, diffscr, debug):
3329	""" Function to compute variable differences
3330	config= experiment configuration
3331	diffallvar= dictonary with the differences to compute
3332	odir= output folder
3333	diffscr= wether it should be done from the scratch
3334	"""
3335	fname = 'compute_var_diffs'
3336
3337	# output folder
3338	ofold = config['ofold']
3339
3340	# Home of the python scripts
3341	pyH = config['pyHOME']
3342
3343	# opsur = operation surnames
3344	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
3345	Opsurs = {}
3346	for opk in opsur:
3347	opn = opk.split('_')[1]
3348	vals = opsur[opk]
3349	Opsurs[opn] = vals.split(':')
3350
3351	# Getting in working dir
3352	os.chdir(odir)
3353
3354	# CF dimensions
3355	CFdims = ['lon', 'lat', 'pres', 'time']
3356
3357	# Trakcing file
3358	trkobjf = open(odir + '/all_diffvar.inf', 'a')
3359
3360	for vnop in diffallvar.keys():
3361	vn = vnop.split('_')[0]
3362	op = vnop.split('_')[1]
3363	opSfinal = op.replace('+','_')
3364
3365	lop = op.split('+')
3366	# `p' header add related to 'pinterp'
3367	if op.find('pinterp') != -1:
3368	print warnmsg
3369	print ' ' + fname + ': there is the vertical interpolation as operation'
3370	print ' variable differences will be computed from that files where '
3371	print ' vertical interpolation was just done'
3372	ipop = lop.index('pinterp')
3373	opS = '_' + '_'.join(lop[0:ipop+1])
3374	doops = lop[ipop+1:]
3375	SgP = 'p'
3376	vninF = varnoper(vn, '+'.join(lop[0:ipop+1]), Opsurs)
3377	print ' operations to perform:', doops
3378	else:
3379	SgP = ''
3380	opS = ''
3381	doops = lop
3382	vninF = vn
3383
3384	Sdoops = '+'.join(doops)
3385
3386	# Variable name within the file
3387
3388	vals = diffallvar[vnop]
3389	mod1 = vals[0]
3390	mod2 = vals[1]
3391	exp1 = vals[2]
3392	exp2 = vals[3]
3393	headerf1 = vals[4]
3394	headerf2 = vals[5]
3395	modexpdiff = mod2 + '/' + exp2 + ' - ' + mod1 + '/' + exp1
3396	modexpdiffS = mod2 + '-' + exp2 + '_' + mod1 + '-' + exp1
3397
3398	ifile1 = ofold + '/' + mod1 + '/' + exp1 + '/' + vn + '_' + headerf1 + SgP + \
3399	opS + '.nc'
3400	ifile2 = ofold + '/' + mod2 + '/' + exp2 + '/' + vn + '_' + headerf2 + SgP + \
3401	opS + '.nc'
3402	difffilen = odir + '/' + vn + '_diffvar_' + modexpdiffS + opS + '.nc'
3403	difffinalfilen = odir + '/' + vn + '_diffvar_' + modexpdiffS + opS + '_' + \
3404	Sdoops + '.nc'
3405
3406	if diffscr:
3407	sout = sub.call('rm ' + difffilen + ' >& /dev/null', shell=True)
3408	sout = sub.call('rm ' + difffinalfilen + ' >& /dev/null', shell=True)
3409
3410	if not os.path.isfile(difffinalfilen):
3411	if debug:
3412	print " Computing variable difference of '" + vn + "' for: '" + \
3413	modexpdiff + "' and performing operation '" + Sdoops + "'"
3414	print " var in file: '" + vninF + "'"
3415
3416	dfiles = 'add\|' + ifile2 + '\|' + vninF + ',sub\|' + ifile1 + '\|' + vninF
3417	values = '\|'.join(CFdims) + '@' + dfiles
3418	pyins = 'python ' + pyH + "/nc_var.py -o compute_opersvarsfiles -S '" + \
3419	values + "' -v " + vninF
3420
3421	try:
3422	with gen.Capturing() as output:
3423	ncvar.compute_opersvarsfiles(values,vninF)
3424	except:
3425	print errmsg
3426	print 'ncvar.compute_opersvarsfiles(' + values + ',' + vninF + ')'
3427	for s1out in output: print s1out
3428	quit(-1)
3429
3430	sout = sub.call('mv opersvarsfiles_'+vninF+'.nc ' + difffilen, shell=True)
3431
3432	# keeping all differences
3433	trkobjf.write('\n')
3434	trkobjf.write("#" + vn + ' ' + op + '\n')
3435	trkobjf.write( pyins + '\n')
3436
3437	# Computing statisitcs
3438	compute_diff_statistics(config, difffilen, odir, vninF, doops, diffscr, \
3439	debug)
3440
3441	trkobjf.close()
3442
3443	return
3444
3445
3446	def draw_diff_plots(config, plots, odir, allvarcomp, kdiff, figscr, debug):
3447	""" Function to draw all plots
3448	config= Configuration of the experiment
3449	plots= dictionary with the plots
3450	odir= output experiment folder
3451	allvarcomp= dictionary with all the variables to compute and their information
3452	kdiff= kind of differences:
3453	'diffop': plots from operation differences
3454	'diffvar': plots from variable differences
3455	figscr= whether figures should be done from the scratch or not
3456
3457	* Plot as
3458	{[kplot]} = [varn1]\|[op1]#[varn2]\|[op2]#[...[varnN]\|[opN]], ...
3459	[kplot] ___
3460	diffmap2Dsfc: 2D map of surface differences values of 1 variable
3461	diffmap2Dz: 2D map of 3D differences values of 1 variable
3462	map2Dsfc: 2D map of surface values of 1 variable
3463	map3D: 2D map of 3D values of 1 variable
3464	shadconthovmsfc: Hovmoeller diagrams of 2 variable at the surface in shadow and the other in contourn
3465	shadcont2Dsfc: 2D map of shadow (1st variable) and countour (2nd variable) [stvar1]#[stvar2]
3466	shadcont2Dzsec: 2D map of vertical section of 2 variables one in shadow and the other in contourn
3467	[varn]
3468	variable
3469	[op]
3470	'+' separated list of operations
3471	in figures with more than 1 variable, use '#' to separate the [varn]\|[op]
3472	"""
3473	fname = 'draw_diff_plots'
3474
3475	os.chdir(odir)
3476
3477	# Dictionary with the operations with surnames for the operated variable
3478	opersurnames = {}
3479	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
3480	for opsr in opsur.keys():
3481	opn = opsr.split('_')[1]
3482	vls = opsur[opsr].split(':')
3483	opersurnames[opn] = vls
3484
3485	# time values
3486	# Units time for the plots
3487	rd = config['CFreftime']
3488	tunits = config['CFunitstime'] + '!since!' + rd[0:4] + '-' + rd[4:6] + '-' + \
3489	rd[6:8] + '!' + rd[8:10] + ':' + rd[10:12] + ':' + rd[12:14]
3490	# time ticks kind
3491	tkind = config['timekind']
3492	# time ticks format
3493	tfmt = config['timefmt']
3494	# time axis label
3495	tlab = config['timelabel']
3496	timevals = [tunits, tkind, tfmt, tlab]
3497
3498	if kdiff == 'diffop':
3499	specplotkeyn = 'specificdiffopplot'
3500	elif kdiff == 'diffvar':
3501	specplotkeyn = 'specificdiffvarplot'
3502	else:
3503	print errmsg
3504	print ' ' + fname + ": differences kind '" + kdiff + "' not ready !!"
3505	quit(-1)
3506
3507	# Dictionary of plot specificities
3508	plotspecifics = {}
3509	if config.has_key(specplotkeyn):
3510	# [minval]: minimum value
3511	# [maxval]: minimum value
3512	# [colorbar]: name of the colorbar (from matplotlib) to use
3513	# [cntformat]: format of the contour labels
3514	# [colorcnt]: color for the countor lines
3515	plotspecs = config[specplotkeyn].split(':')
3516	for pltspc in plotspecs:
3517	pltvls = pltspc.split('\|')
3518	vn = pltvls[0]
3519	op = pltvls[1]
3520	fn = pltvls[2]
3521	plotspecifics[fn + '_' + vn + '_' + op] = pltvls[3:]
3522	if debug:
3523	print 'Specific values for plots _______'
3524	gen.printing_dictionary(plotspecifics)
3525
3526	# Kind of figures
3527	kindfigure = config['kindfig']
3528
3529	# Graphical labels and configuration
3530	modgc, expgc = graphical_characteristics(config, debug)
3531
3532	# Map value
3533	mapvalue = config['mapval']
3534
3535	# pythone scripts HOME
3536	pyHOME = config['pyHOME']
3537
3538	# Title-text of operations
3539	opexplained = {}
3540	optits = config['titleoperations'].split(':')
3541	for optit in optits:
3542	opn = optit.split('\|')[0]
3543	opt = optit.split('\|')[1]
3544	opexplained[opn] = opt
3545	if debug:
3546	print 'Titles for operations _______'
3547	gen.printing_dictionary(opexplained)
3548
3549	for kplot in plots.keys():
3550	varsplt = plots[kplot]
3551	for varplt in varsplt:
3552	if debug:
3553	print " printing '" + kplot + "' var ':" + varplt + "'..."
3554	varops = varplt.split('#')
3555
3556	# CF variables in plot
3557	CFvarsplot = []
3558	# Files in plot
3559	filesplot = []
3560	# Variables in plot within the files
3561	varsplot = []
3562	# Dims in figure
3563	dimsplot = []
3564	# pictoric values in figure
3565	pictplot = []
3566	# Name of the figure
3567	figname = ''
3568	# Title of the figure
3569	titfigure = ''
3570
3571	ivp = 0
3572	for varop in varops:
3573	vn = varop.split('\|')[0]
3574	op = varop.split('\|')[1]
3575
3576	# CF variables in plot
3577	CFvarsplot.append(vn)
3578
3579	vnopS = vn + '_' + op
3580	if not allvarcomp.has_key(vnopS):
3581	print errmsg
3582	print ' ' + fname + ": no file for variable-operation '" + \
3583	vnopS + "' !!"
3584	vopvals = allvarcomp[vnopS]
3585	mod1 = vopvals[0]
3586	mod2 = vopvals[1]
3587	mod3 = vopvals[2]
3588	mod4 = vopvals[3]
3589	headf = vopvals[4]
3590
3591	modv = modgc[mod1]
3592	expv = expgc[mod1+'/'+exp1]
3593	modlab1 = modv.label
3594	explab1 = expv.label
3595	modv = modgc[mod2]
3596	expv = expgc[mod2+'/'+exp2]
3597	modlab2 = modv.label
3598	explab2 = expv.label
3599
3600	modexpdiff = explab2 + '-' + explab1
3601	modexpdiffS = mod2 + '-' + exp2 + '_' + mod1 + '-' + exp1
3602
3603	difffilen = odir + '/' + vn + '_' + kdiff + '_' + modexpdiffS + \
3604	'_' + op.replace('+','_') + '.nc'
3605
3606	filesplot.append(difffilen)
3607	# Do we have processed the given difference?
3608	if not os.path.isfile(difffilen):
3609	print warnmsg
3610	print " " + fname + ": there is no file for '" + kdiff + \
3611	"' difference '" + varop + "' skiping it !!"
3612	break
3613
3614	# Name of the variable inside the file
3615	vnsur = varnoper(vn, op, opersurnames)
3616	varsplot.append(vnsur)
3617
3618	# Dimensions in file
3619	try:
3620	with gen.Capturing() as output:
3621	dims = ncvar.idims(difffilen)
3622	except:
3623	print errmsg
3624	print 'ncvar.idims('+difffilen+')'
3625	for s1out in output: print s1out
3626	quit(-1)
3627
3628	dimsplot.append(dims)
3629
3630	# pictoric values for the figure
3631	Sfivaop = kplot + '_' + vn + '_' + op
3632	if plotspecifics.has_key(Sfivaop):
3633	pictvals = plotspecifics[Sfivaop]
3634	else:
3635	Vvals = gen.variables_values(vn)
3636	pictvals = [Vvals[2], Vvals[3], Vvals[6], '%g', 'fixsigc','black']
3637
3638	pictplot.append(pictvals)
3639
3640	# Header of the name of the figure
3641	if ivp == 0:
3642	figname = kplot + '_' + vn + '_' + kdiff + '_' + modexpdiffS + \
3643	'_' + op.replace('+','-')
3644	else:
3645	figname = figname + '-' + vn + '_' + op.replace('+','-')
3646
3647	# Title of the figure:
3648	if mod1 == mod2:
3649	modexptit = mod1 + '!' + exp2 + '-' + exp1
3650	else:
3651	modexptit = modexpdiff
3652
3653	ivp = ivp + 1
3654	# End of variable-operation
3655	figname = figname + '.' + kindfigure
3656
3657	# Title of figure
3658	titfigure = create_figure_title(modexptit, kdiff, varops, opexplained)
3659
3660	if len(titfigure) > 80:
3661	print warnmsg
3662	print ' ' + fname + ": figure title '" + titfigure.replace('!', ' ')+\
3663	"' larger than 80 characters (actually", len(titfigure), ') !!'
3664	print " simplifying it to '" + modexptit.replace('!',' ') + '!' + \
3665	kdiff + '!' + vnS + "'"
3666	titfigure = modexptit.replace(' ','!') + '!' + kdiff + '!' + vnS
3667
3668	draw_plot(kplot, CFvarsplot, filesplot, varsplot, dimsplot, pictplot, \
3669	figname, titfigure, kindfigure, mapvalue, timevals, expgc, odir, \
3670	pyHOME, figscr, debug)
3671
3672	# End of variables-operations
3673
3674	# End of kind of plots
3675
3676	return
3677
3678	def reproject_modexp(mod,exp,config,REPRJops,fscr,fadd,debug):
3679	""" Function to re-project and re-compute files from a given model/experiment
3680	mod= model
3681	exp= experiment
3682	config= configuration
3683	REPRJops= dictionary with the reprojection operators to use for reproject and for which variable
3684	# remapbil CDO: Bilinear interpolation
3685	# remapbic CDO: Bicubic interpolation
3686	# remapdis CDO: Distance-weighted average remapping
3687	# remapnn CDO: Nearest neighbor remapping
3688	# Operators only available if projections have the corners of the grid points.
3689	# remapcon CDO: First order conservative remapping
3690	# remapcon2 CDO: Second order conservative remapping
3691	# remaplaf CDO: Largest area fraction remapping
3692	####### ####### ####### or additionally ####### ####### #######
3693	# python remapping
3694	# dis python: Distance-weighted average remapping
3695	# pnn python: Nearest neighbor remapping
3696
3697	fscr= re-projected files to compute from the scratch
3698	fadd= re-projected files to be added
3699	"""
3700	fname = 'reproject_modexp'
3701
3702	odir = config['ofold'] + '/' + mod + '/' + exp
3703
3704	# NON re-projectable operations
3705	opNOreproj = config['NOreprojops'].split(':')
3706
3707	# Need to pass to analyze all the data?
3708	inffiles = ['reproj_varcompute.inf', 'all_reproj_computevars.inf']
3709	if fscr:
3710	for inff in inffiles:
3711	if debug: print " removing information file '" + inff + "' ..."
3712	ins = 'rm ' + odir + '/' + inff + ' >& /dev/null'
3713	sout = sub.call(ins, shell=True)
3714
3715	objf = open(odir+'/all_reproj_computevars.inf','w')
3716	objf.write("## Computation of reprojected variables \n")
3717	objf.close()
3718
3719	varcompf = odir + '/reproj_varcompute.inf'
3720	if fadd:
3721	sub.call('rm ' + varcompf +' >& /dev/null', shell=True)
3722
3723	# CF dims-vardims pairs:
3724	CFvardims = ['long@lon', 'lat@lat', 'pres@pres', 'time@time']
3725
3726	# Dictionary with the operations with surnames for the operated variable
3727	opersurnames = {}
3728	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
3729	for opsr in opsur.keys():
3730	opn = opsr.split('_')[1]
3731	vls = opsur[opsr].split(':')
3732	opersurnames[opn] = vls
3733
3734	if not os.path.isfile(varcompf):
3735	# Getting original computing information
3736	origvarcompf = odir + '/varcompute.inf'
3737	origfiles,origtestfiles,origallcompvar, Nvar = read_varcomp_file(origvarcompf)
3738
3739	Svarcompute = ''
3740	allcompvar = dict(origallcompvar)
3741	ivop = 0
3742	for vnop in origallcompvar.keys():
3743	vn = vnop.split('_')[0]
3744	op = vnop.split('_')[1]
3745	values = allcompvar[vnop]
3746	values[0] = 'reproj-' + values[0]
3747	allcompvar[vnop] = values
3748
3749	# Building strings from list values
3750	model = values[1]
3751	diag = values[2]
3752	if model is not None:
3753	if type(model) == type(list([1, 2])):
3754	Smodel = ':'.join(model)
3755	elif type(model) == type('A'):
3756	Smodel = model
3757	else:
3758	Smodel = 'None'
3759	if diag is not None:
3760	if type(diag) == type(list([1, 2])):
3761	Sdiag = ':'.join(diag)
3762	elif type(diag) == type('A'):
3763	Sdiag = diag
3764	else:
3765	Sdiag = 'None'
3766
3767	Svc = vn + '\|' + op + '\|' + values[0] + '\|' + Smodel + '\|' + Sdiag + \
3768	'\|' + values[3]
3769	if ivop == 0:
3770	Svarcompute = Svc
3771	else:
3772	Svarcompute = Svarcompute + ',' + Svc
3773	ivop = ivop + 1
3774
3775	origstrfiles = gen.dictKeysVals_stringList(origfiles)
3776	origstrtestfiles = gen.dictKeysVals_stringList(origtestfiles)
3777
3778	# Outwritting the varcompute to avoid next time (if it is not filescratch!)
3779	objf = open(odir + '/reproj_varcompute.inf', 'w')
3780	objf.write('files: ' + origstrfiles + '\n')
3781	objf.write('testfiles: ' + origstrtestfiles + '\n')
3782	objf.write('varcompute: ' + Svarcompute + '\n')
3783	objf.write('itotv: ' + str(ivop) + '\n')
3784	objf.close()
3785	else:
3786	print warnmsg
3787	print ' ' + main + ": getting variables to compute already from file !!"
3788	origfiles, origtestfiles, allcompvar, Nvar = read_varcomp_file(varcompf)
3789
3790	# End of avoiding to repeat all the experiment search
3791
3792	print " For experiment '" + exp + "' is required to re-project:", Nvar, \
3793	"variables"
3794
3795	if dbg:
3796	print 'Variables to re-project _______'
3797	gen.printing_dictionary(allcompvar)
3798
3799	# Re-projection variables
3800	print " Reprojecting variables ..."
3801	os.chdir(odir)
3802
3803	# Keeping track of reprojection
3804	trckobjf = open(odir + '/all_reproj_computevars.inf','a')
3805
3806	for vnop in allcompvar.keys():
3807	vn = vnop.split('_')[0]
3808	op = vnop.split('_')[1]
3809	values = allcompvar[vnop]
3810	VnOpS = varnoper(vn, op, opersurnames)
3811
3812	# Only are reprojected that files which do contain 'lon' and 'lat',
3813	# otherways, it is only re-projected the original file
3814	inNO, iopNO, iop = gen.list_coincidences(opNOreproj,op.split('+'))
3815
3816	reprjop = gen.dictionary_key_list(REPRJops,vn).split('_')[1]
3817	if reprjop is None:
3818	print errmsg
3819	print ' ' + fname + ": no reprojecting operator found for " + \
3820	" variable '" + vn + "' !!"
3821	print ' available values _______'
3822	gen.printing_dictionary(REPRJops)
3823	quit(-1)
3824
3825	# Re-projecting also the 'original' (at least with 'pinterp') file for varsdif
3826	if op.find('pinterp') != -1:
3827	ifileorig = vn +'_'+ values[0].replace('reproj-','') + 'p_pinterp.nc'
3828	ofileorig = vn +'_'+ values[0] + 'p_pinterp.nc'
3829	ofilen = vn +'_'+ values[0] + 'p_pinterp.nc'
3830	else:
3831	ifileorig = vn +'_'+ values[0].replace('reproj-','') + '.nc'
3832	ofileorig = vn +'_'+ values[0] + '.nc'
3833
3834	ifileop = vn+'_'+values[0].replace('reproj-','')+'_'+op.replace('+','_')+'.nc'
3835	ofileop = vn + '_' + values[0] + '_' + op.replace('+','_') + '.nc'
3836
3837	if fscr:
3838	sout = sub.call('rm ' + odir+'/'+ofileorig + ' >& /dev/null', shell=True)
3839	sout = sub.call('rm ' + odir+'/'+ofileop + ' >& /dev/null', shell=True)
3840
3841	# List of files to re-project (first without operation)
3842	ifiles = [odir+'/'+ifileorig, odir+'/'+ifileop]
3843	# List of output files after re-projection (first without operation)
3844	ofiles = [odir+'/'+ofileorig, odir+'/'+ofileop]
3845
3846	print fname + '; Lluis vn:', vn,'op:',op,'VnOpS:',VnOpS
3847	print fname + '; Lluis ifiles:', ifiles
3848	print fname + '; Lluis ofiles:', ofiles
3849
3850
3851	for iif in range(2):
3852	ifile = ifiles[iif]
3853	ofile = ofiles[iif]
3854
3855	# Only projecting original file-variable if there are not 'lon' and 'lat'
3856	doreproj = True
3857	if iif == 1 and len(inNO) > 0:
3858	doreproj = False
3859	print warnmsg
3860	print ' ' + fname + ": operation '" + op + "' can not be " + \
3861	"reprojected due to the lack of 'lon' or 'lat' skipping it !!"
3862	print " NOTE: difference can only be computed as:"
3863	print " - 1st: compute variable's differences between models"
3864	print " - 2nd: Perform the operations"
3865
3866	# Building up resources to compute re-projected statistics
3867	print ' ' + fname + ': computing from the original reprojected file'
3868	ModI = ModelInf(mod, mod, 'lon', 'lat', 'pres', 'time', 'depth', \
3869	'lon', 'lat', 'pres', 'time', 'depth', None)
3870	VarI = VariableInf(vn, values[0], values[2], values[3])
3871	usefs = {values[0]: ofileorig}
3872
3873	compute_statistics(ModI, config, config['ifold'], usefs, \
3874	config['ofold']+'/'+mod+'/'+exp,VarI,pinterp_var(op), op, fscr, dbg)
3875
3876	if not os.path.isfile(ofile) and doreproj:
3877	if debug and ofile == ofiles[0]:
3878	print " reprojecting '" + vn + "' '" + op + "' into '" + \
3879	ofile + "'"
3880
3881	# Reprojecting...
3882	if reprjop[0:6] == 'remap':
3883	ins = config['cdoHOME'] + '/cdo ' + reprjop + ',' + \
3884	config['RefProjfile'] + ' ' + ifile + ' ' + ofile
3885	try:
3886	with gen.Capturing() as output:
3887	sout = sub.call(ins, shell=True)
3888	except:
3889	print errmsg
3890	print ' ' + fname + ': $ ' + ins
3891	for s1out in output: print s1out
3892	print sout
3893	quit(-1)
3894	if sout != 0:
3895	print errmsg
3896	print ' ' + fname + ': $ ' + ins
3897	sub.call(ins, shell=True)
3898	quit(-1)
3899	else:
3900	# Name of the variable to reproject
3901	if iif == 0:
3902	reprjvn = vn
3903	else:
3904	reprjvn = VnOpS
3905
3906	values = 'lon,lat,' + config['RefProjfile'] + ',lon,lat,' + \
3907	reprjop + ',' + ':'.join(CFvardims)
3908	ins= 'ncvar.reproject('+values + ', ' + ifile + ', '+ reprjvn + ')'
3909	try:
3910	with gen.Capturing() as output:
3911	sout=ncvar.reproject(values, ifile, reprjvn)
3912	except:
3913	print errmsg
3914	print ' ' + fname + ': ' + ins
3915	for s1out in output: print s1out
3916	quit(-1)
3917
3918	sub.call('mv reproject.nc ' + ofile, shell=True)
3919
3920	if debug:
3921	for s1out in output: print s1out
3922	print sout
3923
3924	# Keeping track of re-projection
3925	trckobjf.write('\n')
3926	if iif == 0:
3927	trckobjf.write('# ' + vn + '\n')
3928	else:
3929	trckobjf.write('# ' + vn + ' ' + op + '\n')
3930
3931	trckobjf.write(ins + '\n')
3932
3933	if reprjop[0:6] == 'remap':
3934	# CFing the reprojected file
3935	try:
3936	with gen.Capturing() as output:
3937	ncvar.CDO_toCF(ofile)
3938	except:
3939	print errmsg
3940	print 'ncvar.CDO_toCF(' + ofile + ')'
3941	for s1out in output: print s1out
3942	quit(-1)
3943
3944	if debug:
3945	for s1out in output: print s1out
3946
3947	trckobjf.close()
3948
3949	return
3950
3951	def allmodexps_listconstruct(config, modexps, odir, debug):
3952	""" Function to create the list of all model-experiments to draw
3953	config= Configuration of the experiment
3954	modexps= list with the models-experiments pairs to use
3955	odir= output differences folder
3956	"""
3957	fname='allmodexps_listconstruct'
3958
3959	allplts = gen.get_specdictionary_HMT(config, H='PLOTALLMODEXP_')
3960
3961	if debug:
3962	if allplts is not None:
3963	print " 'all mod-exps' plots ________"
3964	gen.printing_dictionary(allplts)
3965
3966	# Getting Model-Reference Projection
3967	ModComProj = cnf['RefProj'].split(':')
3968
3969	allmodexp = {}
3970	Nallmodexp = 0
3971	for modexp in modexps:
3972	if debug:
3973	print " " + fname + ": '" + modexp + "' ..."
3974	mod = modexp.split('/')[0]
3975	exp = modexp.split('/')[1]
3976
3977	# Getting variable characteristics from each model/experiment
3978	idir = config['ofold'] + '/' + modexp
3979
3980	reproj = not gen.searchInlist(ModComProj,mod)
3981
3982	fvarcomp = 'varcompute.inf'
3983	if reproj: fvc = 'reproj_' + fvarcomp
3984	else: fvc = fvarcomp
3985
3986	files, testfiles, allcompvar, Nvar= read_varcomp_file(idir + '/' + fvc)
3987
3988	if allplts is not None:
3989	# Getting all model-experiments plots characteristics
3990
3991	for allplt in allplts.keys():
3992	pltn = allplt.split('_')[1]
3993	if debug and modexp == modexps[0]:
3994	print ' ' + fname + ": plot '" + pltn + "' "
3995	varops = allplts[allplt].split(':')
3996	for varop in varops:
3997	vn = varop.split('\|')[0]
3998	op = varop.split('\|')[1]
3999	vnop = vn + '_' + op
4000	if debug and modexp == modexps[0]:
4001	print ' ' + fname + ": varop '" + vnop + "' "
4002	if not allcompvar.has_key(vnop):
4003	print warnmsg
4004	print ' ' + fname + ": variable+operation '" + vnop + \
4005	"' in '" + modexp + "' was not computed !!"
4006	print ' skipping it!'
4007	print ' computed values:', allcompvar.keys()
4008	break
4009
4010	vals = allcompvar[vnop]
4011
4012	headerf = vals[0]
4013	if modexp == modexps[0]:
4014	allmodexp[vnop] = [modexp,headerf]
4015	Nallmodexp = Nallmodexp + 1
4016	else:
4017	prevals = allmodexp[vnop]
4018	prevals = prevals + [modexp,headerf]
4019	allmodexp[vnop] = prevals
4020
4021	else:
4022	Nallmodexp = 0
4023
4024	if allplts is not None:
4025	Sallmodexps = gen.dictKeysVals_stringList(allmodexp)
4026	else:
4027	Sallmodexps = 'none'
4028
4029	# Outwritting the all model-experiment plots file to avoid next time
4030	objf = open(odir + '/allmodexp.inf', 'w')
4031	objf.write('allmodexps: ' + Sallmodexps + '\n')
4032	objf.write('Nallmodexp: ' + str(Nallmodexp) + '\n')
4033	objf.close()
4034
4035	return allmodexp, Nallmodexp
4036
4037	def allmodexps_read(filen):
4038	""" Function to read the file with the information about the all model-experiment plots
4039	filen= file with the information
4040	"""
4041	fname = 'allmodexps_read'
4042
4043	if not os.path.isfile(filen):
4044	print errormsg
4045	print ' ' + fname + ": all model-experiment file '" + filen + \
4046	"' does not exist !!"
4047	quit(-1)
4048
4049	objf = open(filen, 'r')
4050	for line in objf:
4051	if line[0:1] != '#' and len(line) > 1:
4052	values = line.replace('\n','').split(' ')
4053	if values[0] == 'allmodexps:':
4054	allplts = gen.stringList_dictKeysVals(values[1],cV=':')
4055	elif values[0] == 'Nallmodexp:':
4056	Nallmodexp = int(values[1])
4057
4058	objf.close()
4059
4060	return allplts, Nallmodexp
4061
4062	def draw_allmodexp_plots(config, allvarcomp, plots, modexp, odir, figscr, debug):
4063	""" Function to draw all model-experiment plots
4064	config= Configuration of the experiment
4065	allvarcomp = dictionary with the file headers for each variable
4066	plots= dictionary with the plots
4067	modexps= list with the names of the model-experiment pairs
4068	odir= output experiment folder
4069	figscr= whether figures should be done from the scratch or not
4070
4071	* Plot as
4072	{[kplot]} = [varn1]\|[op1]#[varn2]\|[op2]#[...[varnN]\|[opN]], ...
4073	[kplot] ___
4074	Nlines: Multiple lines with one line by model-experiment pairs
4075	[varn]
4076	variable
4077	[op]
4078	'+' separated list of operations
4079	in figures with more than 1 variable, use '#' to separate the [varn]\|[op]
4080	"""
4081	fname = 'draw_allmodexp_plots'
4082
4083	os.chdir(odir)
4084
4085	# Folder with the processed files
4086	infold = config['ofold']
4087
4088	# Dictionary with the operations with surnames for the operated variable
4089	opersurnames = {}
4090	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
4091	for opsr in opsur.keys():
4092	opn = opsr.split('_')[1]
4093	vls = opsur[opsr].split(':')
4094	opersurnames[opn] = vls
4095
4096	# time values
4097	# Units time for the plots
4098	rd = config['CFreftime']
4099	tunits = config['CFunitstime'] + '!since!' + rd[0:4] + '-' + rd[4:6] + '-' + \
4100	rd[6:8] + '!' + rd[8:10] + ':' + rd[10:12] + ':' + rd[12:14]
4101	# time ticks kind
4102	tkind = config['timekind']
4103	# time ticks format
4104	tfmt = config['timefmt']
4105	# time axis label
4106	tlab = config['timelabel']
4107	timevals = [tunits, tkind, tfmt, tlab]
4108
4109	# Dictionary of plot specificities
4110	specplotkeyn = 'specificvarplot'
4111	plotspecifics = {}
4112	if config.has_key(specplotkeyn):
4113	# [minval]: minimum value
4114	# [maxval]: minimum value
4115	# [colorbar]: name of the colorbar (from matplotlib) to use
4116	# [cntformat]: format of the contour labels
4117	# [colorcnt]: color for the countor lines
4118	plotspecs = config[specplotkeyn].split(':')
4119	for pltspc in plotspecs:
4120	pltvls = pltspc.split('\|')
4121	vn = pltvls[0]
4122	op = pltvls[1]
4123	fn = pltvls[2]
4124	plotspecifics[fn + '_' + vn + '_' + op] = pltvls[3:]
4125	if debug:
4126	print 'Specific values for plots _______'
4127	gen.printing_dictionary(plotspecifics)
4128
4129	# Graphical labels and configuration
4130	modgc, expgc = graphical_characteristics(config, debug)
4131
4132	# Kind of figures
4133	kindfigure = config['kindfig']
4134
4135	# Map value
4136	mapvalue = config['mapval']
4137
4138	# pythone scripts HOME
4139	pyHOME = config['pyHOME']
4140
4141	# Title-text of operations
4142	opexplained = {}
4143	optits = config['titleoperations'].split(':')
4144	for optit in optits:
4145	opn = optit.split('\|')[0]
4146	opt = optit.split('\|')[1]
4147	opexplained[opn] = opt
4148	if debug:
4149	print 'Titles for operations _______'
4150	gen.printing_dictionary(opexplained)
4151
4152	for kplot in plots.keys():
4153	varsplt = plots[kplot]
4154	for varplt in varsplt:
4155	if debug:
4156	print " printing '" + kplot + "' var ':" + varplt + "'..."
4157	varops = varplt.split('#')
4158
4159	# CF variables in plot
4160	CFvarsplot = []
4161	# Files in plot
4162	filesplot = []
4163	# Variables in plot within the files
4164	varsplot = []
4165	# Dims in figure
4166	dimsplot = []
4167	# pictoric values in figure
4168	pictplot = []
4169	# Name of the figure
4170	figname = ''
4171	# Title of the figure
4172	titfigure = ''
4173
4174	ivp = 0
4175	for varop in varops:
4176	vn = varop.split('\|')[0]
4177	op = varop.split('\|')[1]
4178
4179	# CF variables in plot
4180	CFvarsplot.append(vn)
4181
4182	if op.find('pinterp') != -1: gP = 'p'
4183	else: gP = ''
4184
4185	vnopS = vn + '_' + op
4186	if not allvarcomp.has_key(vnopS):
4187	print errmsg
4188	print ' ' + fname + ": no file for variable-operation '" + \
4189	vnopS + "' !!"
4190	modexpvals = allvarcomp[vnopS]
4191	# modexpvalsS = allvarcomp[vnopS]
4192	# modexpvals = modexpvalsS[0].split('\|')
4193	Nmodexps = len(modexpvals)
4194
4195	for imodexp in range(0,Nmodexps,2):
4196	modexp = modexpvals[imodexp]
4197	headf = modexpvals[imodexp+1]
4198	if debug:
4199	print ' ' + fname + " model/experiment: '" + modexp + "'"
4200
4201	filen = infold + '/' + modexp + '/' + vn +'_'+ headf+gP + '_' + \
4202	op.replace('+','_') + '.nc'
4203
4204	filesplot.append(filen)
4205	# Do we have processed the given variable?
4206	if not os.path.isfile(filen):
4207	print warnmsg
4208	print " " + fname + ": there is no file for variable '" + \
4209	varop + "' skiping it !!"
4210	break
4211
4212	if imodexp == 0:
4213	# Name of the variable inside the file
4214	vnsur = varnoper(vn, op, opersurnames)
4215	varsplot.append(vnsur)
4216
4217	# Dimensions in file
4218	try:
4219	with gen.Capturing() as output:
4220	dims = ncvar.idims(filen)
4221	except:
4222	print errmsg
4223	print 'ncvar.idims('+filen+')'
4224	for s1out in output: print s1out
4225	quit(-1)
4226
4227	if imodexp == 0:
4228	# Dimensions in plot
4229	dimsplot.append(dims)
4230
4231	# Header of the name of the figure
4232	if ivp == 0:
4233	figname = kplot+'_'+vn+'_'+headf+'_'+ op.replace('+','-')
4234	else:
4235	figname = figname+'-'+vn+'_'+headf+'_'+op.replace('+','-')
4236
4237	# pictoric values for the figure
4238	Sfivaop = kplot + '_' + vn + '_' + op
4239	if plotspecifics.has_key(Sfivaop):
4240	pictvals = plotspecifics[Sfivaop]
4241	else:
4242	Vvals = gen.variables_values(vn)
4243	pictvals = [Vvals[2],Vvals[3],Vvals[6], '%g', 'fixc', 'black']
4244
4245	pictplot.append(pictvals)
4246
4247	ivp = ivp + 1
4248	# End of variable-operation
4249	figname = figname + '.' + kindfigure
4250
4251	# Title of figure
4252	titfigure = 'all!model-experiments!' + vn + optitle(op,opexplained)
4253
4254	draw_plot(kplot, CFvarsplot, filesplot, varsplot, dimsplot, pictplot, \
4255	figname, titfigure, kindfigure, mapvalue, timevals, expgc, odir, \
4256	pyHOME, figscr, debug)
4257
4258	# End of variables-operations
4259
4260	# End of kind of plots
4261
4262	return
4263
4264	# Files with information about the configuration of the script
4265	inffiles = ['varcompute.inf', 'all_computevars.inf', 'all_statsvars.inf']
4266
4267	####### #######
4268	## MAIN
4269	#######
4270	if not os.path.isfile('model_graphics.dat'):
4271	print errmsg
4272	print main + ": configuration file 'model_graphics.dat' does not exist!!"
4273	quit(-1)
4274
4275	# Getting configuration from external ASCII file 'model_graphics.dat'
4276	cnf = gen.get_configuration('model_graphics.dat', False)
4277
4278	verify_configuration(cnf, gen.Str_Bool(cnf['debug']))
4279
4280	# scratches
4281	scratch, filescratch, figscratch, diffscratch, figdiffscratch, moddiffscratch, \
4282	figmoddiffscratch, figallmodexpscratch, addfiles, addfigures, adddiffs, \
4283	adddifffigures, addmoddiffs, addmoddifffigures, addallmodexpfigures, dbg = \
4284	scratches(cnf)
4285
4286	if dbg:
4287	print 'Experiment configuration scratches _______'
4288	print ' scratch:', scratch, 'filescratch:', filescratch,'figscratch:', figscratch
4289	print ' diffscratch:', diffscratch, 'figdiffscratch:', figdiffscratch
4290	print ' moddiffscratch:', moddiffscratch, 'figmoddiffscratch:', figmoddiffscratch
4291	print 'Experiment configuration adding _______'
4292	print ' addfiles:', addfiles, 'addfigures:', addfigures
4293	print ' adddiffs:', adddiffs, 'adddifffigures:', adddifffigures
4294	print ' addmoddiffs:', addmoddiffs, 'addmoddifffigures:', addmoddifffigures
4295
4296	# Getting models
4297	mods = cnf['models'].split(':')
4298
4299	# Getting graphical characeristics
4300	modGC, expGC = graphical_characteristics(cnf, dbg)
4301
4302	# Models loop
4303	##
4304
4305	# dictionary with the experiments of each model
4306	modexps = {}
4307
4308	for mod in mods:
4309	print mod
4310	if scratch:
4311	if cnf['ofold'] != cnf['ifold']:
4312	sout = sub.call('rm -rf '+cnf['ofold']+'/'+mod+' > /dev/null', shell=True)
4313	else:
4314	print warnmsg
4315	print ' ' + main + ": input '" + cnf['ifold'] + "' and output '" + \
4316	cnf['ofold'] + "' are the same folder !!"
4317	print " Keeping output folder although it has 'scratch' start"
4318
4319	# Get experiments and headers of model
4320	exps, fheaders = exp_headers(mod,cnf)
4321
4322	modexps[mod] = exps
4323	# Characteristics of the model
4324	Modinf = ncvar.model_characteristics(mod,'None','False')
4325	dnx = Modinf.dimxn
4326	dny = Modinf.dimyn
4327	dnz = Modinf.dimzn
4328	dnt = Modinf.dimtn
4329	dns = Modinf.dimsn
4330	vdnx = Modinf.vardxn
4331	vdny = Modinf.vardyn
4332	vdnz = Modinf.vardzn
4333	vdnt = Modinf.vardtn
4334	vdns = Modinf.vardsn
4335
4336	modgraphv = modGC[mod]
4337	Modinf = ModelInf(mod, mod, dnx, dny, dnz, dnt, dns, vdnx, vdny, vdnz, vdnt, \
4338	vdns, modgraphv.tmod)
4339
4340	if dbg:
4341	print ' model characteristics _______'
4342	gen.printing_class(Modinf)
4343
4344	moddims = dnx + ',' + dny + ',' + dnz + ',' + dnt
4345	modvdims = vdnx + ',' + vdny + ',' + vdnz + ',' + vdnt
4346
4347	# Experiments loop
4348	##
4349	for exp in exps:
4350	print ' ' + exp + '...'
4351
4352	# input folder
4353	iwdir = cnf['ifold'] + '/' + mod + '/' + exp
4354
4355	# Does input folder exist?
4356	if not os.path.isdir(iwdir):
4357	if filescratch*addfiles:
4358	print errmsg
4359	print " " + main + ": folder '" + iwdir + "' does not exist !!"
4360	quit(-1)
4361	else:
4362	print warnmsg
4363	print " " + main + ": folder '" + iwdir + "' does not exist !!"
4364	print " but start of files from scratch 'filescratch':", \
4365	filescratch, "and there is not the add of any file 'addfiles':", \
4366	addfiles
4367	print ' so, keep moving'
4368
4369	owdir = cnf['ofold'] + '/' + mod + '/' + exp
4370	sout = sub.call('mkdir -p ' + owdir, shell=True)
4371
4372	# Need to pass to analyze all the data?
4373	if filescratch:
4374	for inff in inffiles:
4375	if dbg: print " removing information file '" + inff + "' ..."
4376	ins = 'rm ' + owdir + '/' + inff + ' >& /dev/null'
4377	sout = sub.call(ins, shell=True)
4378
4379	objf = open(owdir+'/all_computevars.inf','w')
4380	objf.write("## Computation of variables \n")
4381	objf.close()
4382	objf = open(owdir+'/all_statsvars.inf','w')
4383	objf.write("## Computation of statistics \n")
4384	objf.close()
4385
4386	varcompf = owdir + '/varcompute.inf'
4387	if addfiles:
4388	if dbg:
4389	print ' ' + main + ": adding variables to copmute removing " + \
4390	"file '" + varcompf + "' ..."
4391	sub.call('rm ' + varcompf +' >& /dev/null', shell=True)
4392
4393	if not os.path.isfile(varcompf):
4394	# Does input folder has header files?
4395	ih=1
4396	# Dictionary with the list of files for each headers
4397	files = {}
4398	# Dictionary with a file fromor each headers
4399	testfiles = {}
4400
4401	for fh in fheaders:
4402	if filescratch:
4403	ins = 'rm '+ owdir+'/_' + fh + '.nc >& /dev/null'
4404	sout = sub.call(ins, shell=True)
4405	files1h = gen.files_folder(iwdir,fh)
4406	if len(files1h) < 1:
4407	print errmsg
4408	print ' ' + main + ": folder '" + iwdir + "' does not " + \
4409	"contain files '" + fh + "*' !!"
4410	quit(-1)
4411	files[fh] = files1h
4412	testfiles[fh] = files1h[0]
4413
4414	if dbg:
4415	print ' Dictionary of files _______'
4416	gen.printing_dictionary(files)
4417
4418	print ' Dictionary of test-files _______'
4419	gen.printing_dictionary(testfiles)
4420
4421	allcompvar, Nvar = compvars_listconstruct(cnf, Modinf, files, testfiles, \
4422	iwdir, owdir, dbg)
4423	else:
4424	print warnmsg
4425	print ' ' + main + ": getting variables to compute already from file !!"
4426	files, testfiles, allcompvar, Nvar = read_varcomp_file(varcompf)
4427
4428	# End of avoiding to repeat all the experiment search
4429
4430	print " For experiment '" + exp + "' is required to compute:", Nvar, \
4431	"variables"
4432
4433	if dbg:
4434	print 'Variables to compute _______'
4435	gen.printing_dictionary(allcompvar)
4436
4437	# Computing variables
4438	print " Computing variables ..."
4439	compute_vars(cnf, Modinf, iwdir, owdir, files, allcompvar, filescratch, dbg)
4440
4441	# Figures of variables direct from files
4442	print " Plotting direct figures ..."
4443	dirfigf = owdir + '/directplotsdraw.inf'
4444	if figscratch:
4445	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4446
4447	objf = open(owdir+'/all_figures.inf','w')
4448	objf.write("## Drawing of all figures \n")
4449	objf.close()
4450
4451	if addfigures:
4452	if dbg:
4453	print ' ' + main + ": adding direct figures removing file '" + \
4454	dirfigf + "' ..."
4455	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4456
4457	if not os.path.isfile(dirfigf):
4458	listplots, Nplt = plots_listconstruct(cnf, 'DIRPLT', dirfigf, owdir, dbg)
4459	else:
4460	print warnmsg
4461	print ' ' + main + ": getting plots to draw already from file !!"
4462	listplots, Nplt = read_plot_file(dirfigf)
4463
4464	# End of avoiding to repeat all the plots search
4465
4466	print " For experiment '" + exp + "' is required to plot:", Nplt, "plots"
4467
4468	if dbg:
4469	print 'Plots to draw _______'
4470	gen.printing_dictionary(listplots)
4471
4472	draw_plots(cnf, listplots, mod, exp, owdir, allcompvar, figscratch, dbg)
4473
4474	# end of experiments loop
4475
4476	### ## #
4477	# Experiment differences
4478	## # ## #
4479
4480	# Experiments loop
4481	##
4482	print ' ***** ** * * ** * * ** * ** *****'
4483	print " ** '" + mod + "': Inter experiments differences "
4484	print ' ***** ** * * ** * * ** * ** *****'
4485	print ' experiments:', exps
4486	# There are two kind of differences:
4487	# DIFFop: differences between operations of each given variable.
4488	# Differences are computed directly from the last stage of the operation
4489	# DIFFvar: operations of differences between variables
4490	# First are computed the differences from the initial variable file
4491	# and then operations are made
4492	# NOTE: remember that: meanvar2 - meanvar1 = mean(var2 - var1)
4493	difffiles = ['diffop.inf', 'diffvar.inf']
4494
4495	Nexps = len(exps)
4496	for iexp1 in range(0,Nexps-1):
4497	exp1 = exps[iexp1]
4498	for iexp2 in range(iexp1+1,Nexps):
4499	exp2 = exps[iexp2]
4500	Sexps = exp2 + '-' + exp1
4501	print ' ' + Sexps + '...'
4502	owdir = cnf['ofold'] + '/' + mod + '/' + exp2 + '-' + exp1
4503	sout = sub.call('mkdir -p ' + owdir, shell=True)
4504
4505	# Removing files with list of differences if should be started from scratch or add differences
4506	for fdiff in difffiles:
4507	difff = owdir + '/' + fdiff
4508	if diffscratch:
4509	sub.call('rm -rf' + owdir +' >& /dev/null', shell=True)
4510	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4511	objf = open(owdir+'/all_' + fdiff,'w')
4512	objf.write("## Computation and drawing of differences " + \
4513	"between '" + exp2 + "'-'" + exp1 + "'\n")
4514	objf.close()
4515	difff = owdir + '/all_vardiffstatistics.inf'
4516	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4517	objf = open(owdir+'/all_' + fdiff,'w')
4518	objf.write("## Computation of all differences statistics " + \
4519	"between '" + exp2 + "'-'" + exp1 + "'\n")
4520	objf.close()
4521
4522	if adddiffs:
4523	if dbg:
4524	print ' ' + main + ": adding differences removing file '" + \
4525	difff + "' ..."
4526	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4527
4528	# op and var differences
4529	diffvarcompf = owdir + '/' + difffiles[0]
4530	if not os.path.isfile(diffvarcompf):
4531	alldiffop, alldiffvar,Nopdiffs,Nvardiffs=diffvarop_listconstruct(cnf,\
4532	[mod, mod], [exp1, exp2], [False, False], owdir, dbg)
4533	else:
4534	print warnmsg
4535	print ' ' + main + ": getting 'op' and 'var'differences to " + \
4536	"compute already from file !!"
4537	alldiffop, Nopdiffops = read_diff_file(diffvarcompf)
4538	alldiffvar, Nopdiffvars = read_diff_file(owdir+'/'+difffiles[1])
4539
4540	# End of avoiding to repeat all the experiment search
4541
4542	print " For experiments '"+exp2+"'-'"+exp1+"' is required to compute:", \
4543	Nvar, "differences"
4544
4545	if dbg:
4546	print 'Differences to compute _______'
4547	gen.printing_dictionary(alldiffop)
4548	gen.printing_dictionary(alldiffvar)
4549
4550	# Computing differences
4551	##
4552	print " Computing operation differences ..."
4553	compute_op_diffs(cnf, alldiffop, owdir, diffscratch, dbg)
4554	print " Computing variable differences ..."
4555	compute_var_diffs(cnf, alldiffvar, owdir, diffscratch, dbg)
4556
4557	# Plotting operation differences
4558	##
4559	print " " + main + ": Plotting operation differences' figures ..."
4560	dirfigf = owdir + '/diffopplotsdraw.inf'
4561	if figscratch:
4562	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4563
4564	objf = open(owdir+'/all_diffopfigures.inf','w')
4565	objf.write("## Drawing of all operation difference figures \n")
4566	objf.close()
4567
4568	if adddifffigures:
4569	if dbg:
4570	print ' ' + main + ": adding differences' figures removing file '" +\
4571	dirfigf + "' ..."
4572
4573	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4574
4575	if not os.path.isfile(dirfigf):
4576	listplots, Nplt = plots_listconstruct(cnf, 'PLOTDIFFOP', dirfigf, owdir, \
4577	dbg)
4578	else:
4579	print warnmsg
4580	print ' ' + main + ": getting plots to draw already from file !!"
4581	listplots, Nplt = read_plot_file(dirfigf)
4582
4583	# End of avoiding to repeat all the plots search
4584
4585	print " For experiment 'operation' differences '" + Sexps + "' is " + \
4586	"required to plot:" , Nplt, "plots"
4587
4588	if dbg:
4589	print ' Plots to draw _______'
4590	gen.printing_dictionary(listplots)
4591
4592	draw_diff_plots(cnf, listplots, owdir, alldiffop, 'diffop', figdiffscratch, \
4593	dbg)
4594
4595	# Plotting variable differences
4596	##
4597	print " " + main + ": Plotting variable differences' figures ..."
4598	dirfigf = owdir + '/diffvarplotsdraw.inf'
4599	if figscratch:
4600	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4601
4602	objf = open(owdir+'/all_diffvarfigures.inf','w')
4603	objf.write("## Drawing of all variables difference figures \n")
4604	objf.close()
4605
4606	if adddifffigures:
4607	if dbg:
4608	print ' ' + main + ": adding differences' figures removing file '" +\
4609	dirfigf + "' ..."
4610	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4611
4612	if not os.path.isfile(dirfigf):
4613	listplots, Nplt = plots_listconstruct(cnf, 'PLOTDIFFVAR', dirfigf, owdir,\
4614	dbg)
4615	else:
4616	print warnmsg
4617	print ' ' + main + ": getting plots to draw already from file !!"
4618	listplots, Nplt = read_plot_file(dirfigf)
4619
4620	# End of avoiding to repeat all the plots search
4621
4622	print " For experiment 'variables' differences '" + Sexps + "' is " + \
4623	"required to plot:" , Nplt, "plots"
4624
4625	if dbg:
4626	print ' Plots to draw _______'
4627	gen.printing_dictionary(listplots)
4628
4629	draw_diff_plots(cnf, listplots, owdir, alldiffvar, 'diffvar', figdiffscratch,\
4630	dbg)
4631
4632	# end of mods loop
4633
4634	### ## #
4635	# Model differences
4636	### ## #
4637
4638	# Models already at the reference projection
4639	ModComProj = cnf['RefProj'].split(':')
4640	if dbg:
4641	print 'Models to use as reference for the projection:', ModComProj
4642
4643	# Operators to use for the reprojection of each variable
4644	REPRJvar0 = gen.get_specdictionary_HMT(cnf, H='reprojectvar_')
4645	REPRJvar = {}
4646	for Skey in REPRJvar0.keys():
4647	REPRJvar[Skey] = gen.str_list(REPRJvar0[Skey],':')
4648
4649	if dbg:
4650	print 'Remaping operators to use _______'
4651	gen.printing_dictionary(REPRJvar)
4652
4653	# Getting common base weights for grid point remapping with CDO
4654	frefgridn = 'RefProj.nc'
4655	for mod in mods:
4656	frefgriddes = cnf['ofold'] + '/' + mod + '/' + frefgridn
4657	if moddiffscratch:
4658	sout = sub.call('rm ' + frefgriddes + ' >& /dev/null', shell=True)
4659	if gen.searchInlist(ModComProj,mod) and not os.path.isfile(frefgriddes):
4660	# Looking for a file to use (all files are CF compilant, but we need to
4661	# be sure that at least they have 'lon', 'lat' variables)
4662	exps = modexps[mod]
4663	exp = exps[0]
4664	reprjfdir = cnf['ofold'] + '/' + mod + '/' + exp
4665	reprjfile = None
4666	for reprjop in REPRJvar.keys():
4667	reprjvars = REPRJvar[cdoop]
4668	for reprjvar in reprjvars:
4669	reprjfs= gen.files_folder_HMT(folder=reprjfdir,head=reprjvar+'_',tail='.nc')
4670	for reprjf in reprjfs:
4671	# Getting direct files as [var]_[fhead].nc
4672	if len(reprjf.split('_')) == 2:
4673	reprjfile = reprjfdir + '/' + reprjf
4674	reprjvn = reprjvar
4675	break
4676	if reprjfile is None:
4677	print errmsg
4678	print ' ' + main + ": no proper file to get projection information " + \
4679	"from '" + reprjfdir + "' has been found !!"
4680	quit(-1)
4681
4682	print " Creation of reference projection information file from '" + mod+ "'"
4683	if dbg:
4684	print " using file: '" + reprjfile
4685
4686	# Getting only 1 time-step for disk space issues
4687	try:
4688	with gen.Capturing() as output:
4689	ncvar.DataSetSection('time,0,1,1', reprjfile)
4690	except:
4691	print errmsg
4692	print 'ncvar.DataSetSection(time,0,1,1, ' + reprjfile + ')'
4693	for s1out in output: print s1out
4694	quit(-1)
4695
4696	of = reprjfile.split('.')[0] + '_time_B0-E1-I1.nc'
4697
4698	# Cleaning variables
4699	try:
4700	with gen.Capturing() as output:
4701	ncvar.selvar('lon@lon,lat@lat,time@time', of, reprjvn)
4702	except:
4703	print errmsg
4704	print 'ncvar.selvar(lon@lon,lat@lat, ' + of + ', ' + reprjvn + ')'
4705	for s1out in output: print s1out
4706	quit(-1)
4707	sout = sub.call('mv selvar_new.nc ' + frefgriddes, shell=True)
4708	sout = sub.call('rm ' + of + ' >& /dev/null', shell=True)
4709
4710	cnf['RefProjfile'] = frefgriddes
4711
4712	if dbg:
4713	for s1out in output: print s1out
4714	break
4715
4716	elif gen.searchInlist(ModComProj,mod) and os.path.isfile(frefgriddes):
4717	# Including reprojecting reference file in configuration dictionary
4718	cnf['RefProjfile'] = frefgriddes
4719
4720	Nmods = len(mods)
4721	for imod1 in range(0,Nmods-1):
4722	mod1 = mods[imod1]
4723	exps1 = modexps[mod1]
4724
4725	for imod2 in range(imod1+1,Nmods):
4726	mod2 = mods[imod2]
4727	exps2 = modexps[mod2]
4728
4729	Smods = mod2 + '-' + mod1
4730	print ' ***** ** * * ** * * ** * ** *****'
4731	print " ** '" + Smods + "': Inter models differences "
4732	print ' ***** ** * * ** * * ** * ** *****'
4733	print ' experiments mod1:', exps1
4734	print ' experiments mod2:', exps2
4735
4736	# Experiments loop
4737	##
4738	difffiles = ['diffop.inf', 'diffvar.inf']
4739
4740	Nexps1 = len(exps1)
4741	Nexps2 = len(exps2)
4742	for iexp1 in range(0,Nexps1):
4743	exp1 = exps1[iexp1]
4744	for iexp2 in range(0,Nexps2):
4745	exp2 = exps2[iexp2]
4746	Sexps = exp2 + '-' + exp1
4747	print ' ' + Sexps + '...'
4748	owdir = cnf['ofold'] + '/' + Smods + '/' + Sexps
4749	sout = sub.call('mkdir -p ' + owdir, shell=True)
4750
4751	# Getting the right projection in order to perform differences
4752	difmods = [mod1, mod2]
4753	difexps = [exp1, exp2]
4754	difreproj = [False, False]
4755
4756	for imod in range(2):
4757	mod = difmods[imod]
4758	exp = difexps[imod]
4759	if not gen.searchInlist(ModComProj,mod):
4760	print " Projecting files of model '" + mod + "' exp: '" + \
4761	exp + "' with file '" + cnf['RefProjfile'] + "'"
4762	reproject_modexp(mod, exp, cnf, REPRJvar, moddiffscratch, \
4763	addmoddiffs, dbg)
4764	difreproj[imod] = True
4765
4766	# Removing files with list of differences if should be started from
4767	# scratch or add differences
4768	for fdiff in difffiles:
4769	difff = owdir + '/' + fdiff
4770	if moddiffscratch:
4771	sub.call('rm -rf' + owdir +' >& /dev/null', shell=True)
4772	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4773	objf = open(owdir+'/all_' + fdiff,'w')
4774	objf.write("## Computation and drawing of differences " + \
4775	"between '" + mod2+'@'+exp2 + "'-'" + mod1+'@'+exp1 + "'\n")
4776	objf.close()
4777	difff = owdir + '/all_vardiffstatistics.inf'
4778	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4779	objf = open(owdir+'/all_' + fdiff,'w')
4780	objf.write("## Computation of all differences statistics " + \
4781	"between '" + mod2+'@'+exp2 + "'-'" + mod1+'@'+exp1 + "'\n")
4782	objf.close()
4783
4784	if addmoddiffs:
4785	if dbg:
4786	print ' ' + main + ": adding model differences " + \
4787	"removing file '" + difff + "' ..."
4788	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4789
4790	# op and var differences
4791	diffvarcompf = owdir + '/' + difffiles[0]
4792	if not os.path.isfile(owdir+'/'+difffiles[0]) or \
4793	not os.path.isfile(owdir+'/'+difffiles[1]):
4794	alldiffop, alldiffvar, Nopdiffs, Nvardiffs = \
4795	diffvarop_listconstruct(cnf, difmods, difexps, difreproj,owdir,\
4796	dbg)
4797	else:
4798	print warnmsg
4799	print ' ' + main + ": getting 'op' and 'var' differences to " + \
4800	"compute already from file !!"
4801	alldiffop, Nopdiffops = read_diff_file(diffvarcompf)
4802	alldiffvar, Nopdiffvars = read_diff_file(owdir+'/'+difffiles[1])
4803
4804	# End of avoiding to repeat all the experiment search
4805
4806	print " For experiments '" + mod2+'@'+exp2 + "'-'" + mod+'@'+exp1+\
4807	"' is required to compute:", Nvar, "differences"
4808
4809	if dbg:
4810	print ' op differences to compute _______'
4811	gen.printing_dictionary(alldiffop)
4812	print ' var differences to compute _______'
4813	gen.printing_dictionary(alldiffvar)
4814
4815	# Computing differences
4816	##
4817	print " Computing operation differences ..."
4818	compute_op_diffs(cnf, alldiffop, owdir, moddiffscratch, dbg)
4819	print " Computing variable differences ..."
4820	compute_var_diffs(cnf, alldiffvar, owdir, moddiffscratch, dbg)
4821
4822	# Plotting operation differences
4823	##
4824	print " " + main + ": Plotting operation differences' figures ..."
4825	dirfigf = owdir + '/diffopplotsdraw.inf'
4826	if figmoddiffscratch:
4827	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4828
4829	objf = open(owdir+'/all_diffopfigures.inf','w')
4830	objf.write("## Drawing of all operation difference figures \n")
4831	objf.close()
4832
4833	if addmoddifffigures:
4834	if dbg:
4835	print ' ' + main + ": adding model differences' figures " + \
4836	"removing file '" + dirfigf + "' ..."
4837	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4838
4839	if not os.path.isfile(dirfigf):
4840	listplots, Nplt = plots_listconstruct(cnf, 'PLOTDIFFOP', dirfigf,\
4841	owdir, dbg)
4842	else:
4843	print warnmsg
4844	print ' ' + main + ": getting plots to draw already from file !!"
4845	listplots, Nplt = read_plot_file(dirfigf)
4846
4847	# End of avoiding to repeat all the plots search
4848
4849	print " For experiment 'operation' differences '" + Sexps + "' is "+\
4850	"required to plot:" , Nplt, "plots"
4851
4852	if dbg:
4853	print 'Plots to draw _______'
4854	gen.printing_dictionary(listplots)
4855
4856	draw_diff_plots(cnf, listplots, owdir, alldiffop, 'diffop', \
4857	figmoddiffscratch, dbg)
4858
4859	# Plotting variable differences
4860	##
4861	print " " + main + ": Plotting variable differences' figures ..."
4862	dirfigf = owdir + '/diffvarplotsdraw.inf'
4863	if figscratch:
4864	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4865
4866	objf = open(owdir+'/all_diffvarfigures.inf','w')
4867	objf.write("## Drawing of all variables difference figures \n")
4868	objf.close()
4869
4870	if adddifffigures:
4871	if dbg:
4872	print ' ' + main + ": adding differences's figures " + \
4873	"removing file '" + dirfigf + "' ..."
4874	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4875
4876	if not os.path.isfile(dirfigf):
4877	listplots, Nplt = plots_listconstruct(cnf,'PLOTDIFFVAR',dirfigf, \
4878	owdir, dbg)
4879	else:
4880	print warnmsg
4881	print ' ' + main + ": getting plots to draw already from file !!"
4882	listplots, Nplt = read_plot_file(dirfigf)
4883
4884	# End of avoiding to repeat all the plots search
4885
4886	print " For experiment 'variables' differences '" + Sexps+ "' is "+ \
4887	"required to plot:" , Nplt, "plots"
4888
4889	if dbg:
4890	print ' Plots to draw _______'
4891	gen.printing_dictionary(listplots)
4892
4893	draw_diff_plots(cnf, listplots, owdir, alldiffvar, 'diffvar', \
4894	figdiffscratch, dbg)
4895
4896	# end of exp2 loop
4897	# end of exp1 loop
4898	# end of mod2 loop
4899	# end of mod1 loop
4900
4901	### ## #
4902	# All model/exps linear plots
4903	### ## #
4904
4905	allmodexps = []
4906	for mod in mods:
4907	for exp in modexps[mod]:
4908	allmodexps.append(mod + '/' + exp)
4909
4910	print ' ***** ** * * ** * * ** * ** *****'
4911	print ' ** Multi models, multi experiments plots '
4912	print ' ***** ** * * ** * * ** * ** *****'
4913	print ' Model/experiments:', allmodexps
4914
4915	owdir = cnf['ofold'] + '/allmodexps'
4916	sout = sub.call('mkdir -p ' + owdir, shell=True)
4917
4918	# Plotting all model-experiments lines
4919	##
4920	print " " + main + ": Plotting all model-experiments variable figures ..."
4921	allf = owdir + '/allmodexp.inf'
4922	dirfigf = owdir + '/allmodexpfigures.inf'
4923	allfigf = owdir+'/all_figures.inf'
4924	if figallmodexpscratch:
4925	sout = sub.call('rm ' + allf + ' >& /dev/null', shell=True)
4926	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4927	sout = sub.call('rm ' + allfigf + ' >& /dev/null', shell=True)
4928
4929	objf = open(allfigf,'w')
4930	objf.write("## Drawing of all variables figures for all model-experiments\n")
4931	objf.close()
4932
4933	if addallmodexpfigures:
4934	if dbg:
4935	print ' ' + main + ": adding model-experiment differences removing " + \
4936	" file '" + allfigf + "' ..."
4937	print ' ' + main + ": adding model-experiment differences' figures " + \
4938	"removing file '" + dirfigf + "' ..."
4939	sout = sub.call('rm ' + allfigf + ' >& /dev/null', shell=True)
4940	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4941
4942	if not os.path.isfile(dirfigf):
4943	listplots, Nplt = plots_listconstruct(cnf, 'PLOTALLMODEXP', dirfigf, owdir, dbg)
4944	else:
4945	print warnmsg
4946	print ' ' + main + ": getting plots to draw already from file !!"
4947	listplots, Nplt = read_plot_file(dirfigf)
4948
4949	# End of avoiding to repeat all the plots search
4950
4951	print " Is required to plot:", Nplt, " all model-experiment plots"
4952
4953	if dbg:
4954	print ' Plots to draw _______'
4955	gen.printing_dictionary(listplots)
4956
4957	if not os.path.isfile(allf):
4958	# Constructing the information of modexp / variable
4959	allmodexpvar, Nallmodexpvar = allmodexps_listconstruct(cnf, allmodexps, owdir, dbg)
4960	else:
4961	allmodexpvar, Nallmodexpvar = allmodexps_read(allf)
4962
4963	draw_allmodexp_plots(cnf, allmodexpvar, listplots, modexps, owdir, figallmodexpscratch, dbg)
4964
4965	print main + ': all files and figures have been properly done !!'

Note: See TracBrowser for help on using the repository browser.

Download in other formats: