Context Navigation

model_graphics.py @ 2074

Last change on this file since 2074 was 1643, checked in by lfita, 7 years ago
Fixing concatenate instruction
File size: 217.5 KB

Line
1	# Python script to generate the files and the plots for a sensitivity study with multiple modle configurations and models
2	# L. Fita
3	# LMD, Jussieu/Palaisseau, France
4	# Script configuration get from ASCII file 'model_graphics.dat'
5
6	# `Wheeler-Kiladis' diagram: https://github.com/UV-CDAT/wk
7	# $ cd /home/lluis/Downloads
8	# $ git clone https://github.com/UV-CDAT/wk
9	# $ cd wk
10	# $ su
11	# # python setup.py build >& run_pysetup.log
12	# # python setup.py install >& run_pyinstall.log
13
14	import numpy as np
15	import nc_var_tools as ncvar
16	import generic_tools as gen
17	import drawing_tools as drw
18	import time as tim
19	# To avoid errors like: '/bin/sh: 1: Syntax error: Bad fd number'
20	# Make sure that /bin/sh directs to bash and not to dash:
21	# http://stackoverflow.com/questions/15809060/sh-syntax-error-bad-fd-number
22	import subprocess as sub
23	import os
24
25	main = 'model_graphics.py'
26
27	errmsg = ncvar.errormsg
28	warnmsg = ncvar.warnmsg
29
30	def verify_configuration(ExpConf, debug):
31	""" Function to verify mandatory keys from experiment configuration
32	ExpConf= dictionary with the configuration
33	"""
34	fname = 'verify_configuration'
35
36	# List with the mandatory keys
37	mandatorykeys = ['pyHOME', 'cdoHOME', 'scratch', 'filescratch', 'figscratch', \
38	'diffscratch', 'figdiffscratch', 'figallmodexpscratch', 'addfiles', \
39	'addfigures', 'adddiffs', 'adddifffigures', 'addallmodexpfigures', 'debug', \
40	'models', 'modgraphchar', 'expgraphchar', 'ifold', 'ofold', 'warnmsg', \
41	'errmsg', 'titleoperations', 'kindfig', 'CFreftime', 'CFunitstime', 'mapval', \
42	'timekind', 'timefmt', 'timelabel' ]
43
44	# List with the optional keys
45	optionalkeys = ['specificvarplot', 'specificdiffopplot', 'specificdiffvarplot', \
46	'RefProj']
47
48	# Dictionary with the mandatory keys which are required as function of a given
49	# key from the dictionary
50	# if [key] must exist [keyB] = [values]
51	ifkeys = {'WRFexps': ['models','WRF'], 'LMDZexps': ['models','LMDZ'], \
52	'WRF_LMDZexps': ['models','WRF_LMDZ'], 'DYNAMICOexps': ['models','DYNAMICO'], \
53	'mthDYNAMICOexps': ['models','mthDYNAMICO'], \
54	'WRFheaders': ['models','WRF'], 'WRF_LMDZheaders': ['models','WRF_LMDZ'], \
55	'LMDZheaders': ['models','LMDZ'], 'DYNAMICOheaders': ['models','DYNAMICO'], \
56	'mthDYNAMICOheaders': ['models','mthDYNAMICO']}
57
58	# Dictionary with the optional keys (at least one) which are required as function
59	# of a given key from the dictionary
60	# if [key] must exist at least one of the [keyA, keyB, keyC, ...] with values
61	ifopkeys = {'RefProj': ['reprojectvar_remapbil', 'reprojectvar_remapbic', \
62	'reprojectvar_remapdis', 'reprojectvar_remapnn', 'reprojectvar_remapcon', \
63	'reprojectvar_remapcon2', 'reprojectvar_remaplaf', 'reprojectvar_dis', \
64	'reprojecvar_dis']}
65
66	for kdict in mandatorykeys:
67	if not ExpConf.has_key(kdict):
68	print errmsg
69	print ' ' + fname + "' configuration without required '" + kdict + "' !!"
70	quit(-1)
71	if debug:
72	print ExpConf[kdict]
73
74	for kdict in ifkeys.keys():
75	vals = ifkeys[kdict]
76	keyn = vals[0]
77	keyv = vals[1]
78	if type(ExpConf[keyn]) == type('A'):
79	if not ExpConf.has_key(kdict) and ExpConf[keyn] == keyv:
80	print errmsg
81	print ' ' + fname + "' configuration without '" + kdict + \
82	"' when it is required with configuration '" + keyn + '=' + keyv + \
83	"' !!"
84	quit(-1)
85	elif type(ExpConf[keyn]) == type(['A', 'B']):
86	keyvals = ExpConf[keyn]
87	if not ExpConf.has_key(kdict) and gen.searchInlist(keyvals, keyv):
88	print errmsg
89	print ' ' + fname + "' configuration without '" + kdict + \
90	"' when it is required with configuration '" + keyn + '= [..., ' + \
91	keyv + ", ...]' !!"
92	quit(-1)
93	else:
94	print errmsg
95	print ' ' +fname+ 'dictionary type ', type(ExpConf[keyn]), ' not ready!!'
96	quit(-1)
97	if debug:
98	print ' Experiments configuration _______'
99	gen.printing_dictionary(ExpConf)
100
101	for kdict in ifopkeys.keys():
102	if ExpConf.has_key(kdict):
103	opkeys = ifopkeys[kdict]
104	hasopkey = False
105	opdictvals = {}
106	for opkv in opkeys:
107	if ExpConf.has_key(opkv):
108	hasopkey = True
109	opdictvals[opkv] = ExpConf[opkv]
110
111	if not hasopkey:
112	print errmsg
113	print ' ' + fname + "' configuration wit '" + kdict + "' but " + \
114	"without any key value:", opkeys, "' when it is required !!"
115	quit(-1)
116
117	if debug:
118	print " Optional key '" + kdict + "' with values ________"
119	gen.printing_dictionary(opdictvals)
120
121	print fname + ': configuration seems to be fine'
122
123	return
124
125	def scratches(config):
126	""" Function to set-up if it is needed to start from the scratch
127	config: dictionary with the configuration
128	"""
129	fname = 'scratches'
130
131	if gen.Str_Bool(config['scratch']):
132	scr = True
133	print warnmsg
134	print " " + fname + ": starting from the SCRATCH !!"
135	print " 10 seconds left!!"
136	filescr = True
137	figscr = True
138	difscr = True
139	figdifscr = True
140	moddifscr = True
141	figmoddifscr = True
142	figallmodexpscr = True
143	tim.sleep(10)
144	else:
145	scr = False
146	if gen.Str_Bool(config['filescratch']):
147	filescr = True
148	print warnmsg
149	print " " + main + ": files starting from the SCRATCH !!"
150	print " 5 seconds left!!"
151	tim.sleep(5)
152	else:
153	filescr = False
154
155	if gen.Str_Bool(config['figscratch']):
156	figscr = True
157	print warnmsg
158	print " " + main + ": figures starting from the SCRATCH !!"
159	print " 5 seconds left!!"
160	tim.sleep(5)
161	else:
162	figscr = False
163
164	if gen.Str_Bool(config['diffscratch']):
165	difscr = True
166	print warnmsg
167	print " " + main + ": experiment differences starting from the SCRATCH !!"
168	print " 5 seconds left!!"
169	tim.sleep(5)
170	else:
171	difscr = False
172
173	if gen.Str_Bool(config['figdiffscratch']):
174	figdifscr = True
175	print warnmsg
176	print " " + main + ": figures experiment differences starting from the SCRATCH !!"
177	print " 5 seconds left!!"
178	tim.sleep(5)
179	else:
180	figdifscr = False
181
182	if gen.Str_Bool(config['moddiffscratch']):
183	moddifscr = True
184	print warnmsg
185	print " " + main + ": model differences starting from the SCRATCH !!"
186	print " 5 seconds left!!"
187	tim.sleep(5)
188	else:
189	moddifscr = False
190
191	if gen.Str_Bool(config['figmoddiffscratch']):
192	figmoddifscr = True
193	print warnmsg
194	print " " + main + ": figures model differences starting from the SCRATCH !!"
195	print " 5 seconds left!!"
196	tim.sleep(5)
197	else:
198	figmoddifscr = False
199
200	if gen.Str_Bool(config['figallmodexpscratch']):
201	figallmodexpscr = True
202	print warnmsg
203	print " " + main + ": figures all model-experiment starting from the SCRATCH !!"
204	print " 5 seconds left!!"
205	tim.sleep(5)
206	else:
207	figallmodexpscr = False
208
209	if gen.Str_Bool(config['addall']):
210	print warnmsg
211	print " " + fname + ": adding new values everywhere !!"
212	addfils = True
213	addfigs = True
214	adddiffs = True
215	adddifffigs = True
216	addmoddiffs = True
217	addmoddifffigs = True
218	addallmodexpfigs = True
219	else:
220	addfils = gen.Str_Bool(config['addfiles'])
221	addfigs = gen.Str_Bool(config['addfigures'])
222	adddiffs = gen.Str_Bool(config['adddiffs'])
223	adddifffigs = gen.Str_Bool(config['adddifffigures'])
224	addmoddiffs = gen.Str_Bool(config['addmoddiffs'])
225	addmoddifffigs = gen.Str_Bool(config['addmoddifffigures'])
226	addallmodexpfigs = gen.Str_Bool(config['addallmodexpfigures'])
227
228	debug = gen.Str_Bool(config['debug'])
229
230	return scr, filescr, figscr, difscr, figdifscr, moddifscr, figmoddifscr, \
231	figallmodexpscr, addfils, addfigs, adddiffs, adddifffigs, addmoddiffs, \
232	addmoddifffigs, addallmodexpfigs, debug
233
234
235	class gc(object):
236	""" Class which holds all the graphical characteristics
237	[label]: label as it should appear in the graphics
238	[color]: specific color of the model
239	[linetype]: specific type of line of the model
240	[marker]: specific marker of the model
241	[sizes]: line width and point size for the model
242	[tmod]: time modification to apply to the model files (None for nothing)
243	'setorigin',[YYYYMMDDHHMISS]: re-set origin of times at [YYYYMMDDHHMISS]
244	"""
245	def __init__( self, label, color, linetype, marker, sizes, Tmod):
246	self.label = None
247	self.color = None
248	self.linetype = None
249	self.marker = None
250	self.sizes = None
251	self.tmod = None
252	if label is not None:
253	self.label = label
254	self.color = color
255	self.linetype = linetype
256	self.marker = marker
257	self.sizes = sizes
258	if Tmod != 'None':
259	self.tmod = Tmod
260
261	def graphical_characteristics(config, debug):
262	""" Function to provide the graphical characteristics of models and experiments
263	config= configuration of the experiment
264	"""
265	fname = 'graphical_characteristics'
266
267	modgraphchar = {}
268	modvals = config['modgraphchar'].split(':')
269	for modval in modvals:
270	modv = modval.split('\|')
271	modgraphchar[modv[0]] = gc(modv[1], modv[2], modv[3], modv[4], modv[5], modv[6])
272
273	expgraphchar = {}
274	expvals = config['expgraphchar'].split(':')
275	for expval in expvals:
276	expvs = expval.split('\|')
277	modexp = expvs[0] + '/' + expvs[1]
278	if not modgraphchar.has_key(expvs[0]):
279	print errmsg
280	print ' ' + fname + ": a model called '" +expvs[0]+ "' does not exist !!"
281	print ' existing models with graphic characteristics:', \
282	modgraphchar.keys()
283	quit(-1)
284
285	modvs = modgraphchar[expvs[0]]
286	expv = expvs[2:7]
287	if expv[1] == 'asmodel': expv[1] = modvs.color
288	if expv[2] == 'asmodel': expv[2] = modvs.linetype
289	if expv[3] == 'asmodel': expv[3] = modvs.marker
290	if expv[4] == 'asmodel': expv[4] = modvs.sizes
291
292	expgraphchar[modexp] = gc(expv[0], expv[1], expv[2], expv[3], expv[4], 'None')
293
294	# Running some tests
295	mods = config['models'].split(':')
296	for mod in mods:
297	if not modgraphchar.has_key(mod):
298	print errmsg
299	print ' ' + fname + ": model called '" + mod + "' does not have " + \
300	"graphical characteristics !!"
301	print ' existing models with graphic characteristics:', \
302	modgraphchar.keys()
303	quit(-1)
304
305	exps = config[mod + 'exps'].split(':')
306	for exp in exps:
307	modexp = mod + '/' + exp
308	if not expgraphchar.has_key(modexp):
309	print errmsg
310	print ' ' + fname + ": model/experiment called '" + modexp + \
311	"' does not have graphical characteristics !!"
312	print ' existing model/experiments with graphic ' + \
313	'characteristics:', expgraphchar.keys()
314	quit(-1)
315
316	if debug:
317	print ' ' + fname + ': model graphical characteristics _______'
318	for modn in modgraphchar.keys():
319	with gen.Capturing() as output:
320	gen.printing_class(modgraphchar[modn])
321	print "'" + modn + "'; " + ', '.join(output)
322	print ' ' + fname + ': experiment graphical characteristics _______'
323	for modexpn in expgraphchar.keys():
324	with gen.Capturing() as output:
325	gen.printing_class(expgraphchar[modexpn])
326	print "'" + modexpn + "'; " + ', '.join(output)
327
328	if debug: print " End of '" + fname + "' "
329
330	return modgraphchar, expgraphchar
331
332	def exp_headers(mod,config):
333	""" Function to provide the headers and the experiments of a given model
334	model= model
335	config= configuration of the experiment
336	"""
337	fname = 'exp_headers'
338
339	# No CASE in python!
340	# case ${mod} in ...
341	if mod == 'WRF':
342	expers = config['WRFexps'].split(':')
343	fhs = config['WRFheaders'].split(':')
344	elif mod == 'LMDZ':
345	expers = config['LMDZexps'].split(':')
346	fhs = config['LMDZheaders'].split(':')
347	elif mod == 'WRF_LMDZ':
348	expers = config['WRF_LMDZexps'].split(':')
349	fhs = config['WRF_LMDZheaders'].split(':')
350	elif mod == 'DYNAMICO':
351	expers = config['DYNAMICOexps'].split(':')
352	fhs = config['DYNAMICOheaders'].split(':')
353	elif mod == 'mthDYNAMICO':
354	expers = config['mthDYNAMICOexps'].split(':')
355	fhs = config['mthDYNAMICOheaders'].split(':')
356	else:
357	print errmsg
358	print " " + fname + ": model '" + mod + "' not ready!!"
359	quit(-1)
360
361	return expers, fhs
362
363	class VariableInf(object):
364	""" Class which holds all information of a given variable
365	name= CF name of the variabgle
366	header= header of the file from which it can be computed
367	model= file-header variable from which it can be directly computed
368	diag= file-header diagnostics from which it can be computed
369	"""
370	def __init__( self, name, fheader, model, diag):
371	self.name= None
372	self.fheader= None
373	self.model= None
374	self.diag= None
375	if name is not None:
376	self.name = name
377	self.fheader= fheader
378	self.model= model
379	self.diag= diag
380
381	class ModelInf(object):
382	""" Class which holds all information from a given model
383	name= Acronym of the model
384	model= Long description of the model
385	dim[x/y/z/t/s]n= name of the dimensions of the model
386	vdim[x/y/z/t/s]n= name of the variable-dimensions of the model
387	dimensions= list of the dimensions
388	vardimensions= list of the variable-dimensions
389	timemodif= time modification to apply to the model files ('None' for nothing)
390	'setorigin',[YYYYMMDDHHMISS]: re-set origin of times at [YYYYMMDDHHMISS]
391	"""
392	def __init__( self, name, model, dx, dy, dz, dt, ds, vdx, vdy, vdz, vdt, vds, \
393	tmod):
394	self.name= None
395	self.model= None
396	self.dimxn= None
397	self.dimyn= None
398	self.dimzn= None
399	self.dimtn= None
400	self.dimsn= None
401	self.vardxn= None
402	self.vardyn= None
403	self.vardzn= None
404	self.vardtn= None
405	self.vardsn= None
406	self.timemodif = None
407	if name is not None:
408	self.name = name
409	self.model= model
410	self.dimxn= dx
411	self.dimyn= dy
412	self.dimzn= dz
413	self.dimtn= dt
414	self.dimsn= ds
415	self.vardxn= vdx
416	self.vardyn= vdy
417	self.vardzn= vdz
418	self.vardtn= vdt
419	self.vardsn= vds
420	if tmod is not None:
421	self.timemodif = tmod
422	self.dimensions = [dx, dy, dz, dt, ds]
423	self.vardimensions = [vdx, vdy, vdz, vdt, vds]
424
425	def variable_compute(idir,var,ftests,db):
426	""" Function to retrieve the computation way of a given variable using a series of test files
427	iwdir= directory with the test files
428	var= name of the variable to test
429	filtests= dictionary with the test files for each header
430	"""
431	fname='variable_compute'
432
433	cancompute = True
434	for headerf in ftests.keys():
435	filen=ftests[headerf]
436	try:
437	with gen.Capturing() as output:
438	vmod, vdiag = ncvar.computevar_model(var, idir + '/' + filen)
439	except:
440	print errmsg
441	print 'ncvar.computevar_model(' + var + ', ' + idir + '/' + filen + ')'
442	for s1out in output: print s1out
443	quit(-1)
444
445	if vmod is None and vdiag is None:
446	cancompute = False
447	else:
448	cancompute = True
449	# Should be considered that variable can also be computed by both ways?
450	break
451
452	if not cancompute:
453	print warnmsg
454	print ' ' + fname + ": there is no way to compute '" + var + \
455	"' for model '" + mod
456	# Too extrict!
457	# quit(-1)
458
459	# Getting only the first possibility of model and diagnostic
460	if vmod is not None:
461	modV = vmod[0]
462	else:
463	modV = None
464	if vdiag is not None:
465	diagV = vdiag[0]
466	else:
467	diagV = None
468
469	varcomp = VariableInf(var, headerf, modV, diagV)
470
471	# a ';' list 'varcompute' it is created for each variable giving:
472	# [var]\|[vark]\|[headerf][varmod]\|[vardiag]
473	# This list will be used to compute a new file for each variable
474
475	if db:
476	print 'Variable information _______'
477	gen.printing_class(varcomp)
478
479	if db: print " End of '" + fname + "' "
480
481	return varcomp
482
483	def get_operations_var(vc,db):
484	""" Function to provide the operations to make for each variable from 'VAR_' dictionary
485	vc= 'VAR_' dictionary, dictionary with all the parameters started with 'VAR_'
486	"""
487	fname = 'get_operations_var'
488
489	LH = len('VAR_')
490	iop = 1
491	# list of operations
492	doopers = {}
493	# operations by variable
494	operationsvar = {}
495	# individual operations by variable (not repeated)
496	indivoperationsvar = {}
497
498	# Variables with a calculation whic requires vertical interpolation to all the
499	# file (operations stating by 'VAR_pinterp')
500	LVp = len('VAR_pinterp')
501	varglobalp = []
502
503	for oper in vc.keys():
504	Loper = len(oper)
505	opn = oper[LH:Loper+1]
506	vns = vc[oper].split(':')
507	ops1 = opn.split('+')
508	doopers[opn] = ops1
509	for vn in vns:
510	if not operationsvar.has_key(vn):
511	operationsvar[vn] = [opn]
512	indivoperationsvar[vn] = ops1
513	else:
514	opers = operationsvar[vn]
515	opers.append(opn)
516	operationsvar[vn] = opers
517	opers1 = indivoperationsvar[vn]
518	for op1 in ops1:
519	if not gen.searchInlist(opers1, op1):
520	opers1.append(op1)
521	indivoperationsvar[vn] = opers1
522
523	if oper[0:LVp] == 'VAR_pinterp':
524	for vn in vns:
525	if not gen.searchInlist(varglobalp,vn): varglobalp.append(vn)
526
527	iop = iop + 1
528
529	if db:
530	print ' operations to make _______'
531	gen.printing_dictionary(doopers)
532	print ' operations by variable _______'
533	gen.printing_dictionary(operationsvar)
534	print ' individual operations by variable _______'
535	gen.printing_dictionary(indivoperationsvar)
536	print ' variables with the vertical interpolation for all the file _______'
537	print ' #', varglobalp
538
539	if db: print " End of '" + fname + "' "
540
541	return doopers, operationsvar, indivoperationsvar, varglobalp
542
543	def pinterp_var(oper):
544	""" Function to retrieve characteristics of the vertical interpolation for the operation
545	oper= operation
546	Wether vertical interpolation is:
547	# 'global': for all file
548	# 'local': at a given step of the process
549	# 'none': no vertical interpolation
550	>>> pinterp_var('last+pinterp+xmean')
551	local
552	>>> pinterp_var('pinterp+tmean+xmean')
553	global
554	>>> pinterp_var('xmean')
555	none
556	"""
557	fname = 'pinterp_var'
558
559	pinn = 'pinterp'
560
561	Lpinterp = len(pinn)
562	if oper[0:Lpinterp] == pinn:
563	pkind = 'global'
564	return pkind
565
566	if oper.find(pinn) != -1:
567	pkind = 'local'
568	else:
569	pkind = 'none'
570
571	return pkind
572
573	def compvars_listconstruct(config, minf, Files, TestFiles, idir, odir, debug):
574	""" Function to construct the list of variables to compute
575	config= dictionary with the configuration of the execution
576	variables to compute are get with all values started by `VAR_' in 'module_graphics.dat', and then
577	providing a consecutive number of calculations separated by '+'
578	VAR_[calc1]+[calc2] = tas:wss
579	will compute first [calc1] and then [calc2] for 'tas' and 'wss'
580	minf= class with information about the model
581	Files= dictionary of files for heach header
582	TestFiles= dictionary of files to test how to compute the variable
583	odir= output directory
584	"""
585	fname='compvars_listconstruct'
586
587	varcomp = gen.get_specdictionary_HMT(config,H='VAR_')
588
589	if debug:
590	print ' variables to compute ________'
591	gen.printing_dictionary(varcomp)
592
593	# Getting operations by variable
594	opers, varoper, indivaroper, varglobp = get_operations_var(varcomp,debug)
595
596	strfiles = gen.dictKeysVals_stringList(Files)
597	strtestfiles = gen.dictKeysVals_stringList(TestFiles)
598
599	Svarcompute = ''
600
601	# Main dictionary with all the variables and their calculation
602	allvarcomp = {}
603
604	ivop = 0
605	for vn in varoper:
606	vcomp = variable_compute(idir,vn,TestFiles,False)
607	for op in varoper[vn]:
608	# Creation of a String as: [CFname]\|[operation]\|[fheader]\|[vmodel]\|[vdiag]\|[globalP]
609	# [CFname]: CF-name of the variable (must appear in 'variables_values.dat')
610	# [operation]: operation to compute
611	# [fheader]: header of file with the required variables
612	# [vmodel]: Variable from model output
613	# [vdiag]: Varible as diagnostic from different variables of the model output
614	# [globalP]: Wether vertical interpolation is:
615	# 'global': for all file
616	# 'local': at a given step of the process
617	# 'none': no vertical interpolation
618	globalP = pinterp_var(op)
619	if vcomp.model is not None:
620	if type(vcomp.model) == type(list([1, 2])):
621	Smodel = ':'.join(vcomp.model)
622	elif type(vcomp.model) == type('A'):
623	Smodel = vcomp.model
624	else:
625	Smodel = 'None'
626	if vcomp.diag is not None:
627	if type(vcomp.diag) == type(list([1, 2])):
628	Sdiag = ':'.join(vcomp.diag)
629	elif type(vcomp.diag) == type('A'):
630	Sdiag = vcomp.diag
631	else:
632	Sdiag = 'None'
633
634	# To get a combination of operations, all the precedent steps are kept
635	# thus, they are also available !
636	seqops = op.split('+')
637	for seqop in seqops:
638	if seqop == seqops[0]: seqopS=seqop
639	else: seqopS = seqopS + '+' + seqop
640	allvarcomp[vn+'_'+seqopS] = [vcomp.fheader, vcomp.model, vcomp.diag, \
641	globalP]
642
643	Svc = vcomp.name + '\|' + seqopS + '\|' + vcomp.fheader + '\|' + \
644	Smodel + '\|' + Sdiag + '\|' + globalP
645	if ivop == 0:
646	Svarcompute = Svc
647	else:
648	Svarcompute = Svarcompute + ',' + Svc
649
650	ivop = ivop + 1
651
652	if debug:
653	print ' Variables to compute _______'
654	gen.printing_dictionary(allvarcomp)
655
656	# Outwritting the varcompute to avoid next time (if it is not filescratch!)
657	objf = open(odir + '/varcompute.inf', 'w')
658	objf.write('files: ' + strfiles + '\n')
659	objf.write('testfiles: ' + strtestfiles + '\n')
660	objf.write('varcompute: ' + Svarcompute + '\n')
661	objf.write('itotv: ' + str(ivop) + '\n')
662	objf.close()
663
664	if debug: print " End of '" + fname + "' "
665
666	return allvarcomp, ivop
667
668	def read_varcomp_file(filen):
669	""" Function to read 'varcompute.inf' and reconstruct the dictionaries
670	"""
671	fname = 'read_varcomp_file'
672
673	allvarcomp = {}
674
675	objf = open(filen, 'r')
676
677	for line in objf:
678	vals = line.replace('\n','').split(' ')
679	if vals[0] == 'files:':
680	Files = gen.stringList_dictKeysVals(vals[1])
681	elif vals[0] == 'testfiles:':
682	TestFiles = gen.stringList_dictKeysVals(vals[1])
683	elif vals[0] == 'varcompute:':
684	for VvalsS in vals[1].split(','):
685
686	Vvals = VvalsS.split('\|')
687	if Vvals[3] == 'None':
688	mod = None
689	else:
690	mod = Vvals[3].split(':')
691	if Vvals[4] == 'None':
692	diag = None
693	else:
694	diag = Vvals[4].split(':')
695
696	allvarcomp[Vvals[0]+'_'+Vvals[1]] = [Vvals[2], mod, diag, Vvals[5]]
697
698	# # To get a combination of operations, all the precedent steps are kept
699	# # thus, they are also available !
700	# seqops = Vvals[1].split('+')
701	# for seqop in seqops:
702	# if seqop == seqops[0]: seqopS=seqop
703	# else: seqopS = seqopS + '+' + seqop
704	# allvarcomp[Vvals[0]+'_'+seqopS] = [Vvals[2], mod, diag, Vvals[5]]
705	elif vals[0] == 'itotv:':
706	ivop = int(vals[1])
707
708	objf.close()
709
710	return Files, TestFiles, allvarcomp, ivop
711
712	def pinterpS(gP):
713	""" For that variables which require vertical interpolation 'p' suffix to the file header is added
714	gP= kind of pinterp: 'global', 'local', 'none'
715	"""
716	fname = 'pinterpS'
717
718	if gP != 'none':
719	gPS = 'p'
720	else:
721	gPS = ''
722
723	return gPS
724
725	def compute_variable(minf, idir, usefiles, odir, cvar, gP, scr, pyH, Tref, Tunits, db):
726	""" Function to compute a variable
727	minf= class with the information of the model
728	idir= directory with the input files
729	usefiles= dictionary of files as dict([headerf]) = [file1], ..., [fileN]
730	odir= directory to write the output files
731	cvar= class with the information of the variable: 'name', 'fheader', 'varmod', 'vardiag'
732	gP= kind of vertical interpolation ('global', 'local', 'none')
733	scr= should it be done from the scratch?
734	pyH= location of the python HOME
735	Tref= CF time reference
736	Tunits= CF time units
737	"""
738	fname='compute_variable'
739
740	CFvarn=cvar.name
741	headerf = cvar.fheader
742	modvar = cvar.model
743	diagvar = cvar.diag
744
745	cfiles = usefiles[headerf]
746
747	# dimensions
748	dnx = minf.dimxn
749	dny = minf.dimyn
750	# var-dimensions
751	vdnx = minf.vardxn
752	vdny = minf.vardyn
753
754	# Computing separately and then joinging for all files
755	Ntotfiles = len(cfiles)
756	Nzeros = len(str(Ntotfiles))
757	NStot = str(Ntotfiles).zfill(Nzeros)
758	# For that variables which require vertical interpolation 'p' suffix to the
759	# file header is added
760	SgP = pinterpS(gP)
761
762	# Getting in working dir
763	os.chdir(odir)
764
765	# File to keep track of all operations
766	otrackf = open( odir + '/all_computevars.inf', 'a')
767
768	# Computing variable
769	ifile=1
770	for cf in cfiles:
771	ifS = str(ifile).zfill(Nzeros)
772	ifilen = odir + '/' + CFvarn + '_' + headerf + SgP + '_' + ifS + '-' + \
773	NStot + '.nc'
774	fileon = odir + '/' + CFvarn + '_' + headerf + SgP + '.nc'
775
776	if scr:
777	sout = sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
778	sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
779
780	if not os.path.isfile(ifilen) and not os.path.isfile(fileon):
781	# Since model direct values are retrieved from `variables_valules.dat' which was
782	# initially coincived as a way to only give variable attributes, range and color
783	# bars, if a variable has a diagnostic way to be computed, the later one will be
784	# preferred
785	if db:
786	print ' ' + fname + ": creation of variable file '" + CFvarn + \
787	"' in file '" + ifilen + "' ..."
788	print ' model variable:', modvar
789	print ' diagostic variable:', diagvar
790
791	if diagvar is None:
792	# model variable
793	if db: print ' '+fname+ ": variable direct from model '" +modvar+ "'"
794	values = modvar + ',0,-1,-1'
795	vs = modvar + ',' + vdnx + ',' + vdny + ',' + vdnz + ',' + vdnt
796
797	try:
798	with gen.Capturing() as output:
799	ncvar.DataSetSection_multivars(values, idir + '/' + cf, vs)
800	except:
801	print errmsg
802	print 'ncvar.DataSetSection_multivars('+values+', '+idir+'/'+cf+ \
803	', '+vs+')'
804	for s1out in output: print s1out
805	quit(-1)
806
807	newfile, loc = gen.search_sec_list(output,'succesfull')
808	ofile = newfile[0].split(' ')[7]
809	sub.call('mv ' + ofile + ' ' + ifilen, shell=True)
810
811	# Keeping track of the operations
812	pyins = pyH + "/nc_var.py -f " + idir + '/' + cf + \
813	" -o DataSetSection_multivars -v " + vs + " -S '" + values + "'"
814	otrackf.write('\n')
815	otrackf.write('# ' + CFvarn + " " + modvar + '\n')
816	otrackf.write('python ' + pyins + '\n')
817
818	# CF renaming of variable
819	ncvar.chvarname(CFvarn,ifilen,modvar)
820	else:
821	# diagnostic variable
822	if db: print ' '+fname+ ": variable as diagnostic '" +diagvar[0]+ "'"
823	if minf.name == 'WRF':
824	dims = dnt+'@WRFtime,'+dnz+'@'+vdnz+','+dny+'@'+vdny+','+dnx+ \
825	'@'+vdnx
826	else:
827	dims = dnt+'@'+vdnt+','+dnz+'@'+vdnz+','+dny+'@'+vdny+','+dnx+ \
828	'@'+vdnx
829	diagn = diagvar[0]
830	Ndiagvars = len(diagvar)
831	diagc = '@'.join(diagvar[1:])
832
833	values = '-f ' + idir + '/' + cf + " -d '" + dims + "' -v '" + \
834	diagn + '\|' + diagc + "'"
835	try:
836	with gen.Capturing() as output:
837	sout = sub.call('python ' +pyH+ '/diagnostics.py ' + values, \
838	shell=True)
839	except:
840	print errmsg
841	print 'python ' + pyH + '/diagnostics.py ' + values
842	for s1out in output: print s1out
843	quit(-1)
844	if db:
845	for s1out in output: print s1out
846
847	sout = sub.call(' mv diagnostics.nc ' + ifilen, shell=True)
848
849	# Keeping track of the operations
850	pyins = 'python ' + pyH + '/diagnostics.py ' + values
851	otrackf.write('\n')
852	otrackf.write('# ' + CFvarn + " " + diagn + '\n')
853	otrackf.write(pyins + '\n')
854
855	# Attaching necessary variables for the pressure interpolation
856	if gP != 'none':
857	if minf.name == 'WRF' or minf.name == 'WRF_LMDZ':
858	requiredinterpvars = ['P', 'PB', 'PSFC', 'PH', 'PHB', 'HGT', 'T',\
859	'QVAPOR', 'XLONG', 'XLAT', 'Times']
860	elif minf.name == 'LMDZ':
861	requiredinterpvars = ['pres', 'psol', 'geop', 'phis', 'temp', \
862	'ovap', 'lon', 'lat', 'time_counter']
863	elif minf.name == 'DYNAMICO':
864	requiredinterpvars = ['pres', 'psol', 'geop', 'phis', 'temp', \
865	'ovap', 'lon', 'lat', 'time_counter']
866	elif minf.name == 'mthDYNAMICO':
867	requiredinterpvars = ['pres', 'psol', 'geop', 'phis', 'temp', \
868	'ovap', 'lon', 'lat', 'time_centered']
869	else:
870	print errmsg
871	print fname + ": for model '" + minf.name + "' required " + \
872	" variables for vertical interpolation are not known !!"
873	quit(-1)
874
875	print " " + fname + ": adding variables:", requiredinterpvars, \
876	' to allow pressure interpolation'
877	for rqv in requiredinterpvars:
878	try:
879	with gen.Capturing() as output:
880	ncvar.fvaradd(idir+'/'+cf+','+rqv,ifilen)
881	except:
882	print errmsg
883	print 'fvaradd('+idir+'/'+cf+','+rqv+', '+ifilen+')'
884	for s1out in output: print s1out
885	quit(-1)
886	if db:
887	for s1out in output: print s1out
888
889	# CFification of files
890	if minf.name == 'WRF' or minf.name == 'WRF_LMDZ':
891	# Assuming that all diagnostics are outputed with CF-times
892	values = vdnx + ':' + vdny + ':' + 'time:'+ Tref + ':' + Tunits
893	try:
894	with gen.Capturing() as output:
895	ncvar.WRF_to_newCF(values, ifilen, 'all')
896	sub.call('mv CF_WRF.nc '+ifilen + ' >& /dev/null', shell=True)
897	except:
898	print errmsg
899	print 'WRF_to_newCF('+ values +', ' + ifilen + ',all)'
900	for s1out in output: print s1out
901	# Removing file in order to make sure that it will be redone
902	print ' ' + fname + " removing file '" + ifilen + "' ..."
903	sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
904	quit(-1)
905	if db:
906	for s1out in output: print s1out
907	elif minf.name == 'LMDZ':
908	try:
909	with gen.Capturing() as output:
910	ncvar.LMDZ_toCF(Tunits + '!since!' + Tref, ifilen)
911	except:
912	print errmsg
913	print 'LMDZ_toCF('+ Tunits +'!since!'+ Tref + ', ' + ifilen + ')'
914	for s1out in output: print s1out
915	# Removing file in order to make sure that it will be redone
916	print ' ' + fname + " removing file '" + ifilen + "' ..."
917	sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
918	quit(-1)
919	if db:
920	for s1out in output: print s1out
921	# At this stage, let's be simple. It might be necessary to complicate it! Since
922	# DYNAMICO might have 'time_counter' or 'time_instant'
923	elif minf.name == 'DYNAMICO':
924	try:
925	with gen.Capturing() as output:
926	ncvar.DYNAMICO_toCF(Tunits + '!since!' + Tref, ifilen)
927	except:
928	print errmsg
929	print 'DYNAMICO_toCF('+Tunits+'!since!'+Tref+ ', ' + ifilen + ')'
930	for s1out in output: print s1out
931	# Removing file in order to make sure that it will be redone
932	print ' ' + fname + " removing file '" + ifilen + "' ..."
933	sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
934	quit(-1)
935	if db:
936	for s1out in output: print s1out
937	elif minf.name == 'mthDYNAMICO':
938	try:
939	with gen.Capturing() as output:
940	ncvar.mthDYNAMICO_toCF(Tunits + '!since!' + Tref, ifilen)
941	except:
942	print errmsg
943	print 'mthDYNAMICO_toCF('+Tunits+'!since!'+Tref+ ', ' + ifilen + ')'
944	for s1out in output: print s1out
945	# Removing file in order to make sure that it will be redone
946	print ' ' + fname + " removing file '" + ifilen + "' ..."
947	sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
948	quit(-1)
949	if db:
950	for s1out in output: print s1out
951	else:
952	print errmsg
953	print ' ' + fname + ": no CFification for model '" +minf.name+ "' !!"
954	print " available ones: 'WRF', 'WRF_LMDZ', 'LMDZ', 'DYNAMICO'" + \
955	", 'mthDYNAMICO'"
956	quit(-1)
957
958	ifile = ifile + 1
959
960	otrackf.close()
961
962	# Joining variable files
963	if not os.path.isfile(fileon):
964	if Ntotfiles > 1:
965	if db:
966	print ' ' + fname + ": concatenating all variable files to " + \
967	"create '" + fileon + "..."
968	try:
969	with gen.Capturing() as output:
970	ncvar.netcdf_fold_concatenation_HMT(odir+',time,time', CFvarn + \
971	'_' + headerf + SgP +'_,-,.nc', 'all')
972	except:
973	print errmsg
974	print 'netcdf_fold_concatenation_HMT('+odir+',time,time ' + CFvarn + \
975	'_' + headerf +SgP + '_,-,.nc, all)'
976	for s1out in output: print s1out
977	quit(-1)
978	if db:
979	for s1out in output: print s1out
980
981	sout = sub.call('mv netcdf_fold_concatenated_HMT.nc '+fileon, shell=True)
982	if os.path.isfile(fileon):
983	sout = sub.call('rm '+CFvarn + '_' + headerf + SgP + '_-.nc >& ' + \
984	'/dev/null', shell=True)
985	else:
986	sout = sub.call('mv ' + CFvarn + '_' + headerf + SgP + '_1-1.nc ' + \
987	fileon, shell=True)
988
989	# Re-setting time of final concatenated file
990	if minf.timemodif is not None:
991	if db:
992	print ' ' + fname + ": Modifying times of the file by '" + \
993	minf.timemodif + "' !!"
994	try:
995	with gen.Capturing() as output:
996	ncvar.time_reset(minf.timemodif, fileon,'time')
997	except:
998	print errmsg
999	print 'time_reset(' + minf.timemodif + ', ' + fileon + ', time)'
1000	for s1out in output: print s1out
1001	# Removing file in order to make sure that it will be redone
1002	print ' ' + fname + " removing file '" + fileon + "' ..."
1003	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1004	quit(-1)
1005	if db:
1006	for s1out in output: print s1out
1007
1008	if db: print " End of '" + fname + "' "
1009
1010	return
1011
1012	def compute_statistics(minf, config, idir, usefiles, odir, cvar, gP, Opers, scr, db):
1013	""" Function to compute different statistics it will take previous steps if they
1014	are availale
1015	minf= class with the information of the model
1016	config= dictionary with the configuration of the experiment
1017	idir= directory with the input files
1018	usefiles= ',' list of files to use [file1],...,[fileN]
1019	odir= directory to write the output files
1020	cvar= class with the information of the variable: 'name', 'fheader', 'varmod', 'vardiag'
1021	gP= kind of vertical interpolation ('global', 'local', 'none')
1022	Opers= kind of operation: (as possible multiple consecutive combination of operations separated by '+'
1023	[calc1]+[calc2] will compute first [calc1] and then [calc2]
1024	acc: temporal accumulated values
1025	diff: differences between models
1026	direct: no statistics
1027	last: last temporal value
1028	Lmean: latitudinal mean values
1029	Lsec: latitudinal section (latitudinal value must be given, [var]@[lat])
1030	lmean: longitudinal mean values
1031	lsec: longitudinal section (longitudinal value must be given, [var]@[lat])
1032	pinterp: pressure interpolation (to the given $plevels)
1033	smean: spatial-weighted mean values
1034	tmean: temporal mean values
1035	tstd: temporal standard deviation values
1036	tturb: Taylor's turbulence decomposition value (x - <x>) for time
1037	tvar: temporal variance values
1038	xmean: x-axis mean values
1039	xvar: x-axis variance values
1040	ymean: y-axis mean values
1041	zsum: vertical aggregated values
1042	scr= should it be done from the scratch?
1043	"""
1044	fname='compute_statistics'
1045
1046	# Getting a previous file name to continue with(for combinations)
1047	CFvarn=cvar.name
1048	headerf = cvar.fheader
1049	modvar = cvar.model
1050	diagvar = cvar.diag
1051
1052	cfiles = usefiles[headerf]
1053
1054	# dimensions
1055	dnx = minf.dimxn
1056	dny = minf.dimyn
1057	# var-dimensions
1058	vdnx = minf.vardxn
1059	vdny = minf.vardyn
1060	# Model dimension-variable dictionary
1061	Modeldimvardict = {dnx: vdnx, dny: vdny, dnz: vdnz, dnt: vdnt}
1062
1063	# Some experiment configuration values
1064	# plevels= ':' separated list of pressures (in Pa) to use for the vertical interpolation
1065	plevels = config['plevels']
1066	# pyH= location of the python HOME
1067	pyH = config['pyHOME']
1068	# Tref= CF time reference
1069	Tref = config['CFreftime']
1070	# Tunits= CF time units
1071	Tunits = config['CFunitstime']
1072	# opsur = operation surnames
1073	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
1074	Opsurs = {}
1075	for opk in opsur:
1076	opn = opk.split('_')[1]
1077	vals = opsur[opk]
1078	Opsurs[opn] = vals.split(':')
1079	# if db:
1080	# print ' ' + fname + ' operation surnames _______'
1081	# gen.printing_dictionary(Opsurs)
1082
1083	# Getting in working dir
1084	os.chdir(odir)
1085
1086	# File to keep track of all operations
1087	otrackf = open( odir + '/all_statsvars.inf', 'a')
1088
1089	# For that variables which require vertical interpolation 'p' suffix to the file
1090	# header is added.
1091	# After some operations some CF dimension-variables might not be kept
1092
1093	if gP != 'none':
1094	SgP = 'p'
1095	varnCFs = ['lon', 'lat', 'pres', 'time']
1096	else:
1097	SgP = ''
1098	varnCFs = ['lon', 'lat', 'time']
1099
1100	# CF dimension-variable dictionary
1101	CFdimvardict = {'lon': 'lon', 'lat': 'lat', 'pres': 'pres', 'time': 'time'}
1102
1103	# Input file
1104	ifilen = odir + '/' + CFvarn + '_' + headerf + SgP + '.nc'
1105
1106	# Computing variable statisitcs
1107	istats=0
1108
1109	# List of not computed statistics
1110	wrongstats = []
1111
1112	opers = Opers.split('+')
1113	Fopers = ''
1114	if db: print " computing statistics of variable:'", CFvarn, "' operation '" + \
1115	Opers + "' using file '" + ifilen + "...'"
1116	for op in opers:
1117
1118	# File name from previous operations, and name of the variable within the
1119	# file (some operations change the name of the variable)
1120	if op == opers[0]:
1121	Fopers = op
1122	prevfile = ifilen
1123	vninF = CFvarn
1124	else:
1125	Fopers = Fopers + '_' + op
1126	prevfile = fileon
1127	fileon = odir + '/' + CFvarn + '_' + headerf + SgP + '_' + Fopers + '.nc'
1128
1129	# Adding required variables for the vertical interpolation in all operations
1130	SvarnCFs = ',' + ','.join(varnCFs)
1131
1132	if gP != 'none':
1133	if gP == 'local':
1134	CFvarnp = vninF + ',P,PB,PSFC,PH,PHB,HGT,T,QVAPOR,XLONG,XLAT,Times'+ \
1135	SvarnCFs
1136	else:
1137	CFvarnp = vninF + SvarnCFs
1138	else:
1139	CFvarnp = vninF + SvarnCFs
1140
1141	if scr:
1142	sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1143
1144	hprevfile = prevfile[0:len(prevfile)-3]
1145
1146	if db: print ' ', op, "using '" + prevfile + "' "
1147	# Just produce file in case output file does not exist
1148	if not os.path.isfile(fileon):
1149	if db:
1150	print ' ' + fname + ": creation of file '" + fileon + "' ..."
1151	dofile = True
1152	else:
1153	dofile = False
1154
1155	# Variables to be kept in the final file
1156	varkeep = []
1157
1158	if op == 'acc':
1159	# temporal accumulated values
1160	print " " + fname + ": kind '" + op + "' not ready !!"
1161	wrongstats.append(CFvarn + '_' + opers)
1162	break
1163
1164	elif op[0:11] == 'dimvarvalue':
1165	# Slicing along the index at the nearest given value of a dimension-variable
1166	dimvarn = op.split('~')[1]
1167	dimvalue = op.split('~')[2]
1168	vals = dimvarn + ',' + dimvalue + ',' + dimvalue + ',0'
1169	if dofile:
1170	try:
1171	with gen.Capturing() as output:
1172	ncvar.DataSetSection_multivars(vals,prevfile,CFvarnp)
1173	except:
1174	print errmsg
1175	print 'DataSetSection_multivars('+vals+',', prevfile, ',' + \
1176	CFvarnp + ')'
1177	for s1out in output: print s1out
1178	quit(-1)
1179
1180	if db:
1181	for s1out in output: print s1out
1182
1183	sout = sub.call('mv DataSetSection_multivars.nc '+fileon, shell=True)
1184
1185	# Keeping the operations
1186	pyins=pyH + "/nc_var.py -o DataSetSection_multivars -S '" + vals + \
1187	"' -f " + prevfile + " -v " + CFvarnp
1188	otrackf.write("\n")
1189	otrackf.write("# " + CFvarnp + " " + Fopers + "\n")
1190	otrackf.write(pyins + "\n")
1191
1192	# removing the given variable-dimension
1193	varnCFs.remove(dimvarn)
1194
1195	elif op == 'direct':
1196	# no statistics
1197	if dofile:
1198	sout = sub.call('mv ' + prevfile + ' ' + fileon, shell=True)
1199	elif op == 'last':
1200	# last temporal value
1201	vals='time,-9,0,0'
1202	if dofile:
1203	try:
1204	with gen.Capturing() as output:
1205	ncvar.DataSetSection(vals,prevfile)
1206	except:
1207	print errmsg
1208	print 'DataSetSection('+vals+',', prevfile, ')'
1209	for s1out in output: print s1out
1210	quit(-1)
1211
1212	if db:
1213	for s1out in output: print s1out
1214
1215	sout = sub.call('mv ' + hprevfile + '_time_B-9-E0-I0.nc ' + fileon, \
1216	shell=True)
1217
1218	# Keeping the operations
1219	pyins=pyH + "/nc_var.py -o DataSetSection -S '" + vals + "' -f " + \
1220	prevfile
1221	otrackf.write("\n")
1222	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1223	otrackf.write(pyins + "\n")
1224
1225	elif op =='Lmean':
1226	# latitudinal mean values
1227	print " " + fname + ": kind '" + op + "' not ready !!"
1228	wrongstats.append(CFvarn + '_' + opers)
1229	break
1230	elif op =='Lsec':
1231	# latitudinal section (latitudinal value must be given, [var]@[lat])
1232	print " " + fname + ": kind '" + op + "' not ready !!"
1233	wrongstats.append(CFvarn + '_' + opers)
1234	break
1235	elif op =='lmean':
1236	# longitudinal mean values
1237	print " " + fname + ": kind '" + op + "' not ready !!"
1238	wrongstats.append(CFvarn + '_' + opers)
1239	break
1240	elif op =='lsec':
1241	# longitudinal section (longitudinal value must be given, [var]@[lon])
1242	print " " + fname + ": kind '" + op + "' not ready !!"
1243	wrongstats.append(CFvarn + '_' + opers)
1244	break
1245	elif op == 'pinterp':
1246	# pinterp: pressure interpolation (to the given $plevels)
1247	vals=plevels + ',1,1'
1248	if dofile:
1249	try:
1250	with gen.Capturing() as output:
1251	ncvar.pinterp(vals,prevfile,vninF)
1252	except:
1253	print errmsg
1254	print 'pinterp('+vals+',', prevfile, ','+vninF+')'
1255	for s1out in output: print s1out
1256	ncvar.pinterp(vals,prevfile,vninF)
1257	quit(-1)
1258
1259	if db:
1260	for s1out in output: print s1out
1261
1262	sout = sub.call('mv pinterp.nc ' + fileon, shell=True)
1263
1264	# Keeping the operations
1265	pyins=pyH + "/nc_var.py -o pinterp -S '" + vals + "' -f " + \
1266	prevfile + "-v " + vninF
1267	otrackf.write("\n")
1268	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1269	otrackf.write(pyins + "\n")
1270
1271	# adding CF lon,lat,time in WRF files
1272	if minf.name == 'WRF' or minf.name == 'WRF_LMDZ':
1273	values = vdnx + ':' + vdny + ':'+ Tref + ':' + Tunits
1274	if db:
1275	print " CFification of '" + fileon + "' with:", values, "..."
1276	try:
1277	with gen.Capturing() as output:
1278	ncvar.WRF_toCF(values, fileon)
1279	except:
1280	print errmsg
1281	print 'ncvar.WRF_toCF(' + values+ ', ' + fileon + ')'
1282	for s1out in output: print s1out
1283	# Removing file in order to make sure that it will be redone
1284	print ' ' + fname + " removing file '" + fileon + "' ..."
1285	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1286	quit(-1)
1287
1288	# vertical interpolation variables are no more needed
1289	CFvarnp = vninF
1290	gP = 'none'
1291
1292	elif op == 'smean':
1293	# spatial-weighted mean values
1294	noLlvdims = list(varnCFs)
1295	noLlvdims.remove('lon')
1296	noLlvdims.remove('lat')
1297	Saddvars = ':'.join(noLlvdims)
1298	if len(Saddvars) > 1:
1299	vals = 'reglonlat,lon,lat,lon,lat,' + Saddvars
1300	else:
1301	vals = 'reglonlat,lon,lat,lon,lat,None'
1302
1303	if dofile:
1304	try:
1305	with gen.Capturing() as output:
1306	ncvar.SpatialWeightedMean(vals,prevfile,vninF)
1307	except:
1308	print errmsg
1309	print 'SpatialWeightedMean('+vals+',', prevfile, ','+vninF+')'
1310	for s1out in output: print s1out
1311	quit(-1)
1312
1313	if db:
1314	for s1out in output: print s1out
1315
1316	# renaming variable
1317	ncvar.chvarname(vninF+'mean', 'SpatialWeightedMean_reglonlat.nc', \
1318	vninF+'spaceweightmean')
1319	sout = sub.call('mv SpatialWeightedMean_reglonlat.nc '+fileon, \
1320	shell=True)
1321
1322	# Keeping the operations
1323	pyins=pyH + "/nc_var.py -o SpatialWeightedMean -S '" + vals + \
1324	"' -f " + prevfile + " -v " + vninF
1325	otrackf.write("\n")
1326	otrackf.write("# " + vninF + " " + Fopers + "\n")
1327	otrackf.write(pyins + "\n")
1328
1329	# removing dimension variable-dimension 'lon', 'lat'
1330	varnCFs.remove('lon')
1331	varnCFs.remove('lat')
1332
1333	varkeep.append(vninF + 'mean')
1334
1335	elif op == 'tmean':
1336	# temporal mean values
1337	vals='time\|-1,time,mean,' + ':'.join(varnCFs) + ':' + vdnz
1338	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1339	if dofile:
1340	try:
1341	with gen.Capturing() as output:
1342	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1343	except:
1344	print errmsg
1345	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1346	for s1out in output: print s1out
1347	quit(-1)
1348
1349	if db:
1350	for s1out in output: print s1out
1351
1352	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
1353
1354	# Keeping the operations
1355	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1356	"' -f " + prevfile + " -v " + CFvarnp
1357	otrackf.write("\n")
1358	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1359	otrackf.write(pyins + "\n")
1360
1361	# removing dimension variable-dimension 'time'
1362	varnCFs.remove('time')
1363
1364	varkeep.append('timestats')
1365
1366	elif op == 'tstd':
1367	# temporal standard deviation values
1368	vals='time\|-1,time,std,' + ':'.join(varnCFs) + ':' + vdnz
1369	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1370	if dofile:
1371	try:
1372	with gen.Capturing() as output:
1373	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1374	except:
1375	print errmsg
1376	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1377	for s1out in output: print s1out
1378	quit(-1)
1379
1380	if db:
1381	for s1out in output: print s1out
1382
1383	sout = sub.call('mv file_oper_alongdims_std.nc '+fileon, shell=True)
1384
1385	# Keeping the operations
1386	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1387	"' -f " + prevfile + " -v " + CFvarnp
1388	otrackf.write("\n")
1389	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1390	otrackf.write(pyins + "\n")
1391
1392	# removing dimension variable-dimension 'time'
1393	varnCFs.remove('time')
1394
1395	varkeep.append('timestats')
1396
1397	elif op == 'tturb':
1398	# temporal turbulent values
1399	vals='time\|-1,time,turb,' + ':'.join(varnCFs) + ':' + vdnz
1400	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1401	if dofile:
1402	try:
1403	with gen.Capturing() as output:
1404	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1405	except:
1406	print errmsg
1407	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1408	for s1out in output: print s1out
1409	quit(-1)
1410
1411	if db:
1412	for s1out in output: print s1out
1413
1414	sout = sub.call('mv file_oper_alongdims_turb.nc '+fileon, shell=True)
1415
1416	# Keeping the operations
1417	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1418	"' -f " + prevfile + " -v " + CFvarnp
1419	otrackf.write("\n")
1420	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1421	otrackf.write(pyins + "\n")
1422
1423	# removing dimension variable-dimension 'time'
1424	varkeep.append('timestats')
1425
1426	elif op == 'tvar':
1427	# temporal variance values (Taylor's turbulence)
1428	vals='time\|-1,time,var,' + ':'.join(varnCFs) + ':' + vdnz
1429	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1430	if dofile:
1431	try:
1432	with gen.Capturing() as output:
1433	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1434	except:
1435	print errmsg
1436	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1437	for s1out in output: print s1out
1438	quit(-1)
1439
1440	if db:
1441	for s1out in output: print s1out
1442
1443	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
1444
1445	# Keeping the operations
1446	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1447	"' -f " + prevfile + " -v " + CFvarnp
1448	otrackf.write("\n")
1449	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1450	otrackf.write(pyins + "\n")
1451
1452	# removing dimension variable-dimension 'time'
1453	varnCFs.remove('time')
1454
1455	varkeep.append('timestats')
1456
1457	elif op == 'xmean':
1458	# x-axis mean values
1459	vals='lon\|-1,lon,mean,' + ':'.join(varnCFs) + ':' + vdnz
1460	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1461	if dofile:
1462	try:
1463	with gen.Capturing() as output:
1464	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1465	except:
1466	print errmsg
1467	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1468	for s1out in output: print s1out
1469	quit(-1)
1470
1471	if db:
1472	for s1out in output: print s1out
1473
1474	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
1475
1476	# Keeping the operations
1477	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1478	"' -f " + prevfile + " -v " + CFvarnp
1479	otrackf.write("\n")
1480	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1481	otrackf.write(pyins + "\n")
1482
1483	# removing dimension variable-dimension 'lon'
1484	varnCFs.remove('lon')
1485
1486	varkeep.append('lonstats')
1487
1488	elif op == 'xvar':
1489	# x-axis variance values
1490	vals='lon\|-1,lon,var,' + ':'.join(varnCFs) + ':' + vdnz
1491	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1492	if dofile:
1493	try:
1494	with gen.Capturing() as output:
1495	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1496	except:
1497	print errmsg
1498	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1499	for s1out in output: print s1out
1500	quit(-1)
1501
1502	if db:
1503	for s1out in output: print s1out
1504
1505	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
1506
1507	# Keeping the operations
1508	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1509	"' -f " + prevfile + " -v " + CFvarnp
1510	otrackf.write("\n")
1511	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1512	otrackf.write(pyins + "\n")
1513
1514	# removing dimension variable-dimension 'lon'
1515	varnCFs.remove('lon')
1516
1517	varkeep.append('lonstats')
1518
1519	elif op == 'ymean':
1520	# y-axis mean values
1521	vals='lat\|-1,lat,mean,' + ':'.join(varnCFs) + ':' + vdnz
1522	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1523	if dofile:
1524	try:
1525	with gen.Capturing() as output:
1526	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1527	except:
1528	print errmsg
1529	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1530	for s1out in output: print s1out
1531	quit(-1)
1532
1533	if db:
1534	for s1out in output: print s1out
1535
1536	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
1537
1538	# Keeping the operations
1539	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1540	"' -f " + prevfile + " -v " + CFvarnp
1541	otrackf.write("\n")
1542	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1543	otrackf.write(pyins + "\n")
1544
1545	# removing dimension variable-dimension 'lat'
1546	varnCFs.remove('lat')
1547
1548	varkeep.append('latstats')
1549
1550	elif op == 'zsum':
1551	# vertical aggregated values
1552	print " " + fname + ": kind '" + op + "' not ready !!"
1553	wrongstats.append(CFvarn + '_' + opers)
1554	break
1555	else:
1556	print errmsg
1557	print ' ' + fname + ": operation '" + op + "' not ready !!"
1558	quit(-1)
1559
1560	# End of kind of operation
1561
1562	# Variable name in file (vninF) changed due to operation
1563	# but only if previous operation does not the same 'statistic'
1564	chvn = gen.dictionary_key_list(Opsurs, op)
1565	if chvn is not None:
1566	oldvninF = vninF
1567	vninF = vninF + chvn
1568	CFvarnp = CFvarnp.replace(oldvninF,vninF)
1569
1570	if dofile:
1571	if len(varkeep) > 0:
1572	varkeepS = ',' + ','.join(varkeep)
1573	else:
1574	varkeepS = ''
1575
1576	# Adding such CF dimension variables which might not be in the
1577	# post-operation file
1578	try:
1579	with gen.Capturing() as output:
1580	varinfile = ncvar.ivars(fileon)
1581	except:
1582	print errmsg
1583	print 'ivars('+fileon+')'
1584	for s1out in output: print s1out
1585	# Removing file in order to make sure that it will be redone
1586	print ' ' + fname + " removing file '" + fileon + "' ..."
1587	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1588	quit(-1)
1589
1590	for CFvn in varnCFs:
1591	if not gen.searchInlist(varinfile, CFvn):
1592	if db:
1593	print ' ' + fname + ": recupering CF variable '" + CFvn + "'"
1594	try:
1595	with gen.Capturing() as output:
1596	ncvar.fvaradd(prevfile+','+CFvn, fileon)
1597	except:
1598	print errmsg
1599	print 'fvaradd(' + prevfile + ',' + CFvn +', '+fileon+')'
1600	ncvar.fvaradd(prevfile+','+CFvn, fileon)
1601	for s1out in output: print s1out
1602	# Removing file in order to make sure that it will be redone
1603	print ' ' + fname + " removing file '" + fileon + "' ..."
1604	#sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1605	quit(-1)
1606
1607	if db:
1608	for s1out in output: print s1out
1609
1610	totalvarkeeps = CFvarnp + ',' + ','.join(varnCFs) + varkeepS
1611	try:
1612	with gen.Capturing() as output:
1613	oclean = ncvar.cleaning_varsfile(totalvarkeeps,fileon)
1614	except:
1615	print errmsg
1616	print 'cleaning_varsfile('+totalvarkeeps+','+fileon+')'
1617	ncvar.cleaning_varsfile(totalvarkeeps,fileon)
1618	for s1out in output: print s1out
1619	# Removing file in order to make sure that it will be redone
1620	print ' ' + fname + " removing file '" + fileon + "' ..."
1621	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1622	quit(-1)
1623
1624	if db:
1625	for s1out in output: print s1out
1626
1627	# End of operations
1628	if db:
1629	print fname + ": successful calculation of '" + vninF + "' from '" + \
1630	CFvarn + "' '" + Opers + "' !!"
1631
1632	if len(wrongstats) > 1:
1633	print warnmsg
1634	print ' ' + fname + ": statisitcs not possible to compute:", wrongstats
1635
1636	if db: print " End of '" + fname + "' "
1637
1638	return
1639
1640	def compute_vars(config, modinf, idir, odir, Files, allvarcomp, fscr, debug):
1641	""" Function to compute the variables
1642	config= Configuration of the experiment
1643	modinf= class with information about the model
1644	idir= input experiment folder
1645	odir= output experiment folder
1646	Files= dictionary with the header and the files' correspondence
1647	allvarcomp= dictionary with all the variables to compute and their information
1648	fscr= whether files should be done from the scratch or not
1649	"""
1650	fname = 'compute_vars'
1651
1652	for vopn in allvarcomp.keys():
1653	# variable & operation
1654	vn = vopn.split('_')[0]
1655	op = vopn.split('_')[1]
1656	vopnV = allvarcomp[vopn]
1657	fheader = vopnV[0]
1658	if type(vopnV[1]) == type([1, 2]) and len(vopnV[1]) == 1:
1659	vals = vopnV[1]
1660	model = vals[0]
1661	else:
1662	model = vopnV[1]
1663	diag = vopnV[2]
1664	globalP = vopnV[3]
1665
1666	if debug:
1667	print ' ' + vn + ': ' + op
1668
1669	# Computing CF variable
1670	if model is not None or diag is not None:
1671	vinf = VariableInf(vn,fheader,model,diag)
1672	if op.find('pinterp') != -1:
1673	SP = 'p'
1674	else:
1675	SP = ''
1676
1677	# Comppute variable
1678	finalfilen = odir + '/' + vn + '_' + fheader + SP + '.nc'
1679	if fscr:
1680	sout = sub.call('rm ' + finalfilen + ' >& /dev/null', shell=True)
1681
1682	if not os.path.isfile(finalfilen):
1683	compute_variable(modinf, idir, Files, odir, vinf, globalP, fscr, \
1684	config['pyHOME'], config['CFreftime'], config['CFunitstime'], debug)
1685
1686	# Compute variable statistics
1687	finalfilen = odir + '/' + vn + '_' + fheader + SP + '_' + \
1688	op.replace('+','_') + '.nc'
1689	if fscr:
1690	sout = sub.call('rm ' + finalfilen + ' >& /dev/null', shell=True)
1691
1692	if not os.path.isfile(finalfilen):
1693	compute_statistics(modinf, cnf, iwdir, Files, owdir, vinf, globalP, \
1694	op, fscr, debug)
1695
1696	else:
1697	print errmsg
1698	print ' ' + fname + ": neither 'model' or 'diag' variables for '" + vn \
1699	+ "' !!"
1700	quit(-1)
1701
1702	return
1703
1704	def get_plots_var(vc,pltk,db):
1705	""" Function to provide the plots to make for each variable from pltk ('DIRPLT_', or 'DIFFPLT_') dictionary
1706	vc= pltk dictionary, dictionary with all the parameters started with pltk
1707	pltk= kind of plots. Values started by 'DIRPLT_', or 'DIFFPLT_' in model configuration
1708	"""
1709	fname = 'get_plots_var'
1710
1711	LH = len(pltk)
1712	# list of plots
1713	doplots = {}
1714	# plots by variable
1715	plotsvar = {}
1716	# individual plots by variable (not repeated)
1717	indivplotsvar = {}
1718
1719	# Variables with a plot which requires vertical interpolation
1720	varplotp = []
1721
1722	ipl = 0
1723	for plot in vc.keys():
1724	Lplot = len(plot)
1725	pln = plot[LH:Lplot+1]
1726	varops = vc[plot].split(':')
1727	for vn in varops:
1728	VOn = vn.split('#')
1729	# Variables and operations in plot
1730	vplt = []
1731	oplt = []
1732	voplt = []
1733	for vop in VOn:
1734	vv = vop.split('\|')[0]
1735	oo = vop.split('\|')[1]
1736	vplt.append(vv)
1737	oplt.append(oo)
1738	if not gen.searchInlist(voplt,vop): voplt.append(vop)
1739	vpltS = '#'.join(vplt)
1740	opltS = '#'.join(oplt)
1741
1742	if not doplots.has_key(pln):
1743	doplots[pln] = [vn]
1744	else:
1745	plots = doplots[pln]
1746	plots.append(vn)
1747	doplots[pln] = plots
1748
1749	if not plotsvar.has_key(vn):
1750	plotsvar[vn] = [pln]
1751	else:
1752	plots = plotsvar[vn]
1753	plots.append(pln)
1754	plotsvar[vn] = plots
1755
1756	for VOv in voplt:
1757	if not indivplotsvar.has_key(VOv):
1758	indivplotsvar[VOv] = [pln]
1759	else:
1760	plots = indivplotsvar[VOv]
1761	plots.append(pln)
1762	indivplotsvar[VOv] = plots
1763
1764	if vc[plot].find('pinterp') != -1:
1765	if not gen.searchInlist(varplotp,vn): varplotp.append(pln)
1766
1767	ipl = ipl + 1
1768
1769	if db:
1770	print ' plots to make _______'
1771	gen.printing_dictionary(doplots)
1772	print ' plots by variable\|operation _______'
1773	gen.printing_dictionary(plotsvar)
1774	print ' individual plots by variable\|operation _______'
1775	gen.printing_dictionary(indivplotsvar)
1776	print ' plots with the vertical interpolation _______'
1777	print ' #', varplotp
1778
1779	if db: print " End of '" + fname + "' "
1780
1781	return doplots, plotsvar, indivplotsvar, varplotp
1782
1783	def plots_listconstruct(config, pkind, finf, odir, debug):
1784	""" Function to create the list of plots to draw
1785	config= Configuration of the experiment
1786	pkind= kind of plots ('DIRPLT', 'DIFFPLT')
1787	finf= file with the instructions of the plots
1788	odir= output experiment folder
1789	"""
1790	fname='plots_listconstruct'
1791
1792	dirplot = gen.get_specdictionary_HMT(config, H=pkind+'_')
1793
1794	# No plots because there are not plots of this kind
1795	if dirplot is None:
1796	print warnmsg
1797	print ' ' + fname + ": no plots for type '" + pkind + "' !!"
1798	return {}, 0
1799
1800	if debug:
1801	print " '" + pkind + "' plots to draw ________"
1802	gen.printing_dictionary(dirplot)
1803
1804	# Getting plots by variable
1805	plots, varplot, indivarplot, plotp = get_plots_var(dirplot, pkind+'_', debug)
1806
1807	Svarplot = gen.dictKeysVals_stringList(plots,cV=':')
1808
1809	Nplots = 0
1810	for pl in Svarplot.split(','):
1811	Nplots = Nplots + len(pl.split(':'))
1812
1813	# Outwritting the varcompute to avoid next time (if it is not filescratch!)
1814	objf = open(finf, 'w')
1815	objf.write('plots: ' + Svarplot + '\n')
1816	objf.write('itotp: ' + str(Nplots) + '\n')
1817	objf.close()
1818
1819	if debug: print " End of '" + fname + "' "
1820
1821	return plots, Nplots
1822
1823	def read_plot_file(figfile):
1824	""" Function to read the file with the information about the plots to draw
1825	figfile= file with the information
1826	"""
1827	fname = 'read_plot_file'
1828
1829	if not os.path.isfile(figfile):
1830	print errormsg
1831	print ' ' + fname + ": figures file '" + figfile + "' does not exist !!"
1832	quit(-1)
1833
1834	objf = open(figfile, 'r')
1835	for line in objf:
1836	if line[0:1] != '#' and len(line) > 1:
1837	values = line.replace('\\n','').split(' ')
1838	if values[0] == 'plots:':
1839	if values[1] != 'none':
1840	plots = gen.stringList_dictKeysVals(values[1],cV=':')
1841	else:
1842	pots = {}
1843	elif values[0] == 'itotp:':
1844	Nplots = int(values[1])
1845
1846	objf.close()
1847
1848	return plots, Nplots
1849
1850	def varnoper(vn, oper, opsurdict):
1851	""" Function to provide name of the variable after operation as result of addition of the 'surnames'
1852	vn= variable name
1853	oper= '+' separated list of operations
1854	opsurdict= dictionary with the surname of operations
1855	>>> OpSurDict = {'mean': ['tmean', 'xmean', 'ymean']}
1856	>>> varnoper('tas', 'pinterp+tmean+xmean', OpSurDict)
1857	tasmeanmean
1858	"""
1859	fname = 'varnoper'
1860
1861	newvn = vn
1862
1863	for op in oper.split('+'):
1864	surname = gen.dictionary_key_list(opsurdict,op)
1865	if surname is not None:
1866	newvn = newvn + surname
1867
1868	return newvn
1869
1870	def draw_plot(kplot, CFvplot, fplot, vplot, dplot, pplot, finame, tfig, kfig, mapval,\
1871	tvals, expgraphc, od, pyH, fscr, db):
1872	""" Function to draw a plot
1873	kplot= kind of plot
1874	CFvplot= list of the CFnames of the variables
1875	fplot= list with the files in the plot
1876	vplot= list with the name of the variables in each file
1877	dplot= list of the dimensions in each file
1878	pplot= list of pictoric characteristics for each variable
1879	finame= name of the figure
1880	tfig= title of the figure
1881	kfig= kind of the figure
1882	mapval= value of the map
1883	tvals= list with time values
1884	tunits: units of time ('!' for spaces)
1885	timekind: kind of time ticks in the plot
1886	timefmt: format of time ticks in the plot
1887	timelabel: label of time-axis in the plot
1888	expgraphc= dictionary with expriment graphical characteristics
1889	od= output directory
1890	pyH= python HOME
1891	fscr= whether figure should be done from scratch
1892	"""
1893	fname = 'draw_plot'
1894
1895	if fscr:
1896	sout = sub.call('rm ' + finame + ' >& /dev/null', shell=True)
1897
1898	os.chdir(od)
1899
1900	# CF variable-dimensions
1901	CFvardims = {'lon':'lon', 'lat': 'lat', 'pres': 'pres', 'time': 'time'}
1902
1903	# Tracking file with all the figures
1904	trkobjf = open('all_figures.inf', 'a')
1905
1906	# Time-characteristics
1907	tunits = tvals[0]
1908	timekind = tvals[1]
1909	timefmt = tvals[2]
1910	timelabel = tvals[3]
1911
1912	# Graphical labels and configuration
1913
1914
1915	if not os.path.isfile(finame):
1916	if db:
1917	print " drawing '" + finame + "' ..."
1918	print ' plot kind:', kplot
1919	print ' CFvars:', CFvplot
1920	print ' files:', fplot
1921	print ' in file vars:', vplot
1922	print ' dims:', dplot
1923	print ' pictoric values:', pplot
1924	print ' figure name:', finame
1925	print ' title:', tfig.replace('!',' ')
1926	print ' kind:', kfig
1927	print ' map:', mapval
1928	print ' time units:', tunits
1929
1930	if kplot == 'diffmap2Dsfc':
1931	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1932	quit(-1)
1933	elif kplot == 'diffmap2Dz':
1934	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1935	quit(-1)
1936	elif kplot == 'map2Dsfc':
1937	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1938	quit(-1)
1939	elif kplot == 'map3D':
1940	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1941	quit(-1)
1942	elif kplot == '2lines':
1943	figtit = tfig.replace('!','\|')
1944
1945	lineAstdn = CFvplot[0]
1946	lineBstdn = CFvplot[1]
1947	figfs = ','.join(fplot)
1948
1949	lineA = pplot[0]
1950	Arange = str(lineA[0]) + ',' + str(lineA[1])
1951	# Values are changed from specific values in `model_graphics.dat'
1952	if lineA[3].find('%') == -1:
1953	Aline = lineA[2]
1954	Akind = lineA[3]
1955	Amark = lineA[4]
1956	Asize = lineA[5]
1957	else:
1958	Aline = 'red'
1959	Akind = '-'
1960	Amark = ','
1961	Asize = 2.
1962
1963	lineB = pplot[1]
1964	Brange = str(lineB[0]) + ',' + str(lineB[1])
1965	# Values are changed from specific values in `model_graphics.dat'
1966	if lineB[3].find('%') == -1:
1967	Bline = lineB[2]
1968	Bkind = lineB[3]
1969	Bmark = lineB[4]
1970	Bsize = lineB[5]
1971	else:
1972	Bline = 'blue'
1973	Bkind = '-'
1974	Bmark = ','
1975	Bsize = '2.'
1976
1977	# It is assumed that if the space variable is 'lon': is desired a
1978	# (lon, vals) plot if it is 'lat': then (vals, lat) plot
1979	if gen.searchInlist(dplot[0],'lon'):
1980	spacedim ='lon'
1981	axisvals = 'y'
1982	figmid = 'longitudinal'
1983	elif gen.searchInlist(dplot[0],'lat'):
1984	spacedim = 'lat'
1985	axisvals = 'x'
1986	figmid = 'meridional'
1987	elif gen.searchInlist(dplot[0],'pres'):
1988	spacedim = 'pres'
1989	axisvals = 'x'
1990	figmid = 'vertical'
1991	else:
1992	print errmsg
1993	print ' ' + fname + ": in '2lines' only ready for: 'lon', 'lat'," + \
1994	" 'pres' as common dimension !!"
1995	print ' dimensions in plot:', dplot[0]
1996	quit(-1)
1997
1998	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
1999	'!'.join(tfig.split('!')[2:])
2000
2001	graphvals = spacedim + ':' + Arange + ':' + Brange + ':Extrs:' + \
2002	axisvals + ':' + ','.join(CFvplot) + ':' + Aline + ',' + Bline + \
2003	':2.,2.:' + Akind + ',' + Bkind + ':2.,2.:,:' + figtit + ':' + \
2004	spacedim + ':0\|auto:' + finame.replace('.'+kfig, '') + ':' + kfig + \
2005	':True'
2006
2007	fvarS = ','.join(vplot)
2008
2009	drwins = 'draw_2lines'
2010	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2011	" -S '" + graphvals + "' -v " + fvarS
2012
2013	try:
2014	with gen.Capturing() as output:
2015	sout = sub.call(plotins, shell=True)
2016	except:
2017	print errmsg
2018	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2019	print sout
2020	for s1out in output: print s1out
2021	quit(-1)
2022
2023	# keeping all figures
2024	trkobjf.write('\n')
2025	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2026	trkobjf.write(plotins + '\n')
2027
2028	elif kplot == '2linesTime':
2029	figtit = tfig.replace('!','\|')
2030
2031	lineAstdn = CFvplot[0]
2032	lineBstdn = CFvplot[1]
2033	figfs = ','.join(fplot)
2034
2035	lineA = pplot[0]
2036	Arange = str(lineA[0]) + ',' + str(lineA[1])
2037	# Values are changed from specific values in `model_graphics.dat'
2038	if lineA[3].find('%') == -1:
2039	Aline = lineA[2]
2040	Akind = lineA[3]
2041	Amark = lineA[4]
2042	Asize = lineA[5]
2043	else:
2044	Aline = 'red'
2045	Akind = '-'
2046	Amark = ','
2047	Asize = '2.'
2048
2049	lineB = pplot[1]
2050	Brange = str(lineB[0]) + ',' + str(lineB[1])
2051	# Values are changed from specific values in `model_graphics.dat'
2052	if lineB[3].find('%') == -1:
2053	Bline = lineB[2]
2054	Bkind = lineB[3]
2055	Bmark = lineB[4]
2056	Bsize = lineB[5]
2057	else:
2058	Bline = 'blue'
2059	Bkind = '-'
2060	Bmark = ','
2061	Bsize = '2.'
2062
2063	timeaxis = 'x'
2064	figmid = 'evolution'
2065
2066	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
2067	'!'.join(tfig.split('!')[2:])
2068
2069	graphvals = 'time:' + Arange + ':' + Brange + ':' + timekind + ';' + \
2070	timefmt + ':' + timeaxis + ':' + ','.join(CFvplot) + ':' + Aline + ',' \
2071	+ Bline + ':2.,2.:' + Akind + ',' + Bkind + ':2.,2.:,:' + figtit + ':' \
2072	+ timelabel + ':0\|auto:' + finame.replace('.'+kfig, '') + ':' + \
2073	kfig + ':True'
2074
2075	fvarS = ','.join(vplot)
2076
2077	drwins = 'draw_2lines_time'
2078	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2079	" -S '" + graphvals + "' -v " + fvarS
2080
2081	try:
2082	with gen.Capturing() as output:
2083	sout = sub.call(plotins, shell=True)
2084	except:
2085	print errmsg
2086	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2087	print sout
2088	for s1out in output: print s1out
2089	quit(-1)
2090
2091	# keeping all figures
2092	trkobjf.write('\n')
2093	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2094	trkobjf.write(plotins + '\n')
2095
2096	elif kplot == 'Nlines':
2097	figtit = tfig.replace('!','\|')
2098
2099	linestdn = CFvplot[0]
2100	figfs = ','.join(fplot)
2101
2102	Nlines = len(fplot)
2103
2104	lablines = []
2105	for il in range(Nlines):
2106	linev = pplot[il]
2107	# Values are changed from specific values in `model_graphics.dat'
2108	if linev[3].find('%') == -1:
2109	line = linev[2]
2110	kind = linev[3]
2111	mark = linev[4]
2112	size = linev[5]
2113	if il == 0:
2114	vrange = str(linev[0]) + ',' + str(linev[1])
2115	kinds = kind
2116	lines = line
2117	marks = mark
2118	sizes = str(size)
2119	else:
2120	kinds = kinds + ',' + kind
2121	lines = lines + ',' + line
2122	marks = marks + '@' + mark
2123	sizes = sizes + ',' + str(size)
2124	else:
2125	lines = 'None'
2126	kinds = '-'
2127	marks = ','
2128	sizes = '2.'
2129
2130	secsf = fplot[il].split('/')
2131	Nsecsf = len(secsf)
2132	expvs = expgraphc[secsf[Nsecsf-3] + '/' + secsf[Nsecsf-2]]
2133	lablines.append(expvs.label)
2134
2135	# It is assumed that if the space variable is 'lon': is desired a
2136	# (lon, vals) plot if it is 'lat': then (vals, lat) plot
2137	if gen.searchInlist(dplot[0],'lon'):
2138	spacedim ='lon'
2139	sdlab ='lon ($degrees\ East$)'
2140	axisvals = 'y'
2141	figmid = 'longitudinal'
2142	elif gen.searchInlist(dplot[0],'lat'):
2143	spacedim = 'lat'
2144	sdlab = 'lat ($degrees\ North$)'
2145	axisvals = 'x'
2146	figmid = 'meridional'
2147	elif gen.searchInlist(dplot[0],'pres'):
2148	spacedim = 'pres'
2149	sdlab = 'pres ($Pa$)'
2150	axisvals = 'x'
2151	figmid = 'vertical'
2152	else:
2153	print errmsg
2154	print ' ' + fname + ": in 'Nlines' only ready for: 'lon', 'lat'," + \
2155	" 'pres' as common dimension !!"
2156	print ' dimensions in plot:', dplot[0]
2157	quit(-1)
2158
2159	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
2160	'!'.join(tfig.split('!')[2:])
2161	figtit = figtit.replace('!',' ')
2162	leglabels = ','.join(lablines)
2163
2164	graphvals = spacedim + ':' + axisvals + ':' + sdlab + ':auto:' + \
2165	leglabels + ':' + CFvplot[0] +':'+ figtit + ':0\|auto:' + lines + ':' + \
2166	kinds + ':' + marks +':'+ sizes +':'+ sizes +':all:'+ \
2167	finame.replace('.'+kfig, '') + ':' + kfig + ':True'
2168
2169	fvarS = vplot[0]
2170
2171	drwins = 'draw_lines'
2172	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2173	" -S '" + graphvals + "' -v " + fvarS
2174
2175	try:
2176	with gen.Capturing() as output:
2177	sout = sub.call(plotins, shell=True)
2178	except:
2179	print errmsg
2180	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2181	print sout
2182	for s1out in output: print s1out
2183	quit(-1)
2184
2185	# keeping all figures
2186	trkobjf.write('\n')
2187	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2188	trkobjf.write(plotins + '\n')
2189
2190	elif kplot == 'Nlines_time':
2191	figtit = tfig.replace('!','\|')
2192
2193	linestdn = CFvplot[0]
2194	figfs = ','.join(fplot)
2195
2196	Nlines = len(fplot)
2197
2198	lablines = []
2199	for il in range(Nlines):
2200	linev = pplot[il]
2201	# Values are changed from specific values in `model_graphics.dat'
2202	if linev[3].find('%') == -1:
2203	line = linev[2]
2204	kind = linev[3]
2205	mark = linev[4]
2206	size = linev[5]
2207	if il == 0:
2208	vrange = str(linev[0]) + ',' + str(linev[1])
2209	kinds = kind
2210	lines = line
2211	marks = mark
2212	sizes = str(size)
2213	else:
2214	kinds = kinds + ',' + kind
2215	lines = lines + ',' + line
2216	marks = marks + '@' + mark
2217	sizes = sizes + ',' + str(size)
2218	else:
2219	lines = 'None'
2220	kinds = '-'
2221	marks = ','
2222	sizes = '2.'
2223
2224	secsf = fplot[il].split('/')
2225	Nsecsf = len(secsf)
2226	expvs = expgraphc[secsf[Nsecsf-3] + '/' + secsf[Nsecsf-2]]
2227	lablines.append(expvs.label)
2228
2229	valsaxis = 'y'
2230	figmid = 'evolution'
2231
2232	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
2233	'!'.join(tfig.split('!')[2:])
2234	figtit = figtit.replace('!',' ')
2235	leglabels = ','.join(lablines)
2236
2237	graphvals = 'time;' + valsaxis + ';time;' + leglabels + ';' + \
2238	CFvplot[0] + ';' + figtit + ';None;time\|' + '\|'.join(tvals) + \
2239	';0\|auto;' + kfig + ';' + kinds + ';' + lines + ';' + marks + ';' + \
2240	sizes + ';' + sizes + ';all;-1;True'
2241
2242	fvarS = vplot[0]
2243
2244	drwins = 'draw_lines_time'
2245	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2246	" -S '" + graphvals + "' -v " + fvarS
2247
2248	try:
2249	with gen.Capturing() as output:
2250	sout = sub.call(plotins, shell=True)
2251	except:
2252	print errmsg
2253	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2254	print sout
2255	for s1out in output: print s1out
2256	quit(-1)
2257
2258	sout = sub.call('mv lines_time.' + kfig + ' ' + finame, shell=True)
2259
2260	# keeping all figures
2261	trkobjf.write('\n')
2262	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2263	trkobjf.write(plotins + '\n')
2264
2265	elif kplot == 'shadcont2Dsfc':
2266	figtit = tfig.replace('!','\|')
2267	shdstdn = CFvplot[0]
2268	cntstdn = CFvplot[1]
2269	figfs = ','.join(fplot)
2270
2271	shad = pplot[0]
2272	srange = str(shad[0]) + ',' + str(shad[1])
2273	cbar = shad[2]
2274	cnt = pplot[1]
2275	crange = str(cnt[0]) + ',' + str(cnt[1])
2276	cfmt = cnt[3]
2277	ckind = cnt[4]
2278	cline = cnt[5]
2279
2280	graphvals = ','.join(CFvplot)
2281	graphvals = graphvals + ':lon\|-1,lat\|-1:lon\|-1,lat\|-1:lon:lat:auto:' + \
2282	cbar + ',auto,auto:' + ckind + ',' + cline + ':' + cfmt + ':' + \
2283	srange + ':' + crange + ',9:' + figtit + ':' + kfig + ':None:' + \
2284	mapval + ':True'
2285
2286	fvarS = ','.join(vplot)
2287
2288	drwins = 'draw_2D_shad_cont'
2289	plotins = "python " + pyH + "/drawing.py -f " + figfs + " -o " + drwins +\
2290	" -S '" + graphvals + "' -v " + fvarS
2291	try:
2292	with gen.Capturing() as output:
2293	sout = sub.call(plotins, shell=True)
2294	except:
2295	print errmsg
2296	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2297	print sout
2298	for s1out in output: print s1out
2299	quit(-1)
2300
2301	sout = sub.call('mv 2Dfields_shadow-contour.'+kfig+' '+finame, shell=True)
2302
2303	# keeping all figures
2304	trkobjf.write('\n')
2305	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2306	trkobjf.write(plotins + '\n')
2307
2308	elif kplot == 'shadconthovmsfc':
2309	figtit = tfig.replace('!','\|')
2310	shdstdn = CFvplot[0]
2311	cntstdn = CFvplot[1]
2312	figfs = ','.join(fplot)
2313
2314	shad = pplot[0]
2315	srange = str(shad[0]) + ',' + str(shad[1])
2316	cbar = shad[2]
2317	cnt = pplot[1]
2318	crange = str(cnt[0]) + ',' + str(cnt[1])
2319	cfmt = cnt[3]
2320	ckind = cnt[4]
2321	cline = cnt[5]
2322	# It is assumed that if the space variable is 'lon': is desired a
2323	# (lon, time) plot if it is 'lat': then (time, lat) plot
2324	if gen.searchInlist(dplot,'lon'):
2325	spacedim ='lon'
2326	figmid = 'longitudinal\|evolution\|of'
2327	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2328	'\|'.join(tfig.split('!')[2:])
2329	reverse= 'None'
2330	dims= spacedim+'\|-1,time\|-1;'+spacedim+'\|-1,time\|-1;'+spacedim+';time'
2331	else:
2332	spacedim='lat'
2333	figmid = 'meridional\|evolution\|of'
2334	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2335	'\|'.join(tfig.split('!')[2:])
2336	reverse='None'
2337	dims= spacedim+'\|-1,time\|-1;'+spacedim+'\|-1,time\|-1;time;'+spacedim
2338
2339	graphvals = ','.join(CFvplot)
2340	graphvals = graphvals + ';' + dims +';auto;' + cbar + ',auto,auto' + ';' \
2341	+ ckind + ',' + cline + ';' + cfmt + ';' + srange + ';' + crange + \
2342	',9;' + figtit + ';' + kfig + ';' + reverse + ';' + 'time\|' + tunits + \
2343	'\|'+ timekind + '\|' + timefmt + '\|' + timelabel + ';True'
2344
2345	fvarS = ','.join(vplot)
2346
2347	drwins = 'draw_2D_shad_cont_time'
2348	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2349	" -S '" + graphvals + "' -v " + fvarS
2350
2351	try:
2352	with gen.Capturing() as output:
2353	sout = sub.call(plotins, shell=True)
2354	except:
2355	print errmsg
2356	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2357	print sout
2358	for s1out in output: print s1out
2359	quit(-1)
2360
2361	sout = sub.call('mv 2Dfields_shadow-contour.'+kfig+' '+finame, shell=True)
2362
2363	# keeping all figures
2364	trkobjf.write('\n')
2365	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2366	trkobjf.write(plotins + '\n')
2367
2368	elif kplot == 'shadcont2Dzsec':
2369	figtit = tfig.replace('!','\|')
2370	shdstdn = CFvplot[0]
2371	cntstdn = CFvplot[1]
2372	figfs = ','.join(fplot)
2373
2374	shad = pplot[0]
2375	srange = str(shad[0]) + ',' + str(shad[1])
2376	cbar = shad[2]
2377	cnt = pplot[1]
2378	crange = str(cnt[0]) + ',' + str(cnt[1])
2379	cfmt = cnt[3]
2380	ckind = cnt[4]
2381	cline = cnt[5]
2382
2383	# It is assumed that if the space variable is 'lon': is desired a
2384	# (lon, pres) plot it it is 'lat': then (pres, lat) plot
2385	if gen.searchInlist(dplot, 'lon'):
2386	spacedim='lon'
2387	figmid = 'long.\|vert.\|cross\|sec.\|of'
2388	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2389	'\|'.join(tfig.split('!')[2:])
2390	dims= spacedim+'\|-1,pres\|-1:'+spacedim+'\|-1,pres\|-1:'+spacedim+':pres'
2391	else:
2392	spacedim='lat'
2393	figmid = 'mer.\|vert.\|cross\|sec.\|of'
2394	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2395	'\|'.join(tfig.split('!')[2:])
2396	# NOT WORKING? dims= spacedim+'\|-1,pres\|-1:'+spacedim+'\|-1,pres\|-1:'+'pres:'+spacedim
2397	dims= spacedim+'\|-1,pres\|-1:'+spacedim+'\|-1,pres\|-1:'+spacedim+':pres'
2398
2399	reverse = 'None'
2400
2401	graphvals = ','.join(CFvplot)
2402	graphvals = graphvals + ':' + dims + ':auto:' + cbar + ',auto,auto:' + \
2403	ckind + ',' + cline + ':' + cfmt + ':' + srange + ':' + crange + ',9:' \
2404	+ figtit + ':' + kfig + ':' + reverse + ':None:True'
2405
2406	fvarS = ','.join(vplot)
2407
2408	drwins = 'draw_2D_shad_cont'
2409	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs + ' -o ' + drwins + \
2410	" -S '" + graphvals + "' -v " + fvarS
2411	try:
2412	with gen.Capturing() as output:
2413	sout0 = sub.call(plotins, shell=True)
2414	except:
2415	print errmsg
2416	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2417	for s1out in output: print s1out
2418	quit(-1)
2419
2420	sout = sub.call('mv 2Dfields_shadow-contour.'+kfig+' '+finame, shell=True)
2421
2422	# keeping all figures
2423	trkobjf.write('\n')
2424	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2425	trkobjf.write(plotins + '\n')
2426
2427	else:
2428	print errmsg
2429	print " " + fname + ": plot kind '" + kplot + "' not ready !!"
2430	quit(-1)
2431
2432	trkobjf.close()
2433
2434	if db: print " End of '" + fname + "' "
2435
2436	return
2437
2438	def opexplained(Opn,Exp):
2439	""" Function to provide the final explanation for the title in the graph. Some operations have
2440	values separated by `~' which need to be processed
2441	Opn= full operation string
2442	Exp= explanation from dictionary
2443	>>> opexplained('dimvarvalue~pres~85000','@')
2444	pres@85000
2445	>>> opexplained('xmean','xmean')
2446	xmean
2447	"""
2448	fname = 'opexplained'
2449
2450	if Opn.find('~') == -1:
2451	explanation = Exp
2452	else:
2453	Opnv = Opn.split('~')
2454	if Opnv[0] == 'dimvarvalue':
2455	explanation = Opnv[1] + Exp + Opnv[2]
2456	else:
2457	print errormsg
2458	print ' ' + fname + ": unknow '~' separated operation '" + Opnv[0] + \
2459	"' !!"
2460	quit(-1)
2461	return explanation
2462
2463	def optitle(op,opexp):
2464	""" Function to creat the operation section in the title of a figure ('!' for spaces)
2465	op= '+' separated list of operations
2466	opexp= dictionary with the `explanation'(text in title) to appear for each operation
2467	>>> optitle('pinterp+xmean+tmean',{'pinterp':'pinterp', 'xmean':'xmean', 'tmean':'tmean'})
2468	$_{[pinterp\!xmean\!tmean]}$
2469	>>> optitle('pinterp+dimvarvalue~pres~85000+tmean',{'dimvarvalue':'@', 'pinterp':'pinterp', 'xmean':'xmean', 'tmean':'tmean'})
2470	$_{[pinterp\!pres@85000\!tmean]}$
2471	"""
2472	fname = 'optitle'
2473
2474	opvals = op.split('+')
2475	for op1 in opvals:
2476	op1n = op1.split('~')[0]
2477	if not opexp.has_key(op1n):
2478	print errmsg
2479	print ' ' + fname + ": no explanation for operation '" + op1n + "' !!"
2480	print ' provided:', opexp.keys()
2481	quit(-1)
2482	# Final explanation
2483	op1en = opexplained(op1,opexp[op1n]).replace(' ','!')
2484
2485	if op1n == opvals[0].split('~')[0]:
2486	titop = '$_{[' + op1en
2487	if len(opvals) == 1: titop = titop + ']}$'
2488	elif op1n == opvals[len(opvals)-1].split('~')[0]:
2489	titop = titop + '\!' + op1en + ']}$'
2490	else:
2491	titop = titop + '\!' + op1en
2492
2493	return titop
2494
2495	def create_figure_title(mod,exp,varops,explop):
2496	""" Function to create the title of a figure ('!' for spaces)
2497	mod: model name
2498	exp: experiment name
2499	varops: list with [var]\|[op] values from wich create the title
2500	explop: dictionary with the `explanation'(text in title) to appear for each operation
2501	>>> expops = {'pinterp':'pinterp', 'xmean':'xmean', 'tmean':'tmean', 'last':'last'}
2502	>>> create_figure_title('WRF','current',['ua\|pinterp+xmean+tmean', 'va\|pinterp+xmean+tmean'], expops)
2503	WRF!current!ua!&!va$_{[pinterp\!xmean\!tmean]}$
2504	>>> create_figure_title('WRF','current',['ua\|pinterp+xmean+tmean', 'ua\|pinterp+xmean+last'], expops)
2505	WRF!current!ua$_{[pinterp\!xmean\!tmean]}$!&!!$_{[pinterp\!xmean\!last]}$
2506	>>> create_figure_title('WRF','current',['ua\|pinterp+xmean+tmean', 'va\|pinterp+xmean+last'], expops)
2507	WRF!current!ua$_{[pinterp\!xmean\!tmean]}$!&!va$_{[pinterp\!xmean\!last]}$
2508	>>> create_figure_title('WRF','current',['uas\|dimvarvalue~lon~23.23', 'vas\|dimvarvalue~lon~23.23'], expops)
2509	WRF!current!uas!&!vas$_{[lon@23.23]}$
2510	>>> create_figure_title('WRF','current',['hus\|pinterp+tmean+dimvarvalue~pres~85000.', 'ta\|pinterp+tmean+dimvarvalue~pres~85000.'], expops)
2511	WRF!current!hus!&!ta$_{[pinterp\!tmean\!pres@85000.]}$
2512	"""
2513	fname = 'create_figure_title'
2514
2515	titfig = mod + '!' + exp + '!'
2516
2517	varns = []
2518	opns = []
2519	for varop in varops:
2520	vn = varop.split('\|')[0]
2521	op = varop.split('\|')[1]
2522	varns.append(vn)
2523	opns.append(op)
2524	if varop == varops[0]:
2525	allvarequal = True
2526	allopequal = True
2527	varinfig = vn
2528	opinfig = op
2529	else:
2530	if vn != varinfig: allvarequal = False
2531	if op != opinfig: allopequal = False
2532
2533	Nvars = len(varns)
2534	Nops = len(opns)
2535
2536	if allvarequal:
2537	titfig = titfig + '!' + varns[0]
2538	if allopequal:
2539	opS = optitle(op,explop)
2540	titfig = titfig + opS
2541	else:
2542	for opn in opns:
2543	opS = optitle(opn,explop)
2544	if opn == opns[0]:
2545	titfig = titfig + opS
2546	elif opn == opns[Nops-1]:
2547	titfig = titfig + '!&!' + opS
2548	else:
2549	titfig = titfig + ',!' + opS
2550	else:
2551	if allopequal:
2552	opS = optitle(op,explop)
2553	for vn in varns:
2554	if vn == varns[0]:
2555	titfig = titfig + '!' + vn
2556	elif vn == varns[Nvars-1]:
2557	titfig = titfig + '!&!' + vn
2558	else:
2559	titfig = titfig + ',!' + vn
2560	titfig = titfig + opS
2561	else:
2562	for ivop in range(Nvars):
2563	vn = varns[ivop]
2564	op = opns[ivop]
2565	vnop = vn + optitle(op,explop)
2566
2567	if vn == varns[0]:
2568	titfig = titfig + '!' + vnop
2569	elif vn == varns[Nvars-1]:
2570	titfig = titfig + '!&!' + vnop
2571	else:
2572	titfig = titfig + ',!' + vnop
2573
2574	return titfig.replace('!!','!')
2575	#expops = {'dimvarvalue':'@', 'last':'last', 'pinterp':'pinterp', 'xmean':'xmean', 'tmean':'tmean'}
2576
2577	def draw_plots(config, plots, mod, exp, odir, allvarcomp, figscr, debug):
2578	""" Function to draw all plots
2579	config= Configuration of the experiment
2580	plots= dictionary with the plots
2581	mod= model of the plots
2582	exp= experiment of the plots
2583	odir= output experiment folder
2584	allvarcomp= dictionary with all the variables to compute and their information
2585	figscr= whether figures should be done from the scratch or not
2586
2587	* Plot as
2588	{[kplot]} = [varn1]\|[op1]#[varn2]\|[op2]#[...[varnN]\|[opN]], ...
2589	[kplot] ___
2590	diffmap2Dsfc: 2D map of surface differences values of 1 variable
2591	diffmap2Dz: 2D map of 3D differences values of 1 variable
2592	map2Dsfc: 2D map of surface values of 1 variable
2593	map3D: 2D map of 3D values of 1 variable
2594	shadconthovmsfc: Hovmoeller diagrams of 2 variable at the surface in shadow and the other in contourn
2595	shadcont2Dsfc: 2D map of shadow (1st variable) and countour (2nd variable) [stvar1]#[stvar2]
2596	shadcont2Dzsec: 2D map of vertical section of 2 variables one in shadow and the other in contourn
2597	[varn]
2598	variable
2599	[op]
2600	'+' separated list of operations
2601	in figures with more than 1 variable, use '#' to separate the [varn]\|[op]
2602	"""
2603	fname = 'draw_plots'
2604
2605	os.chdir(odir)
2606
2607	# Dictionary with the operations with surnames for the operated variable
2608	opersurnames = {}
2609	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
2610	for opsr in opsur.keys():
2611	opn = opsr.split('_')[1]
2612	vls = opsur[opsr].split(':')
2613	opersurnames[opn] = vls
2614
2615	# time values
2616	# Units time for the plots
2617	rd = config['CFreftime']
2618	tunits = config['CFunitstime'] + '!since!' + rd[0:4] + '-' + rd[4:6] + '-' + \
2619	rd[6:8] + '!' + rd[8:10] + ':' + rd[10:12] + ':' + rd[12:14]
2620	# time ticks kind
2621	tkind = config['timekind']
2622	# time ticks format
2623	tfmt = config['timefmt']
2624	# time axis label
2625	tlab = config['timelabel']
2626	timevals = [tunits, tkind, tfmt, tlab]
2627
2628	# Dictionary of plot specificities
2629	specplotkeyn = 'specificvarplot'
2630	plotspecifics = {}
2631	if config.has_key(specplotkeyn):
2632	# [minval]: minimum value
2633	# [maxval]: minimum value
2634	# [colorbar]: name of the colorbar (from matplotlib) to use
2635	# [cntformat]: format of the contour labels
2636	# [colorcnt]: color for the countor lines
2637	plotspecs = config[specplotkeyn].split(':')
2638	for pltspc in plotspecs:
2639	pltvls = pltspc.split('\|')
2640	vn = pltvls[0]
2641	op = pltvls[1]
2642	fn = pltvls[2]
2643	plotspecifics[fn + '_' + vn + '_' + op] = pltvls[3:]
2644	if debug:
2645	print 'Specific values for plots _______'
2646	gen.printing_dictionary(plotspecifics)
2647
2648	# Kind of figures
2649	kindfigure = config['kindfig']
2650
2651	# Graphical labels and configuration
2652	modgc, expgc = graphical_characteristics(config, debug)
2653	modv = modgc[mod]
2654	modlab = modv.label
2655	expv = expgc[mod+'/'+exp]
2656	explab = expv.label
2657
2658	# Map value
2659	mapvalue = config['mapval']
2660
2661	# pythone scripts HOME
2662	pyHOME = config['pyHOME']
2663
2664	# Title-text of operations
2665	opexplained = {}
2666	optits = config['titleoperations'].split(':')
2667	for optit in optits:
2668	opn = optit.split('\|')[0]
2669	opt = optit.split('\|')[1]
2670	opexplained[opn] = opt
2671	if debug:
2672	print 'Titles for operations _______'
2673	gen.printing_dictionary(opexplained)
2674
2675	for kplot in plots.keys():
2676	varsplt = plots[kplot]
2677	for varplt in varsplt:
2678	if debug:
2679	print " printing '" + kplot + "' var ':" + varplt + "'..."
2680	varops = varplt.split('#')
2681
2682	# CF variables in plot
2683	CFvarsplot = []
2684	# Files in plot
2685	filesplot = []
2686	# Variables in plot within the files
2687	varsplot = []
2688	# Dims in figure
2689	dimsplot = []
2690	# pictoric values in figure
2691	pictplot = []
2692	# Name of the figure
2693	figname = ''
2694	# Title of the figure
2695	titfigure = ''
2696
2697	ivp = 0
2698	for varop in varops:
2699	vn = varop.split('\|')[0]
2700	op = varop.split('\|')[1]
2701
2702	# CF variables in plot
2703	CFvarsplot.append(vn)
2704
2705	vnopS = vn + '_' + op
2706	if not allvarcomp.has_key(vnopS):
2707	print errmsg
2708	print ' ' + fname + ": no file for variable-operation '" + \
2709	vnopS + "' !!"
2710	vopvals = allvarcomp[vnopS]
2711	headf = vopvals[0]
2712	globalP = vopvals[3]
2713	gP = pinterpS(globalP)
2714
2715	filen = vn + '_' + headf + gP + '_' + op.replace('+','_') + '.nc'
2716	filesplot.append(filen)
2717	# Do we have processed the given variable?
2718	if not os.path.isfile(filen):
2719	print warnmsg
2720	print " " + fname + ": there is no file for variable '" + varop \
2721	+ "' skiping it !!"
2722	break
2723
2724	# Name of the variable inside the file
2725	vnsur = varnoper(vn, op, opersurnames)
2726	varsplot.append(vnsur)
2727
2728	# Dimensions in file
2729	try:
2730	with gen.Capturing() as output:
2731	dims = ncvar.idims(filen)
2732	except:
2733	print errmsg
2734	print 'ncvar.idims('+filen+')'
2735	for s1out in output: print s1out
2736	quit(-1)
2737
2738	dimsplot.append(dims)
2739
2740	# pictoric values for the figure
2741	Sfivaop = kplot + '_' + vn + '_' + op
2742	if plotspecifics.has_key(Sfivaop):
2743	pictvals = plotspecifics[Sfivaop]
2744	else:
2745	Vvals = gen.variables_values(vn)
2746	pictvals = [Vvals[2], Vvals[3], Vvals[6], '%g', 'fixc', 'black']
2747
2748	pictplot.append(pictvals)
2749
2750	# Header of the name of the figure
2751	if ivp == 0:
2752	figname = kplot +'_'+ vn + '_' + headf + '_' + op.replace('+','-')
2753	else:
2754	figname = figname + '-'+vn+'_'+headf+'_'+op.replace('+','-')
2755
2756	ivp = ivp + 1
2757	# End of variable-operation
2758	figname = figname + '.' + kindfigure
2759
2760	# Removig double '..' (from `dimvarvalue')
2761	figname = figname.replace('..', '.')
2762
2763	# Title of figure
2764	titfigure = create_figure_title(modlab, explab, varops, opexplained)
2765
2766	draw_plot(kplot, CFvarsplot, filesplot, varsplot, dimsplot, pictplot, \
2767	figname, titfigure, kindfigure, mapvalue, timevals, expgc, odir, \
2768	pyHOME, figscr, debug)
2769
2770	# End of variables-operations
2771
2772	# End of kind of plots
2773
2774	if debug: print " End of '" + fname + "' "
2775
2776	return
2777
2778	def get_differences_var(vc,kdiff,db):
2779	""" Function to provide the operations to make for each variable from kdiff ('DIFFOP_' or 'DIFFVAR_') dictionary
2780	vc= kdiff dictionary, dictionary with all the parameters started with kdiff
2781	kdiff= kind of differences 'op', 'var'
2782	"""
2783	fname = 'get_differences_var'
2784
2785	if kdiff == 'op':
2786	diffn = 'DIFFOP'
2787	elif kdiff == 'var':
2788	diffn = 'DIFFVAR'
2789	else:
2790	print errmsg
2791	print ' ' + fname + ": differences '" + kdiff + "' not ready !!"
2792	quit(-1)
2793	LD = len(diffn + '_')
2794
2795	iop = 1
2796	# list of operations by difference
2797	doopers = {}
2798	# operations by variable
2799	operationsvar = {}
2800	# individual operations by variable (not repeated)
2801	indivoperationsvar = {}
2802
2803	# Variables with a calculation whic requires vertical interpolation to all the
2804	# file (operations stating by [kdiff]'_pinterp')
2805	LVp = len(diffn + '_pinterp')
2806	varglobalp = []
2807
2808	for oper in vc.keys():
2809	Loper = len(oper)
2810	opn = oper[LD:Loper+1]
2811	vns = vc[oper].split(':')
2812	ops1 = opn.split('+')
2813	doopers[opn] = ops1
2814	for vn in vns:
2815	if not operationsvar.has_key(vn):
2816	operationsvar[vn] = [opn]
2817	indivoperationsvar[vn] = ops1
2818	else:
2819	opers = operationsvar[vn]
2820	opers.append(opn)
2821	operationsvar[vn] = opers
2822	opers1 = indivoperationsvar[vn]
2823	for op1 in ops1:
2824	if not gen.searchInlist(opers1, op1):
2825	opers1.append(op1)
2826	indivoperationsvar[vn] = opers1
2827
2828	if oper[0:LVp] == diffn + '_pinterp':
2829	for vn in vns:
2830	if not gen.searchInlist(varglobalp,vn): varglobalp.append(vn)
2831
2832	iop = iop + 1
2833
2834	if db:
2835	print ' operations to make _______'
2836	gen.printing_dictionary(doopers)
2837	print ' operations by variable _______'
2838	gen.printing_dictionary(operationsvar)
2839	print ' individual operations by variable _______'
2840	gen.printing_dictionary(indivoperationsvar)
2841	print ' variables with the vertical interpolation for all the file _______'
2842	print ' #', varglobalp
2843
2844	if db: print " End of '" + fname + "' "
2845
2846	return doopers, operationsvar, indivoperationsvar, varglobalp
2847
2848	def diffvarop_listconstruct(config, mods, exps, reprj, odir, debug):
2849	""" Function to create the list of differences to compute and draw
2850	config= Configuration of the experiment
2851	mods= list with the models to use
2852	exps= list with the experiments to use
2853	reprj= list with whether re-rpojection needs to be done
2854	odir= output differences folder
2855	"""
2856	fname='diffvarop_listconstruct'
2857
2858	diffop = gen.get_specdictionary_HMT(config, H='DIFFOP_')
2859	diffvar = gen.get_specdictionary_HMT(config, H='DIFFVAR_')
2860
2861	if debug:
2862	if diffop is not None:
2863	print " 'op' differences ________"
2864	gen.printing_dictionary(diffop)
2865	if diffvar is not None:
2866	print " 'var' differences ________"
2867	gen.printing_dictionary(diffvar)
2868
2869	# Getting variable characteristics from each model/experiment
2870	idir1 = config['ofold'] + '/' + mods[0] + '/' + exps[0]
2871	idir2 = config['ofold'] + '/' + mods[1] + '/' + exps[1]
2872
2873	fvarcomp = 'varcompute.inf'
2874	if reprj[0]: fvc1 = 'reproj_' + fvarcomp
2875	else: fvc1 = fvarcomp
2876
2877	if reprj[1]: fvc2 = 'reproj_' + fvarcomp
2878	else: fvc2 = fvarcomp
2879
2880	files1, testfiles1, allcompvar1, Nvar1= read_varcomp_file(idir1 + '/' + fvc1)
2881	files2, testfiles2, allcompvar2, Nvar2= read_varcomp_file(idir2 + '/' + fvc2)
2882
2883	diffops = {}
2884	if diffop is not None:
2885	# Getting 'op' differences characteristics
2886	doops, opvar, indivopvar, varglobalp = get_differences_var(diffop,'op',debug)
2887
2888	Ndiffop = 0
2889	for vn in opvar.keys():
2890	vnops = opvar[vn]
2891	for op in vnops:
2892	vnop = vn + '_' + op
2893	if not allcompvar1.has_key(vnop):
2894	print warnmsg
2895	print ' ' + fname + ": DIFFOP variable+operation '" + vnop + \
2896	"' in '" + mods[0] + '/' + exps[0] + "' was not computed !!"
2897	print ' skipping it!'
2898	print ' variables computed:', allcompvar1
2899	break
2900	if not allcompvar2.has_key(vnop):
2901	print warnmsg
2902	print ' ' + fname + ": DIFFOP variable+operation '" + vnop + \
2903	"' in '" + mods[1] + '/' + exps[1] + "' was not computed !!"
2904	print ' skipping it!'
2905	print ' variables computed:', allcompvar2
2906	break
2907	vals1 = allcompvar1[vnop]
2908	vals2 = allcompvar2[vnop]
2909
2910	headerf1 = vals1[0]
2911	headerf2 = vals2[0]
2912
2913	diffops[vnop] = [mods[0],mods[1],exps[0],exps[1],headerf1,headerf2]
2914
2915	Ndiffop = Ndiffop + 1
2916	else:
2917	Ndiffop = 0
2918
2919	diffvars = {}
2920	if diffvar is not None:
2921	# Getting 'var' differences characteristics
2922	doops,opvar,indivopvar,varglobalp = get_differences_var(diffvar,'var',debug)
2923
2924	Ndiffvar = 0
2925	for vn in opvar.keys():
2926	vnops = opvar[vn]
2927	for op in vnops:
2928	vnop = vn + '_' + op
2929	if not allcompvar1.has_key(vnop):
2930	print warnmsg
2931	print ' ' + fname + ": DIFFVAR variable+operation '" + vnop + \
2932	"' in '" + mods[0] + '/' + exps[0] + "' was not computed !!"
2933	print ' skipping it!'
2934	print ' variables computed:', allcompvar1
2935	break
2936	if not allcompvar2.has_key(vnop):
2937	print warnmsg
2938	print ' ' + fname + ": DIFFVAR variable+operation '" + vnop + \
2939	"' in '" + mods[1] + '/' + exps[1] + "' was not computed !!"
2940	print ' skipping it!'
2941	print ' variables computed:', allcompvar2
2942	break
2943	vals1 = allcompvar1[vnop]
2944	vals2 = allcompvar2[vnop]
2945
2946	headerf1 = vals1[0]
2947	headerf2 = vals2[0]
2948
2949	diffvars[vnop] = [mods[0],mods[1],exps[0],exps[1],headerf1,headerf2]
2950
2951	Ndiffvar = Ndiffvar + 1
2952	else:
2953	Ndiffvar = 0
2954
2955	if diffop is not None:
2956	Sopdiffs = gen.dictKeysVals_stringList(diffops)
2957	else:
2958	Sopdiffs = 'none'
2959	# Outwritting the op diffferences file to avoid next time
2960	objf = open(odir + '/diffop.inf', 'w')
2961	objf.write('differences: ' + Sopdiffs + '\n')
2962	objf.write('Ndiff: ' + str(Ndiffop) + '\n')
2963	objf.close()
2964
2965	if diffvar is not None:
2966	Svardiffs = gen.dictKeysVals_stringList(diffvars)
2967	else:
2968	Svardiffs = 'none'
2969	# Outwritting the var diffferences file to avoid next time
2970	objf = open(odir + '/diffvar.inf', 'w')
2971	objf.write('differences: ' + Svardiffs + '\n')
2972	objf.write('Ndiff: ' + str(Ndiffvar) + '\n')
2973	objf.close()
2974
2975	if debug: print " End of '" + fname + "' "
2976
2977	return diffops, diffvars, Ndiffop, Ndiffvar
2978
2979	def read_diff_file(difffile):
2980	""" Function to read the file with the information about the differences
2981	difffile= file with the information
2982	"""
2983	fname = 'read_diff_file'
2984
2985	if not os.path.isfile(difffile):
2986	print errormsg
2987	print ' ' + fname + ": differences file '" + difffile + "' does not exist !!"
2988	quit(-1)
2989
2990	objf = open(difffile, 'r')
2991	for line in objf:
2992	if line[0:1] != '#' and len(line) > 1:
2993	values = line.replace('\n','').split(' ')
2994	if values[0] == 'differences:':
2995	if values[1] != 'none':
2996	diffs = gen.stringList_dictKeysVals(values[1])
2997	else:
2998	diffs = {}
2999	elif values[0] == 'Ndiff:':
3000	Ndiffs = int(values[1])
3001
3002	objf.close()
3003
3004	return diffs, Ndiffs
3005
3006	def compute_op_diffs(config, diffallop, odir, diffscr, debug):
3007	""" Function to compute operation differences
3008	config= experiment configuration
3009	diffallop= dictonary with the differences to compute
3010	odir= output folder
3011	diffscr= wether it should be done from the scratch
3012	"""
3013	fname = 'compute_op_diffs'
3014
3015	# output folder
3016	ofold = config['ofold']
3017
3018	# Home of the python scripts
3019	pyH = config['pyHOME']
3020
3021	# opsur = operation surnames
3022	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
3023	Opsurs = {}
3024	for opk in opsur:
3025	opn = opk.split('_')[1]
3026	vals = opsur[opk]
3027	Opsurs[opn] = vals.split(':')
3028
3029	# Getting in working dir
3030	os.chdir(odir)
3031
3032	# CF dimensions
3033	CFdims = ['lon', 'lat', 'pres', 'time']
3034
3035	# Trakcing file
3036	trkobjf = open(odir + '/all_diffop.inf', 'a')
3037
3038	for vnop in diffallop.keys():
3039	vn = vnop.split('_')[0]
3040	op = vnop.split('_')[1]
3041	opS = op.replace('+','_')
3042
3043	# `p' header add related to 'pinterp'
3044	if opS.find('pinterp') != -1: SgP = 'p'
3045	else: SgP = ''
3046
3047	# Variable name within the file
3048	vninF = varnoper(vn, op, Opsurs)
3049
3050	vals = diffallop[vnop]
3051	mod1 = vals[0]
3052	mod2 = vals[1]
3053	exp1 = vals[2]
3054	exp2 = vals[3]
3055	headerf1 = vals[4]
3056	headerf2 = vals[5]
3057	modexpdiff = mod2 + '/' + exp2 + ' - ' + mod1 + '/' + exp1
3058	modexpdiffS = mod2 + '-' + exp2 + '_' + mod1 + '-' + exp1
3059
3060	ifile1 = ofold + '/' + mod1 + '/' + exp1 + '/' + vn + '_' + headerf1 + SgP + \
3061	'_' + opS + '.nc'
3062	ifile2 = ofold + '/' + mod2 + '/' + exp2 + '/' + vn + '_' + headerf2 + SgP + \
3063	'_' + opS + '.nc'
3064	difffilen = odir + '/' + vn + '_diffop_' + modexpdiffS + '_' + opS + '.nc'
3065
3066	if diffscr:
3067	sout = sub.call('rm ' + difffilen + ' >& /dev/null', shell=True)
3068
3069	if not os.path.isfile(difffilen):
3070	if debug:
3071	print " Computing operation difference '" + op + "' of '" + vn + \
3072	"' for: '" + modexpdiff
3073	print " var in file: '" + vninF + "'"
3074
3075	# Range of the dimensions
3076	for CFd in CFdims:
3077	if CFd == CFdims[0]:
3078	CFdimS = CFd + '\|' + CFd + '\|-1'
3079	else:
3080	CFdimS = CFdimS + ';' + CFd + '\|' + CFd + '\|-1'
3081
3082	ncvar.ivattrs(ifile2,vninF)
3083
3084	# Attributes of the variable
3085	try:
3086	with gen.Capturing() as output:
3087	varattrs = ncvar.ivattrs(ifile2,vninF)
3088	except:
3089	print errmsg
3090	print 'ncvar.ivarattrs(' + ifile2 + ',' + vninF + ')'
3091	for s1out in output: print s1out
3092	quit(-1)
3093
3094	if debug:
3095	for s1out in output: print s1out
3096	varvals = vninF + ',' + varattrs['long_name'][0] + ',' + \
3097	varattrs['units'][0]
3098
3099	values = CFdimS + '@' + 'add\|' + ifile2 + '\|' + vninF + '%' + CFdimS + \
3100	'@' + 'sub\|' + ifile1 + '\|' + vninF
3101	pyins = 'python ' + pyH + "/nc_var.py -o compute_opersvarsfiles -S '" + \
3102	values + "' -v '" + varvals + "'"
3103
3104	try:
3105	with gen.Capturing() as output:
3106	ncvar.compute_opersvarsfiles(values,varvals)
3107	except:
3108	print errmsg
3109	print 'ncvar.compute_opersvarsfiles(' + values + ',' + varvals + ')'
3110	for s1out in output: print s1out
3111	quit(-1)
3112
3113	sout = sub.call('mv opersvarsfiles_'+vninF+'.nc ' + difffilen, shell=True)
3114
3115	# keeping all differences
3116	trkobjf.write('\n')
3117	trkobjf.write("#" + vn + ' ' + op + '\n')
3118	trkobjf.write( pyins + '\n')
3119
3120	trkobjf.close()
3121	if debug: print " End of '" + fname + "' "
3122
3123	return
3124
3125	def compute_diff_statistics(Ecnf, filen, ofold, varn, Opers, scr, db):
3126	""" Function to compute statistics from var diff files
3127	Ecnf= dictionary with the configuration of the experiment
3128	filen= variable difference file to use
3129	ofold= directory to write the output files
3130	varn= name of the variable
3131	Opers= kind of operation: (as possible multiple consecutive combination of operations separated by '+'
3132	[calc1]+[calc2] will compute first [calc1] and then [calc2]
3133	acc: temporal accumulated values
3134	diff: differences between models
3135	dimvarvalue~[dimvarn]~[value]: Slicing along the index at the nearest given value of a dimension-variable
3136	direct: no statistics
3137	last: last temporal value
3138	Lmean: latitudinal mean values
3139	Lsec: latitudinal section (latitudinal value must be given, [var]@[lat])
3140	lmean: longitudinal mean values
3141	lsec: longitudinal section (longitudinal value must be given, [var]@[lat])
3142	pinterp: pressure interpolation (to the given $plevels)
3143	tmean: temporal mean values
3144	tstd: temporal standard deviation values
3145	tturb: Taylor's turbulence decomposition value (x - <x>) for time
3146	tvar: temporal variance values
3147	xmean: x-axis mean values
3148	xvar: x-axis variance values
3149	ymean: y-axis mean values
3150	zsum: vertical aggregated values
3151	scr= should it be done from the scratch?
3152	"""
3153	fname='compute_diff_statistics'
3154
3155	# Full header of file
3156	Lfilen = len(filen)
3157	Hfile = filen[0:Lfilen-3]
3158
3159	# CF dimension-variables name
3160	OpersS = '+'.join(Opers)
3161	if OpersS.find('pinterp') != -1 or filen.find('pinterp') != -1:
3162	varnCFs = ['lon', 'lat', 'pres', 'time']
3163	else:
3164	varnCFs = ['lon', 'lat', 'time']
3165
3166	# Variable-dimension CF dictionary
3167	CFdimvardict = {'lon': 'lon', 'lat': 'lat', 'pres': 'pres', 'time': 'time'}
3168
3169	# Python scripts HOME
3170	pyH = Ecnf['pyHOME']
3171
3172	# opsur = operation surnames
3173	opsur = gen.get_specdictionary_HMT(Ecnf, H='opsur_',M='',T='')
3174	Opsurs = {}
3175	for opk in opsur:
3176	opn = opk.split('_')[1]
3177	vals = opsur[opk]
3178	Opsurs[opn] = vals.split(':')
3179
3180	# Getting in working dir
3181	os.chdir(ofold)
3182
3183	# File to keep track of all the statistics of var differences
3184	otrackf = open(ofold + '/all_vardiffstatistics.inf', 'a')
3185
3186	if db:
3187	print " computing statistics of diff variable: '", varn ,"' operation '"+ \
3188	OpersS + "' using file '" + filen + " ...'"
3189
3190	for op in Opers:
3191	# File name from previous operations, and name of the variable within the
3192	# file (some operations change the name of the variable)
3193	if op == Opers[0]:
3194	Fopers = op
3195	prevfile = filen
3196	vninF = varn
3197	else:
3198	Fopers = Fopers + '_' + op
3199	prevfile = fileon
3200	fileon = Hfile + '_' + Fopers + '.nc'
3201
3202	SvarnCFs = ',' + ','.join(varnCFs)
3203	CFvarnp = vninF + SvarnCFs
3204
3205	if scr:
3206	sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
3207
3208	# Just produce file in case output file does not exist
3209	if not os.path.isfile(fileon):
3210	if db:
3211	print ' ' + fname + ": creation of file '" + fileon + "' ..."
3212	dofile = True
3213	else:
3214	dofile = False
3215
3216	# Variables to be kept in the final file
3217	varkeep = []
3218
3219	if op == 'acc':
3220	# temporal accumulated values
3221	print " " + fname + ": kind '" + op + "' not ready !!"
3222	wrongstats.append(varn + '_' + opers)
3223	break
3224
3225	elif op[0:11] == 'dimvarvalue':
3226	# Slicing along the index at the nearest given value of a dimension-variable
3227	dimvarn = op.split('~')[1]
3228	dimvalue = op.split('~')[2]
3229	vals = dimvarn + ',' + dimvalue + ',' + dimvalue + ',0'
3230	if dofile:
3231	try:
3232	with gen.Capturing() as output:
3233	ncvar.DataSetSection_multivars(vals,prevfile,CFvarnp)
3234	except:
3235	print errmsg
3236	print 'DataSetSection_multivars('+vals+',', prevfile, ',' + \
3237	CFvarnp + ')'
3238	for s1out in output: print s1out
3239	quit(-1)
3240
3241	if db:
3242	for s1out in output: print s1out
3243
3244	sout = sub.call('mv DataSetSection_multivars.nc '+fileon, shell=True)
3245
3246	# Keeping the operations
3247	pyins=pyH + "/nc_var.py -o DataSetSection_multivars -S '" + vals + \
3248	"' -f " + prevfile + " -v " + CFvarnp
3249	otrackf.write("\n")
3250	otrackf.write("# " + CFvarnp + " " + Fopers + "\n")
3251	otrackf.write(pyins + "\n")
3252
3253	# removing the given variable-dimension
3254	varnCFs.remove(dimvarn)
3255
3256	elif op == 'direct':
3257	# no statistics
3258	if dofile:
3259	sout = sub.call('mv ' + prevfile + ' ' + fileon, shell=True)
3260	elif op == 'last':
3261	# last temporal value
3262	vals='time,-9,0,0'
3263	if dofile:
3264	try:
3265	with gen.Capturing() as output:
3266	ncvar.DataSetSection(vals,prevfile)
3267	except:
3268	print errmsg
3269	print 'DataSetSection('+vals+',', prevfile, ')'
3270	for s1out in output: print s1out
3271	quit(-1)
3272
3273	if db:
3274	for s1out in output: print s1out
3275
3276	hprevfile = prevfile[0:len(prevfile)-3]
3277
3278	sout = sub.call('mv ' + hprevfile + '_time_B-9-E0-I0.nc ' + fileon, \
3279	shell=True)
3280
3281	# Keeping the operations
3282	pyins=pyH + "/nc_var.py -o DataSetSection -S '" + vals + "' -f " + \
3283	prevfile
3284	otrackf.write("\n")
3285	otrackf.write("# " + varn + " " + Fopers + "\n")
3286	otrackf.write(pyins + "\n")
3287
3288	elif op =='Lmean':
3289	# latitudinal mean values
3290	print " " + fname + ": kind '" + op + "' not ready !!"
3291	wrongstats.append(varn + '_' + opers)
3292	break
3293	elif op =='Lsec':
3294	# latitudinal section (latitudinal value must be given, [var]@[lat])
3295	print " " + fname + ": kind '" + op + "' not ready !!"
3296	wrongstats.append(varn + '_' + opers)
3297	break
3298	elif op =='lmean':
3299	# longitudinal mean values
3300	print " " + fname + ": kind '" + op + "' not ready !!"
3301	wrongstats.append(varn + '_' + opers)
3302	break
3303	elif op =='lsec':
3304	# longitudinal section (longitudinal value must be given, [var]@[lon])
3305	print " " + fname + ": kind '" + op + "' not ready !!"
3306	wrongstats.append(varn + '_' + opers)
3307	break
3308	elif op == 'tmean':
3309	# temporal mean values
3310	vals='time\|-1,time,mean,' + ':'.join(varnCFs)
3311	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3312	if dofile:
3313	try:
3314	with gen.Capturing() as output:
3315	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3316	except:
3317	print errmsg
3318	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3319	for s1out in output: print s1out
3320	quit(-1)
3321
3322	if db:
3323	for s1out in output: print s1out
3324
3325	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
3326
3327	# Keeping the operations
3328	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3329	"' -f " + prevfile + " -v " + CFvarnp
3330	otrackf.write("\n")
3331	otrackf.write("# " + varn + " " + Fopers + "\n")
3332	otrackf.write(pyins + "\n")
3333
3334	# removing dimension variable-dimension 'time'
3335	varnCFs.remove('time')
3336
3337	varkeep.append('timestats')
3338
3339	elif op == 'tstd':
3340	# temporal standard deviation values
3341	vals='time\|-1,time,std,' + ':'.join(varnCFs) + ':' + vdnz
3342	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
3343	if dofile:
3344	try:
3345	with gen.Capturing() as output:
3346	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3347	except:
3348	print errmsg
3349	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3350	for s1out in output: print s1out
3351	quit(-1)
3352
3353	if db:
3354	for s1out in output: print s1out
3355
3356	sout = sub.call('mv file_oper_alongdims_std.nc '+fileon, shell=True)
3357
3358	# Keeping the operations
3359	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3360	"' -f " + prevfile + " -v " + CFvarnp
3361	otrackf.write("\n")
3362	otrackf.write("# " + varn + " " + Fopers + "\n")
3363	otrackf.write(pyins + "\n")
3364
3365	# removing dimension variable-dimension 'time'
3366	varnCFs.remove('time')
3367
3368	varkeep.append('timestats')
3369
3370	elif op == 'tturb':
3371	# temporal turbulent values
3372	vals='time\|-1,time,turb,' + ':'.join(varnCFs)
3373	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3374	if dofile:
3375	try:
3376	with gen.Capturing() as output:
3377	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3378	except:
3379	print errmsg
3380	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3381	for s1out in output: print s1out
3382	quit(-1)
3383
3384	if db:
3385	for s1out in output: print s1out
3386
3387	sout = sub.call('mv file_oper_alongdims_turb.nc '+fileon, shell=True)
3388
3389	# Keeping the operations
3390	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3391	"' -f " + prevfile + " -v " + CFvarnp
3392	otrackf.write("\n")
3393	otrackf.write("# " + varn + " " + Fopers + "\n")
3394	otrackf.write(pyins + "\n")
3395
3396	varkeep.append('timestats')
3397
3398	elif op == 'tvar':
3399	# temporal variance values (Taylor's turbulence)
3400	vals='time\|-1,time,var,' + ':'.join(varnCFs) + ':' + vdnz
3401	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
3402	if dofile:
3403	try:
3404	with gen.Capturing() as output:
3405	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3406	except:
3407	print errmsg
3408	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3409	for s1out in output: print s1out
3410	quit(-1)
3411
3412	if db:
3413	for s1out in output: print s1out
3414
3415	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
3416
3417	# Keeping the operations
3418	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3419	"' -f " + prevfile + " -v " + CFvarnp
3420	otrackf.write("\n")
3421	otrackf.write("# " + varn + " " + Fopers + "\n")
3422	otrackf.write(pyins + "\n")
3423
3424	# removing dimension variable-dimension 'time'
3425	varnCFs.remove('time')
3426
3427	varkeep.append('timestats')
3428
3429	elif op == 'xmean':
3430	# x-axis mean values
3431	vals='lon\|-1,lon,mean,' + ':'.join(varnCFs)
3432	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3433	if dofile:
3434	try:
3435	with gen.Capturing() as output:
3436	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3437	except:
3438	print errmsg
3439	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3440	for s1out in output: print s1out
3441	quit(-1)
3442
3443	if db:
3444	for s1out in output: print s1out
3445
3446	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
3447
3448	# Keeping the operations
3449	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3450	"' -f " + prevfile + " -v " + CFvarnp
3451	otrackf.write("\n")
3452	otrackf.write("# " + varn + " " + Fopers + "\n")
3453	otrackf.write(pyins + "\n")
3454
3455	# removing dimension variable-dimension 'lon'
3456	varnCFs.remove('lon')
3457
3458	varkeep.append('lonstats')
3459
3460	elif op == 'xvar':
3461	# x-axis variance values
3462	vals='lon\|-1,lon,var,' + ':'.join(varnCFs)
3463	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3464	if dofile:
3465	try:
3466	with gen.Capturing() as output:
3467	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3468	except:
3469	print errmsg
3470	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3471	for s1out in output: print s1out
3472	quit(-1)
3473
3474	if db:
3475	for s1out in output: print s1out
3476
3477	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
3478
3479	# Keeping the operations
3480	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3481	"' -f " + prevfile + " -v " + CFvarnp
3482	otrackf.write("\n")
3483	otrackf.write("# " + varn + " " + Fopers + "\n")
3484	otrackf.write(pyins + "\n")
3485
3486	# removing dimension variable-dimension 'lon'
3487	varnCFs.remove('lon')
3488
3489	varkeep.append('lonstats')
3490
3491	elif op == 'ymean':
3492	# y-axis mean values
3493	vals='lat\|-1,lat,mean,' + ':'.join(varnCFs)
3494	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3495	if dofile:
3496	try:
3497	with gen.Capturing() as output:
3498	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3499	except:
3500	print errmsg
3501	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3502	for s1out in output: print s1out
3503	quit(-1)
3504
3505	if db:
3506	for s1out in output: print s1out
3507
3508	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
3509
3510	# Keeping the operations
3511	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3512	"' -f " + prevfile + " -v " + CFvarnp
3513	otrackf.write("\n")
3514	otrackf.write("# " + varn + " " + Fopers + "\n")
3515	otrackf.write(pyins + "\n")
3516
3517	# removing dimension variable-dimension 'lat'
3518	varnCFs.remove('lat')
3519
3520	varkeep.append('latstats')
3521
3522	elif op == 'zsum':
3523	# vertical aggregated values
3524	print " " + fname + ": kind '" + op + "' not ready !!"
3525	wrongstats.append(varn + '_' + opers)
3526	break
3527	else:
3528	print errmsg
3529	print ' ' + fname + ": operation '" + op + "' not ready !!"
3530	quit(-1)
3531
3532	# End of kind of operation
3533
3534	# Variable name in file (vninF) changed due to operation
3535	# but only if previous operation does not the same 'statistic'
3536	chvn = gen.dictionary_key_list(Opsurs, op)
3537	if chvn is not None:
3538	oldvninF = vninF
3539	vninF = vninF + chvn
3540	CFvarnp = CFvarnp.replace(oldvninF,vninF)
3541
3542	if dofile:
3543	if len(varkeep) > 0:
3544	varkeepS = ',' + ','.join(varkeep)
3545	else:
3546	varkeepS = ''
3547
3548	# Adding such CF dimension variables which might not be in the
3549	# post-operation file
3550	try:
3551	with gen.Capturing() as output:
3552	varinfile = ncvar.ivars(fileon)
3553	except:
3554	print errmsg
3555	print 'ivars('+fileon+')'
3556	for s1out in output: print s1out
3557	# Removing file in order to make sure that it will be redone
3558	print ' ' + fname + " removing file '" + fileon + "' ..."
3559	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
3560	quit(-1)
3561
3562	for CFvn in varnCFs:
3563	if not gen.searchInlist(varinfile, CFvn):
3564	if db:
3565	print ' ' + fname + ": recupering CF variable '" + CFvn + "'"
3566	try:
3567	with gen.Capturing() as output:
3568	ncvar.fvaradd(prevfile+','+CFvn, fileon)
3569	except:
3570	print errmsg
3571	print 'fvaradd(' + prevfile + ',' + CFvn +', ' + fileon + ')'
3572	for s1out in output: print s1out
3573	# Removing file in order to make sure that it will be redone
3574	print ' ' + fname + " removing file '" + fileon + "' ..."
3575	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
3576	quit(-1)
3577
3578	if db:
3579	for s1out in output: print s1out
3580
3581	totalvarkeeps = CFvarnp + ',' + ','.join(varnCFs) + varkeepS
3582	try:
3583	with gen.Capturing() as output:
3584	oclean = ncvar.cleaning_varsfile(totalvarkeeps,fileon)
3585	except:
3586	print errmsg
3587	print 'cleaning_varsfile('+totalvarkeeps+','+fileon+')'
3588	for s1out in output: print s1out
3589	# Removing file in order to make sure that it will be redone
3590	print ' ' + fname + " removing file '" + fileon + "' ..."
3591	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
3592	quit(-1)
3593
3594	if db:
3595	for s1out in output: print s1out
3596
3597	# End of operations
3598	otrackf.close()
3599
3600	if db: print " End of '" + fname + "' "
3601
3602	return
3603
3604	def compute_var_diffs(config, diffallvar, odir, diffscr, debug):
3605	""" Function to compute variable differences
3606	config= experiment configuration
3607	diffallvar= dictonary with the differences to compute
3608	odir= output folder
3609	diffscr= wether it should be done from the scratch
3610	"""
3611	fname = 'compute_var_diffs'
3612
3613	# output folder
3614	ofold = config['ofold']
3615
3616	# Home of the python scripts
3617	pyH = config['pyHOME']
3618
3619	# opsur = operation surnames
3620	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
3621	Opsurs = {}
3622	for opk in opsur:
3623	opn = opk.split('_')[1]
3624	vals = opsur[opk]
3625	Opsurs[opn] = vals.split(':')
3626
3627	# Getting in working dir
3628	os.chdir(odir)
3629
3630	# CF dimensions
3631	CFdims = ['lon', 'lat', 'pres', 'time']
3632
3633	# Trakcing file
3634	trkobjf = open(odir + '/all_diffvar.inf', 'a')
3635
3636	for vnop in diffallvar.keys():
3637	vn = vnop.split('_')[0]
3638	op = vnop.split('_')[1]
3639	opSfinal = op.replace('+','_')
3640
3641	lop = op.split('+')
3642	# `p' header add related to 'pinterp'
3643	if op.find('pinterp') != -1:
3644	print warnmsg
3645	print ' ' + fname + ': there is the vertical interpolation as operation'
3646	print ' variable differences will be computed from that files where '
3647	print ' vertical interpolation was just done'
3648	ipop = lop.index('pinterp')
3649	opS = '_' + '_'.join(lop[0:ipop+1])
3650	doops = lop[ipop+1:]
3651	SgP = 'p'
3652	vninF = varnoper(vn, '+'.join(lop[0:ipop+1]), Opsurs)
3653	print ' operations to perform:', doops
3654	else:
3655	SgP = ''
3656	opS = ''
3657	doops = lop
3658	vninF = vn
3659
3660	Sdoops = '+'.join(doops)
3661
3662	# Variable name within the file
3663
3664	vals = diffallvar[vnop]
3665	mod1 = vals[0]
3666	mod2 = vals[1]
3667	exp1 = vals[2]
3668	exp2 = vals[3]
3669	headerf1 = vals[4]
3670	headerf2 = vals[5]
3671	modexpdiff = mod2 + '/' + exp2 + ' - ' + mod1 + '/' + exp1
3672	modexpdiffS = mod2 + '-' + exp2 + '_' + mod1 + '-' + exp1
3673
3674	ifile1 = ofold + '/' + mod1 + '/' + exp1 + '/' + vn + '_' + headerf1 + SgP + \
3675	opS + '.nc'
3676	ifile2 = ofold + '/' + mod2 + '/' + exp2 + '/' + vn + '_' + headerf2 + SgP + \
3677	opS + '.nc'
3678	difffilen = odir + '/' + vn + '_diffvar_' + modexpdiffS + opS + '.nc'
3679	difffinalfilen = odir + '/' + vn + '_diffvar_' + modexpdiffS + opS + '_' + \
3680	Sdoops + '.nc'
3681
3682	if diffscr:
3683	sout = sub.call('rm ' + difffilen + ' >& /dev/null', shell=True)
3684	sout = sub.call('rm ' + difffinalfilen + ' >& /dev/null', shell=True)
3685
3686	if not os.path.isfile(difffinalfilen):
3687	if debug:
3688	print " Computing variable difference of '" + vn + "' for: '" + \
3689	modexpdiff + "' and performing operation '" + Sdoops + "'"
3690	print " var in file: '" + vninF + "'"
3691
3692	# Range of the dimensions
3693	for CFd in CFdims:
3694	if CFd == CFdims[0]:
3695	CFdimS = CFd + '\|' + CFd + '\|-1'
3696	else:
3697	CFdimS = CFdimS + ';' + CFd + '\|' + CFd + '\|-1'
3698
3699	# Attributes of the variable
3700	with gen.Capturing() as output:
3701	varattrs = ncvar.ivattrs(ifile2,vninF)
3702	if debug:
3703	for s1out in output: print s1out
3704	varvals = vninF + ',' + varattrs['long_name'][0] + ',' + \
3705	varattrs['units'][0]
3706
3707	values = CFdimS + '@' + 'add\|' + ifile2 + '\|' + vninF + '%' + CFdimS + \
3708	'@' + 'sub\|' + ifile1 + '\|' + vninF
3709	pyins = 'python ' + pyH + "/nc_var.py -o compute_opersvarsfiles -S '" + \
3710	values + "' -v '" + varvals + "'"
3711
3712	try:
3713	with gen.Capturing() as output:
3714	ncvar.compute_opersvarsfiles(values,varvals)
3715	except:
3716	print errmsg
3717	print 'ncvar.compute_opersvarsfiles(' + values + ', ' + varvals + ')'
3718	for s1out in output: print s1out
3719	quit(-1)
3720
3721	sout = sub.call('mv opersvarsfiles_'+vninF+'.nc ' + difffilen, shell=True)
3722
3723	# keeping all differences
3724	trkobjf.write('\n')
3725	trkobjf.write("#" + vn + ' ' + op + '\n')
3726	trkobjf.write( pyins + '\n')
3727
3728	# Computing statisitcs
3729	compute_diff_statistics(config, difffilen, odir, vninF, doops, diffscr, \
3730	debug)
3731
3732	trkobjf.close()
3733
3734	if debug: print " End of '" + fname + "' "
3735
3736	return
3737
3738
3739	def draw_diff_plots(config, plots, odir, allvarcomp, kdiff, figscr, debug):
3740	""" Function to draw all plots
3741	config= Configuration of the experiment
3742	plots= dictionary with the plots
3743	odir= output experiment folder
3744	allvarcomp= dictionary with all the variables to compute and their information
3745	kdiff= kind of differences:
3746	'diffop': plots from operation differences
3747	'diffvar': plots from variable differences
3748	figscr= whether figures should be done from the scratch or not
3749
3750	* Plot as
3751	{[kplot]} = [varn1]\|[op1]#[varn2]\|[op2]#[...[varnN]\|[opN]], ...
3752	[kplot] ___
3753	diffmap2Dsfc: 2D map of surface differences values of 1 variable
3754	diffmap2Dz: 2D map of 3D differences values of 1 variable
3755	map2Dsfc: 2D map of surface values of 1 variable
3756	map3D: 2D map of 3D values of 1 variable
3757	shadconthovmsfc: Hovmoeller diagrams of 2 variable at the surface in shadow and the other in contourn
3758	shadcont2Dsfc: 2D map of shadow (1st variable) and countour (2nd variable) [stvar1]#[stvar2]
3759	shadcont2Dzsec: 2D map of vertical section of 2 variables one in shadow and the other in contourn
3760	[varn]
3761	variable
3762	[op]
3763	'+' separated list of operations
3764	in figures with more than 1 variable, use '#' to separate the [varn]\|[op]
3765	"""
3766	fname = 'draw_diff_plots'
3767
3768	os.chdir(odir)
3769
3770	# Dictionary with the operations with surnames for the operated variable
3771	opersurnames = {}
3772	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
3773	for opsr in opsur.keys():
3774	opn = opsr.split('_')[1]
3775	vls = opsur[opsr].split(':')
3776	opersurnames[opn] = vls
3777
3778	# time values
3779	# Units time for the plots
3780	rd = config['CFreftime']
3781	tunits = config['CFunitstime'] + '!since!' + rd[0:4] + '-' + rd[4:6] + '-' + \
3782	rd[6:8] + '!' + rd[8:10] + ':' + rd[10:12] + ':' + rd[12:14]
3783	# time ticks kind
3784	tkind = config['timekind']
3785	# time ticks format
3786	tfmt = config['timefmt']
3787	# time axis label
3788	tlab = config['timelabel']
3789	timevals = [tunits, tkind, tfmt, tlab]
3790
3791	if kdiff == 'diffop':
3792	specplotkeyn = 'specificdiffopplot'
3793	elif kdiff == 'diffvar':
3794	specplotkeyn = 'specificdiffvarplot'
3795	else:
3796	print errmsg
3797	print ' ' + fname + ": differences kind '" + kdiff + "' not ready !!"
3798	quit(-1)
3799
3800	# Dictionary of plot specificities
3801	plotspecifics = {}
3802	if config.has_key(specplotkeyn):
3803	# [minval]: minimum value
3804	# [maxval]: minimum value
3805	# [colorbar]: name of the colorbar (from matplotlib) to use
3806	# [cntformat]: format of the contour labels
3807	# [colorcnt]: color for the countor lines
3808	plotspecs = config[specplotkeyn].split(':')
3809	for pltspc in plotspecs:
3810	pltvls = pltspc.split('\|')
3811	vn = pltvls[0]
3812	op = pltvls[1]
3813	fn = pltvls[2]
3814	plotspecifics[fn + '_' + vn + '_' + op] = pltvls[3:]
3815	if debug:
3816	print 'Specific values for plots _______'
3817	gen.printing_dictionary(plotspecifics)
3818
3819	# Kind of figures
3820	kindfigure = config['kindfig']
3821
3822	# Graphical labels and configuration
3823	modgc, expgc = graphical_characteristics(config, debug)
3824
3825	# Map value
3826	mapvalue = config['mapval']
3827
3828	# pythone scripts HOME
3829	pyHOME = config['pyHOME']
3830
3831	# Title-text of operations
3832	opexplained = {}
3833	optits = config['titleoperations'].split(':')
3834	for optit in optits:
3835	opn = optit.split('\|')[0]
3836	opt = optit.split('\|')[1]
3837	opexplained[opn] = opt
3838	if debug:
3839	print 'Titles for operations _______'
3840	gen.printing_dictionary(opexplained)
3841
3842	for kplot in plots.keys():
3843	varsplt = plots[kplot]
3844	for varplt in varsplt:
3845	if debug:
3846	print " printing '" + kplot + "' var ':" + varplt + "'..."
3847	varops = varplt.split('#')
3848
3849	# CF variables in plot
3850	CFvarsplot = []
3851	# Files in plot
3852	filesplot = []
3853	# Variables in plot within the files
3854	varsplot = []
3855	# Dims in figure
3856	dimsplot = []
3857	# pictoric values in figure
3858	pictplot = []
3859	# Name of the figure
3860	figname = ''
3861	# Title of the figure
3862	titfigure = ''
3863
3864	ivp = 0
3865	for varop in varops:
3866	vn = varop.split('\|')[0]
3867	op = varop.split('\|')[1]
3868
3869	# CF variables in plot
3870	CFvarsplot.append(vn)
3871
3872	vnopS = vn + '_' + op
3873	if not allvarcomp.has_key(vnopS):
3874	print errmsg
3875	print ' ' + fname + ": no file for variable-operation '" + \
3876	vnopS + "' !!"
3877	print ' available ones:', allvarcomp.keys()
3878	quit(-1)
3879	vopvals = allvarcomp[vnopS]
3880	mod1 = vopvals[0]
3881	mod2 = vopvals[1]
3882	mod3 = vopvals[2]
3883	mod4 = vopvals[3]
3884	headf = vopvals[4]
3885
3886	modv = modgc[mod1]
3887	expv = expgc[mod1+'/'+exp1]
3888	modlab1 = modv.label
3889	explab1 = expv.label
3890	modv = modgc[mod2]
3891	expv = expgc[mod2+'/'+exp2]
3892	modlab2 = modv.label
3893	explab2 = expv.label
3894
3895	modexpdiff = explab2 + '-' + explab1
3896	modexpdiffS = mod2 + '-' + exp2 + '_' + mod1 + '-' + exp1
3897
3898	difffilen = odir + '/' + vn + '_' + kdiff + '_' + modexpdiffS + \
3899	'_' + op.replace('+','_') + '.nc'
3900
3901	filesplot.append(difffilen)
3902	# Do we have processed the given difference?
3903	if not os.path.isfile(difffilen):
3904	print warnmsg
3905	print " " + fname + ": there is no file for '" + kdiff + \
3906	"' difference '" + varop + "' skiping it !!"
3907	break
3908
3909	# Name of the variable inside the file
3910	vnsur = varnoper(vn, op, opersurnames)
3911	varsplot.append(vnsur)
3912
3913	# Dimensions in file
3914	try:
3915	with gen.Capturing() as output:
3916	dims = ncvar.idims(difffilen)
3917	except:
3918	print errmsg
3919	print 'ncvar.idims('+difffilen+')'
3920	for s1out in output: print s1out
3921	quit(-1)
3922
3923	dimsplot.append(dims)
3924
3925	# pictoric values for the figure
3926	Sfivaop = kplot + '_' + vn + '_' + op
3927	if plotspecifics.has_key(Sfivaop):
3928	pictvals = plotspecifics[Sfivaop]
3929	else:
3930	Vvals = gen.variables_values(vn)
3931	pictvals = [Vvals[2], Vvals[3], Vvals[6], '%g', 'fixsigc','black']
3932
3933	pictplot.append(pictvals)
3934
3935	# Header of the name of the figure
3936	if ivp == 0:
3937	figname = kplot + '_' + vn + '_' + kdiff + '_' + modexpdiffS + \
3938	'_' + op.replace('+','-')
3939	else:
3940	figname = figname + '-' + vn + '_' + op.replace('+','-')
3941
3942	# Title of the figure:
3943	if mod1 == mod2:
3944	modexptit = mod1 + '!' + exp2 + '-' + exp1
3945	else:
3946	modexptit = modexpdiff
3947
3948	ivp = ivp + 1
3949	# End of variable-operation
3950	figname = figname + '.' + kindfigure
3951
3952	# Title of figure
3953	titfigure = create_figure_title(modexptit, kdiff, varops, opexplained)
3954
3955	if len(titfigure) > 80:
3956	print warnmsg
3957	print ' ' + fname + ": figure title '" + titfigure.replace('!', ' ')+\
3958	"' larger than 80 characters (actually", len(titfigure), ') !!'
3959	print " simplifying it to '" + modexptit.replace('!',' ') + '!' + \
3960	kdiff + '!' + vnS + "'"
3961	titfigure = modexptit.replace(' ','!') + '!' + kdiff + '!' + vnS
3962
3963	draw_plot(kplot, CFvarsplot, filesplot, varsplot, dimsplot, pictplot, \
3964	figname, titfigure, kindfigure, mapvalue, timevals, expgc, odir, \
3965	pyHOME, figscr, debug)
3966
3967	# End of variables-operations
3968
3969	# End of kind of plots
3970
3971	if debug: print " End of '" + fname + "' "
3972
3973	return
3974
3975	def reproject_modexp(mod,exp,config,REPRJops,fscr,fadd,debug):
3976	""" Function to re-project and re-compute files from a given model/experiment
3977	mod= model
3978	exp= experiment
3979	config= configuration
3980	REPRJops= dictionary with the reprojection operators to use for reproject and for which variable
3981	# remapbil CDO: Bilinear interpolation
3982	# remapbic CDO: Bicubic interpolation
3983	# remapdis CDO: Distance-weighted average remapping
3984	# remapnn CDO: Nearest neighbor remapping
3985	# Operators only available if projections have the corners of the grid points.
3986	# remapcon CDO: First order conservative remapping
3987	# remapcon2 CDO: Second order conservative remapping
3988	# remaplaf CDO: Largest area fraction remapping
3989	####### ####### ####### or additionally ####### ####### #######
3990	# python remapping
3991	# dis python: Distance-weighted average remapping
3992	# pnn python: Nearest neighbor remapping
3993
3994	fscr= re-projected files to compute from the scratch
3995	fadd= re-projected files to be added
3996	"""
3997	fname = 'reproject_modexp'
3998
3999	odir = config['ofold'] + '/' + mod + '/' + exp
4000
4001	# NON re-projectable operations
4002	opNOreproj = config['NOreprojops'].split(':')
4003
4004	# Need to pass to analyze all the data?
4005	inffiles = ['reproj_varcompute.inf', 'all_reproj_computevars.inf']
4006	if fscr:
4007	for inff in inffiles:
4008	if debug: print " removing information file '" + inff + "' ..."
4009	ins = 'rm ' + odir + '/' + inff + ' >& /dev/null'
4010	sout = sub.call(ins, shell=True)
4011
4012	objf = open(odir+'/all_reproj_computevars.inf','w')
4013	objf.write("## Computation of reprojected variables \n")
4014	objf.close()
4015
4016	varcompf = odir + '/reproj_varcompute.inf'
4017	if fadd:
4018	sub.call('rm ' + varcompf +' >& /dev/null', shell=True)
4019
4020	# CF dims-vardims pairs:
4021	CFvardims = ['long@lon', 'lat@lat', 'pres@pres', 'time@time']
4022
4023	# Dictionary with the operations with surnames for the operated variable
4024	opersurnames = {}
4025	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
4026	for opsr in opsur.keys():
4027	opn = opsr.split('_')[1]
4028	vls = opsur[opsr].split(':')
4029	opersurnames[opn] = vls
4030
4031	if not os.path.isfile(varcompf):
4032	# Getting original computing information
4033	origvarcompf = odir + '/varcompute.inf'
4034	origfiles,origtestfiles,origallcompvar, Nvar = read_varcomp_file(origvarcompf)
4035
4036	Svarcompute = ''
4037	allcompvar = dict(origallcompvar)
4038	ivop = 0
4039	for vnop in origallcompvar.keys():
4040	vn = vnop.split('_')[0]
4041	op = vnop.split('_')[1]
4042	values = allcompvar[vnop]
4043	values[0] = 'reproj-' + values[0]
4044	allcompvar[vnop] = values
4045
4046	# Building strings from list values
4047	model = values[1]
4048	diag = values[2]
4049	if model is not None:
4050	if type(model) == type(list([1, 2])):
4051	Smodel = ':'.join(model)
4052	elif type(model) == type('A'):
4053	Smodel = model
4054	else:
4055	Smodel = 'None'
4056	if diag is not None:
4057	if type(diag) == type(list([1, 2])):
4058	Sdiag = ':'.join(diag)
4059	elif type(diag) == type('A'):
4060	Sdiag = diag
4061	else:
4062	Sdiag = 'None'
4063
4064	Svc = vn + '\|' + op + '\|' + values[0] + '\|' + Smodel + '\|' + Sdiag + \
4065	'\|' + values[3]
4066	if ivop == 0:
4067	Svarcompute = Svc
4068	else:
4069	Svarcompute = Svarcompute + ',' + Svc
4070	ivop = ivop + 1
4071
4072	origstrfiles = gen.dictKeysVals_stringList(origfiles)
4073	origstrtestfiles = gen.dictKeysVals_stringList(origtestfiles)
4074
4075	# Outwritting the varcompute to avoid next time (if it is not filescratch!)
4076	objf = open(odir + '/reproj_varcompute.inf', 'w')
4077	objf.write('files: ' + origstrfiles + '\n')
4078	objf.write('testfiles: ' + origstrtestfiles + '\n')
4079	objf.write('varcompute: ' + Svarcompute + '\n')
4080	objf.write('itotv: ' + str(ivop) + '\n')
4081	objf.close()
4082	else:
4083	if dbg:
4084	print warnmsg
4085	print ' ' + main + ": getting variables to compute already from file !!"
4086	origfiles, origtestfiles, allcompvar, Nvar = read_varcomp_file(varcompf)
4087
4088	# End of avoiding to repeat all the experiment search
4089
4090	print " For experiment '" + exp + "' is required to re-project:", Nvar, \
4091	"variables"
4092
4093	if dbg:
4094	print 'Variables to re-project _______'
4095	gen.printing_dictionary(allcompvar)
4096
4097	# Re-projection variables
4098	print " Reprojecting variables ..."
4099	os.chdir(odir)
4100
4101	# Keeping track of reprojection
4102	trckobjf = open(odir + '/all_reproj_computevars.inf','a')
4103
4104	for vnop in allcompvar.keys():
4105	vn = vnop.split('_')[0]
4106	op = vnop.split('_')[1]
4107	values = allcompvar[vnop]
4108	VnOpS = varnoper(vn, op, opersurnames)
4109
4110	# Only are reprojected that files which do contain 'lon' and 'lat',
4111	# otherways, it is only re-projected the original file
4112	inNO, iopNO, iop = gen.list_coincidences(opNOreproj,op.split('+'))
4113
4114	reprjop = gen.dictionary_key_list(REPRJops,vn).split('_')[1]
4115	if reprjop is None:
4116	print errmsg
4117	print ' ' + fname + ": no reprojecting operator found for " + \
4118	" variable '" + vn + "' !!"
4119	print ' available values _______'
4120	gen.printing_dictionary(REPRJops)
4121	quit(-1)
4122
4123	# Re-projecting also the 'original' (at least with 'pinterp') file for varsdif
4124	if op.find('pinterp') != -1:
4125	ifileorig = vn +'_'+ values[0].replace('reproj-','') + 'p_pinterp.nc'
4126	ofileorig = vn +'_'+ values[0] + 'p_pinterp.nc'
4127	ofilen = vn +'_'+ values[0] + 'p_pinterp.nc'
4128	ifileop = vn +'_'+ values[0].replace('reproj-','') + 'p_' + \
4129	op.replace('+','_') + '.nc'
4130	else:
4131	ifileorig = vn +'_'+ values[0].replace('reproj-','') + '.nc'
4132	ofileorig = vn +'_'+ values[0] + '.nc'
4133	ifileop = vn +'_'+ values[0].replace('reproj-','') + '_' + \
4134	op.replace('+','_') + '.nc'
4135
4136	ofileop = vn + '_' + values[0] + '_' + op.replace('+','_') + '.nc'
4137
4138	if fscr:
4139	sout = sub.call('rm ' + odir+'/'+ofileorig + ' >& /dev/null', shell=True)
4140	sout = sub.call('rm ' + odir+'/'+ofileop + ' >& /dev/null', shell=True)
4141
4142	# List of files to re-project (first without operation)
4143	ifiles = [odir+'/'+ifileorig, odir+'/'+ifileop]
4144	# List of output files after re-projection (first without operation)
4145	ofiles = [odir+'/'+ofileorig, odir+'/'+ofileop]
4146
4147	for iif in range(2):
4148	ifile = ifiles[iif]
4149	ofile = ofiles[iif]
4150
4151	# Only projecting original file-variable if there are not 'lon' and 'lat'
4152	doreproj = True
4153	if iif == 1 and len(inNO) > 0:
4154	doreproj = False
4155	if dbg:
4156	print warnmsg
4157	print ' ' + fname + ": operation '" + op + "' can not be " + \
4158	"reprojected due to the lack of 'lon' or 'lat' skipping it !!"
4159	print " NOTE: difference can only be computed as:"
4160	print " - 1st: compute variable's differences between models"
4161	print " - 2nd: Perform the operations"
4162
4163	# Building up resources to compute re-projected statistics
4164	if dbg:
4165	print ' '+fname+': computing from the original reprojected file'
4166	ModI = ModelInf(mod, mod, 'lon', 'lat', 'pres', 'time', 'depth', \
4167	'lon', 'lat', 'pres', 'time', 'depth', None)
4168	if type(values) == type([1]):
4169	fileh = values[0]
4170	else:
4171	fileh = values
4172
4173	if type(values) == type([1]):
4174	fileh = values[0]
4175	else:
4176	fileh = values
4177
4178	VarI = VariableInf(vn, values[0], values[2], values[3])
4179	usefs = {values[0]: ofileorig}
4180
4181	compute_statistics(ModI, config, config['ifold'], usefs, \
4182	config['ofold']+'/'+mod+'/'+exp,VarI,pinterp_var(op), op, fscr, dbg)
4183
4184	if not os.path.isfile(ofile) and doreproj:
4185	if debug and ofile == ofiles[0]:
4186	print " reprojecting '" + vn + "' '" + op + "' into '" + \
4187	ofile + "'"
4188
4189	# Reprojecting...
4190	if reprjop[0:6] == 'remap':
4191	ins = config['cdoHOME'] + '/cdo ' + reprjop + ',' + \
4192	config['RefProjfile'] + ' ' + ifile + ' ' + ofile
4193	try:
4194	with gen.Capturing() as output:
4195	sout = sub.call(ins, shell=True)
4196	except:
4197	print errmsg
4198	print ' ' + fname + ': $ ' + ins
4199	for s1out in output: print s1out
4200	print sout
4201	quit(-1)
4202	if sout != 0:
4203	print errmsg
4204	print ' ' + fname + ': $ ' + ins
4205	sub.call(ins, shell=True)
4206	quit(-1)
4207	else:
4208	# Name of the variable to reproject
4209	if iif == 0:
4210	reprjvn = vn
4211	else:
4212	reprjvn = VnOpS
4213
4214	reprojvalues = 'lon,lat,' + config['RefProjfile'] + ',lon,lat,'+ \
4215	reprjop + ',' + ':'.join(CFvardims)
4216	ins= 'ncvar.reproject('+reprojvalues + ', ' + ifile + ', '+ \
4217	reprjvn + ')'
4218	try:
4219	with gen.Capturing() as output:
4220	sout=ncvar.reproject(reprojvalues, ifile, reprjvn)
4221	except:
4222	print errmsg
4223	print ' ' + fname + ': ' + ins
4224	for s1out in output: print s1out
4225	quit(-1)
4226
4227	sub.call('mv reproject.nc ' + ofile, shell=True)
4228
4229	if debug:
4230	for s1out in output: print s1out
4231	print sout
4232
4233	# Keeping track of re-projection
4234	trckobjf.write('\n')
4235	if iif == 0:
4236	trckobjf.write('# ' + vn + '\n')
4237	else:
4238	trckobjf.write('# ' + vn + ' ' + op + '\n')
4239
4240	trckobjf.write(ins + '\n')
4241
4242	if reprjop[0:6] == 'remap':
4243	# CFing the reprojected file
4244	try:
4245	with gen.Capturing() as output:
4246	ncvar.CDO_toCF(ofile)
4247	except:
4248	print errmsg
4249	print 'ncvar.CDO_toCF(' + ofile + ')'
4250	for s1out in output: print s1out
4251	# Removing file in order to make sure that it will be redone
4252	print ' ' + fname + " removing file '" + ofile + "' ..."
4253	sub.call('rm ' + ofile + ' >& /dev/null', shell=True)
4254	quit(-1)
4255
4256	if debug:
4257	for s1out in output: print s1out
4258
4259	trckobjf.close()
4260
4261	if debug: print " End of '" + fname + "' "
4262
4263	return
4264
4265	def allmodexps_listconstruct(config, modexps, odir, debug):
4266	""" Function to create the list of all model-experiments to draw
4267	config= Configuration of the experiment
4268	modexps= list with the models-experiments pairs to use
4269	odir= output differences folder
4270	"""
4271	fname='allmodexps_listconstruct'
4272
4273	allplts = gen.get_specdictionary_HMT(config, H='PLOTALLMODEXP_')
4274
4275	if debug:
4276	if allplts is not None:
4277	print " 'all mod-exps' plots ________"
4278	gen.printing_dictionary(allplts)
4279
4280	# Getting Model-Reference Projection
4281	ModComProj = cnf['RefProj'].split(':')
4282
4283	allmodexp = {}
4284	Nallmodexp = 0
4285	for modexp in modexps:
4286	if debug:
4287	print " " + fname + ": '" + modexp + "' ..."
4288	mod = modexp.split('/')[0]
4289	exp = modexp.split('/')[1]
4290
4291	# Getting variable characteristics from each model/experiment
4292	idir = config['ofold'] + '/' + modexp
4293
4294	reproj = not gen.searchInlist(ModComProj,mod)
4295
4296	fvarcomp = 'varcompute.inf'
4297	if reproj: fvc = 'reproj_' + fvarcomp
4298	else: fvc = fvarcomp
4299
4300	files, testfiles, allcompvar, Nvar= read_varcomp_file(idir + '/' + fvc)
4301
4302	if allplts is not None:
4303	# Getting all model-experiments plots characteristics
4304
4305	for allplt in allplts.keys():
4306	pltn = allplt.split('_')[1]
4307	if debug and modexp == modexps[0]:
4308	print ' ' + fname + ": plot '" + pltn + "' "
4309	varops = allplts[allplt].split(':')
4310	for varop in varops:
4311	vn = varop.split('\|')[0]
4312	op = varop.split('\|')[1]
4313	vnop = vn + '_' + op
4314	if debug and modexp == modexps[0]:
4315	print ' ' + fname + ": varop '" + vnop + "' "
4316	if not allcompvar.has_key(vnop):
4317	print warnmsg
4318	print ' ' + fname + ": variable+operation '" + vnop + \
4319	"' in '" + modexp + "' was not computed !!"
4320	print ' skipping it!'
4321	print ' computed values:', allcompvar.keys()
4322	break
4323
4324	vals = allcompvar[vnop]
4325
4326	headerf = vals[0]
4327	if modexp == modexps[0]:
4328	allmodexp[vnop] = [modexp,headerf]
4329	Nallmodexp = Nallmodexp + 1
4330	else:
4331	prevals = allmodexp[vnop]
4332	prevals = prevals + [modexp,headerf]
4333	allmodexp[vnop] = prevals
4334
4335	else:
4336	Nallmodexp = 0
4337
4338	if allplts is not None:
4339	Sallmodexps = gen.dictKeysVals_stringList(allmodexp)
4340	else:
4341	Sallmodexps = 'none'
4342
4343	# Outwritting the all model-experiment plots file to avoid next time
4344	objf = open(odir + '/allmodexp.inf', 'w')
4345	objf.write('allmodexps: ' + Sallmodexps + '\n')
4346	objf.write('Nallmodexp: ' + str(Nallmodexp) + '\n')
4347	objf.close()
4348
4349	if debug: print " End of '" + fname + "' "
4350
4351	return allmodexp, Nallmodexp
4352
4353	def allmodexps_read(filen):
4354	""" Function to read the file with the information about the all model-experiment plots
4355	filen= file with the information
4356	"""
4357	fname = 'allmodexps_read'
4358
4359	if not os.path.isfile(filen):
4360	print errormsg
4361	print ' ' + fname + ": all model-experiment file '" + filen + \
4362	"' does not exist !!"
4363	quit(-1)
4364
4365	objf = open(filen, 'r')
4366	for line in objf:
4367	if line[0:1] != '#' and len(line) > 1:
4368	values = line.replace('\n','').split(' ')
4369	if values[0] == 'allmodexps:':
4370	allplts = gen.stringList_dictKeysVals(values[1])
4371	elif values[0] == 'Nallmodexp:':
4372	Nallmodexp = int(values[1])
4373
4374	objf.close()
4375
4376	return allplts, Nallmodexp
4377
4378	def draw_allmodexp_plots(config, allvarcomp, plots, modexp, odir, figscr, debug):
4379	""" Function to draw all model-experiment plots
4380	config= Configuration of the experiment
4381	allvarcomp = dictionary with the file headers for each variable
4382	plots= dictionary with the plots
4383	modexps= list with the names of the model-experiment pairs
4384	odir= output experiment folder
4385	figscr= whether figures should be done from the scratch or not
4386
4387	* Plot as
4388	{[kplot]} = [varn1]\|[op1]#[varn2]\|[op2]#[...[varnN]\|[opN]], ...
4389	[kplot] ___
4390	Nlines: Multiple lines with one line by model-experiment pairs
4391	[varn]
4392	variable
4393	[op]
4394	'+' separated list of operations
4395	in figures with more than 1 variable, use '#' to separate the [varn]\|[op]
4396	"""
4397	fname = 'draw_allmodexp_plots'
4398
4399	os.chdir(odir)
4400
4401	# Folder with the processed files
4402	infold = config['ofold']
4403
4404	# Dictionary with the operations with surnames for the operated variable
4405	opersurnames = {}
4406	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
4407	for opsr in opsur.keys():
4408	opn = opsr.split('_')[1]
4409	vls = opsur[opsr].split(':')
4410	opersurnames[opn] = vls
4411
4412	# time values
4413	# Units time for the plots
4414	rd = config['CFreftime']
4415	tunits = config['CFunitstime'] + '!since!' + rd[0:4] + '-' + rd[4:6] + '-' + \
4416	rd[6:8] + '!' + rd[8:10] + ':' + rd[10:12] + ':' + rd[12:14]
4417	# time ticks kind
4418	tkind = config['timekind']
4419	# time ticks format
4420	tfmt = config['timefmt']
4421	# time axis label
4422	tlab = config['timelabel']
4423	timevals = [tunits, tkind, tfmt, tlab]
4424
4425	# Dictionary of plot specificities
4426	specplotkeyn = 'specificvarplot'
4427	plotspecifics = {}
4428	if config.has_key(specplotkeyn):
4429	# [minval]: minimum value
4430	# [maxval]: minimum value
4431	# [colorbar]: name of the colorbar (from matplotlib) to use
4432	# [cntformat]: format of the contour labels
4433	# [colorcnt]: color for the countor lines
4434	plotspecs = config[specplotkeyn].split(':')
4435	for pltspc in plotspecs:
4436	pltvls = pltspc.split('\|')
4437	vn = pltvls[0]
4438	op = pltvls[1]
4439	fn = pltvls[2]
4440	plotspecifics[fn + '_' + vn + '_' + op] = pltvls[3:]
4441	if debug:
4442	print 'Specific values for plots _______'
4443	gen.printing_dictionary(plotspecifics)
4444
4445	# Graphical labels and configuration
4446	modgc, expgc = graphical_characteristics(config, debug)
4447
4448	# Kind of figures
4449	kindfigure = config['kindfig']
4450
4451	# Map value
4452	mapvalue = config['mapval']
4453
4454	# pythone scripts HOME
4455	pyHOME = config['pyHOME']
4456
4457	# Title-text of operations
4458	opexplained = {}
4459	optits = config['titleoperations'].split(':')
4460	for optit in optits:
4461	opn = optit.split('\|')[0]
4462	opt = optit.split('\|')[1]
4463	opexplained[opn] = opt
4464	if debug:
4465	print 'Titles for operations _______'
4466	gen.printing_dictionary(opexplained)
4467
4468	for kplot in plots.keys():
4469	varsplt = plots[kplot]
4470	for varplt in varsplt:
4471	if debug:
4472	print " printing '" + kplot + "' var: '" + varplt + "'..."
4473	varops = varplt.split('#')
4474
4475	# CF variables in plot
4476	CFvarsplot = []
4477	# Files in plot
4478	filesplot = []
4479	# Variables in plot within the files
4480	varsplot = []
4481	# Dims in figure
4482	dimsplot = []
4483	# pictoric values in figure
4484	pictplot = []
4485	# Name of the figure
4486	figname = ''
4487	# Title of the figure
4488	titfigure = ''
4489
4490	ivp = 0
4491	for varop in varops:
4492	vn = varop.split('\|')[0]
4493	op = varop.split('\|')[1]
4494
4495	# CF variables in plot
4496	CFvarsplot.append(vn)
4497
4498	if op.find('pinterp') != -1: gP = 'p'
4499	else: gP = ''
4500
4501	vnopS = vn + '_' + op
4502	if not allvarcomp.has_key(vnopS):
4503	print errmsg
4504	print ' ' + fname + ": no file for variable-operation '" + \
4505	vnopS + "' !!"
4506	modexpvals = allvarcomp[vnopS]
4507	Nmodexps = len(modexpvals)
4508
4509	for imodexp in range(0,Nmodexps,2):
4510	modexp = modexpvals[imodexp]
4511	headf = modexpvals[imodexp+1]
4512	if debug:
4513	print ' ' + fname + " model/experiment: '" + modexp + "'"
4514
4515	filen = infold + '/' + modexp + '/' + vn +'_'+ headf+gP + '_' + \
4516	op.replace('+','_') + '.nc'
4517
4518	filesplot.append(filen)
4519	# Do we have processed the given variable?
4520	if not os.path.isfile(filen):
4521	print warnmsg
4522	print " " + fname + ": there is no file for variable '" + \
4523	varop + "' skiping it !!"
4524	break
4525
4526	if imodexp == 0:
4527	# Name of the variable inside the file
4528	vnsur = varnoper(vn, op, opersurnames)
4529	varsplot.append(vnsur)
4530
4531	# Dimensions in file
4532	try:
4533	with gen.Capturing() as output:
4534	dims = ncvar.idims(filen)
4535	except:
4536	print errmsg
4537	print 'ncvar.idims('+filen+')'
4538	for s1out in output: print s1out
4539	quit(-1)
4540
4541	if imodexp == 0:
4542	# Dimensions in plot
4543	dimsplot.append(dims)
4544
4545	# Header of the name of the figure
4546	if ivp == 0:
4547	figname = kplot+'_'+vn+'_'+headf+'_'+ op.replace('+','-')
4548	else:
4549	figname = figname+'-'+vn+'_'+headf+'_'+op.replace('+','-')
4550
4551	# pictoric values for the figure
4552	Sfivaop = kplot + '_' + vn + '_' + op
4553	if plotspecifics.has_key(Sfivaop):
4554	pictvals = plotspecifics[Sfivaop]
4555	else:
4556	Vvals = gen.variables_values(vn)
4557	pictvals = [Vvals[2],Vvals[3],Vvals[6], '%g', 'fixc', 'black']
4558
4559	pictplot.append(pictvals)
4560
4561	ivp = ivp + 1
4562	# End of variable-operation
4563	figname = figname + '.' + kindfigure
4564
4565	# Title of figure
4566	titfigure = 'all!model-experiments!' + vn + optitle(op,opexplained)
4567
4568	draw_plot(kplot, CFvarsplot, filesplot, varsplot, dimsplot, pictplot, \
4569	figname, titfigure, kindfigure, mapvalue, timevals, expgc, odir, \
4570	pyHOME, figscr, debug)
4571
4572	# End of variables-operations
4573
4574	# End of kind of plots
4575
4576	if debug: print " End of '" + fname + "' "
4577
4578	return
4579
4580	def figinf_figname(figname,config):
4581	""" Function to retrieve figure information from its name
4582	figname= name of the figure
4583	config= configuration of `model_graphics.py'
4584	>>> figinf_figname('2linesTime_wss_wrfout_xvar-ymean-tas_wrfout_xvar-ymean.pdf', cnf)
4585	(2, '2linesTime', 'wss-tas', 'xvar-ymean@xvar-ymean')
4586	>>> figinf_figname('Nlines_tas_wrfout_tturb-xmean-tmean.pdf', cnf)
4587	(1, 'Nlines', 'tas', 'tturb-xmean-tmean')
4588	>>> figinf_figname('2lines_wss_diffvar_WRF-micro1_WRF-current_tturb-xmean-last-tas_tturb-xmean-last.pdf',cnf)
4589	(2, '2lines', 'wss-tas', 'tturb-xmean-last@tturb-xmean-last')
4590	"""
4591	fname = 'figinf_figname'
4592	plotsecs = figname.split('_')
4593	plotk = plotsecs[0]
4594
4595	if gen.searchInlist(config['pairfigures'].split(':'), plotk):
4596	Nvars = 2
4597	varn1 = plotsecs[1]
4598	if plotsecs[2] == 'diffvar' or plotsecs[2] == 'diffop':
4599	statn1varn2 = plotsecs[5].split('-')
4600	statn2 = plotsecs[6].split('.')[0]
4601	Lstatn1varn2 = len(statn1varn2)
4602	statn1 = '-'.join(statn1varn2[0:Lstatn1varn2-1])
4603	varn2 = statn1varn2[Lstatn1varn2-1]
4604	else:
4605	statn1varn2 = plotsecs[3].split('-')
4606	statn2 = plotsecs[5].split('.')[0]
4607	Lstatn1varn2 = len(statn1varn2)
4608	statn1 = '-'.join(statn1varn2[0:Lstatn1varn2-1])
4609	varn2 = statn1varn2[Lstatn1varn2-1]
4610	return Nvars, plotk, varn1+'-'+varn2, statn1+'@'+statn2
4611	elif gen.searchInlist(config['Nsourcesfigures'].split(':'), plotk):
4612	Nvars = 1
4613	varn = plotsecs[1]
4614	statn = plotsecs[3].split('.')[0]
4615	return Nvars, plotk, varn, statn
4616	else:
4617	print errmsg
4618	print ' ' + fname + ": kind of figure '" + plotk + "' not ready !!"
4619	print ' ready ones _______'
4620	print " 'pair_figures': figures of pair of variables: ", \
4621	config['pairfigures'].split(':')
4622	print " 'N-sources_figures': figures of pair of variables: ", \
4623	config['Nsourcesfigures'].split(':')
4624	quit(-1)
4625
4626	# Files with information about the configuration of the script
4627	inffiles = ['varcompute.inf', 'all_computevars.inf', 'all_statsvars.inf']
4628
4629	####### #######
4630	## MAIN
4631	#######
4632	if not os.path.isfile('model_graphics.dat'):
4633	print errmsg
4634	print main + ": configuration file 'model_graphics.dat' does not exist!!"
4635	quit(-1)
4636
4637	# Getting configuration from external ASCII file 'model_graphics.dat'
4638	cnf = gen.get_configuration('model_graphics.dat', False)
4639
4640	verify_configuration(cnf, gen.Str_Bool(cnf['debug']))
4641
4642	# scratches
4643	scratch, filescratch, figscratch, diffscratch, figdiffscratch, moddiffscratch, \
4644	figmoddiffscratch, figallmodexpscratch, addfiles, addfigures, adddiffs, \
4645	adddifffigures, addmoddiffs, addmoddifffigures, addallmodexpfigures, dbg = \
4646	scratches(cnf)
4647
4648	if dbg:
4649	print 'Experiment configuration scratches _______'
4650	print ' scratch:', scratch, 'filescratch:', filescratch,'figscratch:', figscratch
4651	print ' diffscratch:', diffscratch, 'figdiffscratch:', figdiffscratch
4652	print ' moddiffscratch:', moddiffscratch, 'figmoddiffscratch:', figmoddiffscratch
4653	print 'Experiment configuration adding _______'
4654	print ' addfiles:', addfiles, 'addfigures:', addfigures
4655	print ' adddiffs:', adddiffs, 'adddifffigures:', adddifffigures
4656	print ' addmoddiffs:', addmoddiffs, 'addmoddifffigures:', addmoddifffigures
4657
4658	# Getting models
4659	mods = cnf['models'].split(':')
4660
4661	# Getting graphical characeristics
4662	modGC, expGC = graphical_characteristics(cnf, dbg)
4663
4664	# Models loop
4665	##
4666
4667	# dictionary with the experiments of each model
4668	modexps = {}
4669
4670	for mod in mods:
4671	print mod
4672	if scratch:
4673	if cnf['ofold'] != cnf['ifold']:
4674	sout = sub.call('rm -rf '+cnf['ofold']+'/'+mod+' > /dev/null', shell=True)
4675	else:
4676	print warnmsg
4677	print ' ' + main + ": input '" + cnf['ifold'] + "' and output '" + \
4678	cnf['ofold'] + "' are the same folder !!"
4679	print " Keeping output folder although it has 'scratch' start"
4680
4681	# Get experiments and headers of model
4682	exps, fheaders = exp_headers(mod,cnf)
4683
4684	modexps[mod] = exps
4685	# Characteristics of the model
4686	Modinf = ncvar.model_characteristics(mod,'None','False')
4687	dnx = Modinf.dimxn
4688	dny = Modinf.dimyn
4689	dnz = Modinf.dimzn
4690	dnt = Modinf.dimtn
4691	dns = Modinf.dimsn
4692	vdnx = Modinf.vardxn
4693	vdny = Modinf.vardyn
4694	vdnz = Modinf.vardzn
4695	vdnt = Modinf.vardtn
4696	vdns = Modinf.vardsn
4697
4698	modgraphv = modGC[mod]
4699	Modinf = ModelInf(mod, mod, dnx, dny, dnz, dnt, dns, vdnx, vdny, vdnz, vdnt, \
4700	vdns, modgraphv.tmod)
4701
4702	if dbg:
4703	print ' model characteristics _______'
4704	gen.printing_class(Modinf)
4705
4706	moddims = dnx + ',' + dny + ',' + dnz + ',' + dnt
4707	modvdims = vdnx + ',' + vdny + ',' + vdnz + ',' + vdnt
4708
4709	# Experiments loop
4710	##
4711	for exp in exps:
4712	print ' ' + exp + '...'
4713
4714	# input folder
4715	iwdir = cnf['ifold'] + '/' + mod + '/' + exp
4716
4717	# Does input folder exist?
4718	if not os.path.isdir(iwdir):
4719	if filescratch*addfiles:
4720	print errmsg
4721	print " " + main + ": folder '" + iwdir + "' does not exist !!"
4722	quit(-1)
4723	else:
4724	print warnmsg
4725	print " " + main + ": folder '" + iwdir + "' does not exist !!"
4726	print " but start of files from scratch 'filescratch':", \
4727	filescratch, "and there is not the add of any file 'addfiles':", \
4728	addfiles
4729	print ' so, keep moving'
4730
4731	owdir = cnf['ofold'] + '/' + mod + '/' + exp
4732	sout = sub.call('mkdir -p ' + owdir, shell=True)
4733
4734	# Need to pass to analyze all the data?
4735	if filescratch:
4736	for inff in inffiles:
4737	if dbg: print " removing information file '" + inff + "' ..."
4738	ins = 'rm ' + owdir + '/' + inff + ' >& /dev/null'
4739	sout = sub.call(ins, shell=True)
4740
4741	objf = open(owdir+'/all_computevars.inf','w')
4742	objf.write("## Computation of variables \n")
4743	objf.close()
4744	objf = open(owdir+'/all_statsvars.inf','w')
4745	objf.write("## Computation of statistics \n")
4746	objf.close()
4747
4748	varcompf = owdir + '/varcompute.inf'
4749	if addfiles:
4750	if dbg:
4751	print ' ' + main + ": adding variables to compute removing " + \
4752	"file '" + varcompf + "' ..."
4753	sub.call('rm ' + varcompf +' >& /dev/null', shell=True)
4754
4755	if not os.path.isfile(varcompf):
4756	# Does input folder has header files?
4757	ih=1
4758	# Dictionary with the list of files for each headers
4759	files = {}
4760	# Dictionary with a file fromor each headers
4761	testfiles = {}
4762
4763	for fh in fheaders:
4764	if filescratch:
4765	ins = 'rm '+ owdir+'/_' + fh + '.nc >& /dev/null'
4766	sout = sub.call(ins, shell=True)
4767	files1h = gen.files_folder(iwdir,fh)
4768	if len(files1h) < 1:
4769	print errmsg
4770	print ' ' + main + ": folder '" + iwdir + "' does not " + \
4771	"contain files '" + fh + "*' !!"
4772	quit(-1)
4773	files[fh] = files1h
4774	testfiles[fh] = files1h[0]
4775
4776	if dbg:
4777	print ' Dictionary of files _______'
4778	gen.printing_dictionary(files)
4779
4780	print ' Dictionary of test-files _______'
4781	gen.printing_dictionary(testfiles)
4782
4783	allcompvar, Nvar = compvars_listconstruct(cnf, Modinf, files, testfiles, \
4784	iwdir, owdir, dbg)
4785	else:
4786	if dbg:
4787	print warnmsg
4788	print ' '+main+": getting variables to compute already from file !!"
4789	files, testfiles, allcompvar, Nvar = read_varcomp_file(varcompf)
4790
4791	# End of avoiding to repeat all the experiment search
4792
4793	print " For experiment '" + exp + "' is required to compute:", Nvar, \
4794	"variables"
4795
4796	if dbg:
4797	print 'Variables to compute _______'
4798	gen.printing_dictionary(allcompvar)
4799
4800	# Computing variables
4801	print " Computing variables ..."
4802	compute_vars(cnf, Modinf, iwdir, owdir, files, allcompvar, filescratch, dbg)
4803
4804	# Figures of variables direct from files
4805	print " Plotting direct figures ..."
4806	dirfigf = owdir + '/directplotsdraw.inf'
4807	if figscratch:
4808	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4809
4810	objf = open(owdir+'/all_figures.inf','w')
4811	objf.write("## Drawing of all figures \n")
4812	objf.close()
4813
4814	if addfigures:
4815	if dbg:
4816	print ' ' + main + ": adding direct figures removing file '" + \
4817	dirfigf + "' ..."
4818	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4819
4820	if not os.path.isfile(dirfigf):
4821	listplots, Nplt = plots_listconstruct(cnf, 'DIRPLT', dirfigf, owdir, dbg)
4822	else:
4823	if dbg:
4824	print warnmsg
4825	print ' ' + main + ": getting plots to draw already from file !!"
4826	listplots, Nplt = read_plot_file(dirfigf)
4827
4828	# End of avoiding to repeat all the plots search
4829
4830	print " For experiment '" + exp + "' is required to plot:", Nplt, "plots"
4831
4832	if dbg:
4833	print 'Plots to draw _______'
4834	gen.printing_dictionary(listplots)
4835
4836	draw_plots(cnf, listplots, mod, exp, owdir, allcompvar, figscratch, dbg)
4837
4838	# end of experiments loop
4839
4840	### ## #
4841	# Experiment differences
4842	## # ## #
4843
4844	# Experiments loop
4845	##
4846	print ' ***** ** * * ** * * ** * ** *****'
4847	print " ** '" + mod + "': Inter experiments differences "
4848	print ' ***** ** * * ** * * ** * ** *****'
4849	print ' experiments:', exps
4850	# There are two kind of differences:
4851	# DIFFop: differences between operations of each given variable.
4852	# Differences are computed directly from the last stage of the operation
4853	# DIFFvar: operations of differences between variables
4854	# First are computed the differences from the initial variable file
4855	# and then operations are made
4856	# NOTE: remember that: meanvar2 - meanvar1 = mean(var2 - var1)
4857	difffiles = ['diffop.inf', 'diffvar.inf']
4858
4859	Nexps = len(exps)
4860	for iexp1 in range(0,Nexps-1):
4861	exp1 = exps[iexp1]
4862	for iexp2 in range(iexp1+1,Nexps):
4863	exp2 = exps[iexp2]
4864	Sexps = exp2 + '-' + exp1
4865	print ' ' + Sexps + '...'
4866	owdir = cnf['ofold'] + '/' + mod + '/' + exp2 + '-' + exp1
4867	sout = sub.call('mkdir -p ' + owdir, shell=True)
4868
4869	# Removing files with list of differences if should be started from scratch or add differences
4870	for fdiff in difffiles:
4871	difff = owdir + '/' + fdiff
4872	if diffscratch:
4873	sub.call('rm -rf' + owdir +' >& /dev/null', shell=True)
4874	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4875	objf = open(owdir+'/all_' + fdiff,'w')
4876	objf.write("## Computation and drawing of differences " + \
4877	"between '" + exp2 + "'-'" + exp1 + "'\n")
4878	objf.close()
4879	difff = owdir + '/all_vardiffstatistics.inf'
4880	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4881	objf = open(owdir+'/all_' + fdiff,'w')
4882	objf.write("## Computation of all differences statistics " + \
4883	"between '" + exp2 + "'-'" + exp1 + "'\n")
4884	objf.close()
4885
4886	if adddiffs:
4887	if dbg:
4888	print ' ' + main + ": adding differences removing file '" + \
4889	difff + "' ..."
4890	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4891
4892	# op and var differences
4893	diffvarcompf = owdir + '/' + difffiles[0]
4894	if not os.path.isfile(diffvarcompf):
4895	alldiffop, alldiffvar,Nopdiffs,Nvardiffs=diffvarop_listconstruct(cnf,\
4896	[mod, mod], [exp1, exp2], [False, False], owdir, dbg)
4897	else:
4898	if dbg:
4899	print warnmsg
4900	print ' ' + main + ": getting 'op' and 'var' differences to " + \
4901	"compute already from file !!"
4902	alldiffop, Nopdiffops = read_diff_file(diffvarcompf)
4903	alldiffvar, Nopdiffvars = read_diff_file(owdir+'/'+difffiles[1])
4904	print 'Nopdiffops:', Nopdiffops,'Nopdiffvars:', Nopdiffvars
4905
4906	# End of avoiding to repeat all the experiment search
4907	print " For experiments '"+exp2+"'-'"+exp1+"' is required to " + \
4908	"compute:", Nvar, "differences"
4909
4910	if dbg:
4911	print 'Differences to compute _______'
4912	gen.printing_dictionary(alldiffop)
4913	gen.printing_dictionary(alldiffvar)
4914
4915	# Computing differences
4916	##
4917	print " Computing operation differences ..."
4918	compute_op_diffs(cnf, alldiffop, owdir, diffscratch, dbg)
4919	print " Computing variable differences ..."
4920	compute_var_diffs(cnf, alldiffvar, owdir, diffscratch, dbg)
4921
4922	# Plotting operation differences
4923	##
4924	print " " + main + ": Plotting operation differences' figures ..."
4925	dirfigf = owdir + '/diffopplotsdraw.inf'
4926	if figscratch:
4927	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4928
4929	objf = open(owdir+'/all_diffopfigures.inf','w')
4930	objf.write("## Drawing of all operation difference figures \n")
4931	objf.close()
4932
4933	if adddifffigures:
4934	if dbg:
4935	print ' ' + main + ": adding differences' figures removing " + \
4936	"file '" + dirfigf + "' ..."
4937
4938	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4939
4940	if not os.path.isfile(dirfigf):
4941	listplots, Nplt = plots_listconstruct(cnf, 'PLOTDIFFOP', dirfigf, \
4942	owdir, dbg)
4943	else:
4944	if dbg:
4945	print warnmsg
4946	print ' ' + main + ": getting plots to draw already from file !!"
4947	listplots, Nplt = read_plot_file(dirfigf)
4948
4949	# End of avoiding to repeat all the plots search
4950
4951	print " For experiment 'operation' differences '" + Sexps + "' is " + \
4952	"required to plot:" , Nplt, "plots"
4953
4954	if dbg:
4955	print ' Plots to draw _______'
4956	gen.printing_dictionary(listplots)
4957
4958	draw_diff_plots(cnf, listplots, owdir, alldiffop, 'diffop', \
4959	figdiffscratch, dbg)
4960
4961	# Plotting variable differences
4962	##
4963	print " " + main + ": Plotting variable differences' figures ..."
4964	dirfigf = owdir + '/diffvarplotsdraw.inf'
4965	if figscratch:
4966	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4967
4968	objf = open(owdir+'/all_diffvarfigures.inf','w')
4969	objf.write("## Drawing of all variables difference figures \n")
4970	objf.close()
4971
4972	if adddifffigures:
4973	if dbg:
4974	print ' '+main+": adding differences' figures removing file '" +\
4975	dirfigf + "' ..."
4976	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4977
4978	if not os.path.isfile(dirfigf):
4979	listplots, Nplt = plots_listconstruct(cnf, 'PLOTDIFFVAR', dirfigf, \
4980	owdir, dbg)
4981	else:
4982	if dbg:
4983	print warnmsg
4984	print ' ' + main + ": getting plots to draw already from file !!"
4985	listplots, Nplt = read_plot_file(dirfigf)
4986
4987	# End of avoiding to repeat all the plots search
4988
4989	print " For experiment 'variables' differences '" + Sexps + "' is " + \
4990	"required to plot:" , Nplt, "plots"
4991
4992	if dbg:
4993	print ' Plots to draw _______'
4994	gen.printing_dictionary(listplots)
4995
4996	draw_diff_plots(cnf, listplots, owdir, alldiffvar, 'diffvar', \
4997	figdiffscratch, dbg)
4998
4999	# end of mods loop
5000
5001	### ## #
5002	# Model differences
5003	### ## #
5004
5005	# Models already at the reference projection
5006	ModComProj = cnf['RefProj'].split(':')
5007	if dbg:
5008	print 'Models to use as reference for the projection:', ModComProj
5009
5010	# Operators to use for the reprojection of each variable
5011	REPRJvar0 = gen.get_specdictionary_HMT(cnf, H='reprojectvar_')
5012	REPRJvar = {}
5013	for Skey in REPRJvar0.keys():
5014	REPRJvar[Skey] = gen.str_list(REPRJvar0[Skey],':')
5015
5016	if dbg:
5017	print 'Remaping operators to use _______'
5018	gen.printing_dictionary(REPRJvar)
5019
5020	# Getting common base weights for grid point remapping with CDO
5021	frefgridn = 'RefProj.nc'
5022	for mod in mods:
5023	frefgriddes = cnf['ofold'] + '/' + mod + '/' + frefgridn
5024	if moddiffscratch:
5025	sout = sub.call('rm ' + frefgriddes + ' >& /dev/null', shell=True)
5026	if gen.searchInlist(ModComProj,mod) and not os.path.isfile(frefgriddes):
5027	# Looking for a file to use (all files are CF compilant, but we need to
5028	# be sure that at least they have 'lon', 'lat' variables)
5029	exps = modexps[mod]
5030	exp = exps[0]
5031	reprjfdir = cnf['ofold'] + '/' + mod + '/' + exp
5032	reprjfile = None
5033	for reprjop in REPRJvar.keys():
5034	reprjvars = REPRJvar[reprjop]
5035	for reprjvar in reprjvars:
5036	reprjfs= gen.files_folder_HMT(folder=reprjfdir,head=reprjvar+'_',tail='.nc')
5037	for reprjf in reprjfs:
5038	# Getting direct files as [var]_[fhead].nc
5039	if len(reprjf.split('_')) == 2:
5040	reprjfile = reprjfdir + '/' + reprjf
5041	reprjvn = reprjvar
5042	break
5043	if reprjfile is None:
5044	print errmsg
5045	print ' ' + main + ": no proper file to get projection information " + \
5046	"from '" + reprjfdir + "' has been found !!"
5047	quit(-1)
5048
5049	print " Creation of reference projection information file from '" + mod+ "'"
5050	if dbg:
5051	print " using file: '" + reprjfile
5052
5053	# Getting only 1 time-step for disk space issues
5054	try:
5055	with gen.Capturing() as output:
5056	ncvar.DataSetSection('time,0,1,1', reprjfile)
5057	except:
5058	print errmsg
5059	print 'ncvar.DataSetSection(time,0,1,1, ' + reprjfile + ')'
5060	for s1out in output: print s1out
5061	quit(-1)
5062
5063	of = reprjfile.split('.')[0] + '_time_B0-E1-I1.nc'
5064
5065	# Cleaning variables
5066	try:
5067	with gen.Capturing() as output:
5068	ncvar.selvar('lon@lon,lat@lat,time@time', of, reprjvn)
5069	except:
5070	print errmsg
5071	print 'ncvar.selvar(lon@lon,lat@lat, ' + of + ', ' + reprjvn + ')'
5072	for s1out in output: print s1out
5073	quit(-1)
5074	sout = sub.call('mv selvar_new.nc ' + frefgriddes, shell=True)
5075	sout = sub.call('rm ' + of + ' >& /dev/null', shell=True)
5076
5077	cnf['RefProjfile'] = frefgriddes
5078
5079	if dbg:
5080	for s1out in output: print s1out
5081	break
5082
5083	elif gen.searchInlist(ModComProj,mod) and os.path.isfile(frefgriddes):
5084	# Including reprojecting reference file in configuration dictionary
5085	cnf['RefProjfile'] = frefgriddes
5086
5087	Nmods = len(mods)
5088	for imod1 in range(0,Nmods-1):
5089	mod1 = mods[imod1]
5090	exps1 = modexps[mod1]
5091
5092	for imod2 in range(imod1+1,Nmods):
5093	mod2 = mods[imod2]
5094	exps2 = modexps[mod2]
5095
5096	Smods = mod2 + '-' + mod1
5097	print ' ***** ** * * ** * * ** * ** *****'
5098	print " ** '" + Smods + "': Inter models differences "
5099	print ' ***** ** * * ** * * ** * ** *****'
5100	print ' experiments mod1:', exps1
5101	print ' experiments mod2:', exps2
5102
5103	# Experiments loop
5104	##
5105	difffiles = ['diffop.inf', 'diffvar.inf']
5106
5107	Nexps1 = len(exps1)
5108	Nexps2 = len(exps2)
5109	for iexp1 in range(0,Nexps1):
5110	exp1 = exps1[iexp1]
5111	for iexp2 in range(0,Nexps2):
5112	exp2 = exps2[iexp2]
5113	Sexps = exp2 + '-' + exp1
5114	print ' ' + Sexps + '...'
5115	owdir = cnf['ofold'] + '/' + Smods + '/' + Sexps
5116	sout = sub.call('mkdir -p ' + owdir, shell=True)
5117
5118	# Getting the right projection in order to perform differences
5119	difmods = [mod1, mod2]
5120	difexps = [exp1, exp2]
5121	difreproj = [False, False]
5122
5123	for imod in range(2):
5124	mod = difmods[imod]
5125	exp = difexps[imod]
5126	if not gen.searchInlist(ModComProj,mod):
5127	print " Projecting files of model '" + mod + "' exp: '" + \
5128	exp + "' with file '" + cnf['RefProjfile'] + "'"
5129	reproject_modexp(mod, exp, cnf, REPRJvar, moddiffscratch, \
5130	addmoddiffs, dbg)
5131	difreproj[imod] = True
5132
5133	# Removing files with list of differences if should be started from
5134	# scratch or add differences
5135	for fdiff in difffiles:
5136	difff = owdir + '/' + fdiff
5137	if moddiffscratch:
5138	sub.call('rm -rf' + owdir +' >& /dev/null', shell=True)
5139	sub.call('rm ' + difff +' >& /dev/null', shell=True)
5140	objf = open(owdir+'/all_' + fdiff,'w')
5141	objf.write("## Computation and drawing of differences " + \
5142	"between '" + mod2+'@'+exp2 + "'-'" + mod1+'@'+exp1 + "'\n")
5143	objf.close()
5144	difff = owdir + '/all_vardiffstatistics.inf'
5145	sub.call('rm ' + difff +' >& /dev/null', shell=True)
5146	objf = open(owdir+'/all_' + fdiff,'w')
5147	objf.write("## Computation of all differences statistics " + \
5148	"between '" + mod2+'@'+exp2 + "'-'" + mod1+'@'+exp1 + "'\n")
5149	objf.close()
5150
5151	if addmoddiffs:
5152	if dbg:
5153	print ' ' + main + ": adding model differences " + \
5154	"removing file '" + difff + "' ..."
5155	sub.call('rm ' + difff +' >& /dev/null', shell=True)
5156
5157	# op and var differences
5158	diffvarcompf = owdir + '/' + difffiles[0]
5159	if not os.path.isfile(owdir+'/'+difffiles[0]) or \
5160	not os.path.isfile(owdir+'/'+difffiles[1]):
5161	alldiffop, alldiffvar, Nopdiffs, Nvardiffs = \
5162	diffvarop_listconstruct(cnf, difmods, difexps, difreproj,owdir,\
5163	dbg)
5164	else:
5165	if dbg:
5166	print warnmsg
5167	print ' '+main+": getting 'op' and 'var' differences to " + \
5168	"compute already from file !!"
5169	alldiffop, Nopdiffops = read_diff_file(diffvarcompf)
5170	alldiffvar, Nopdiffvars = read_diff_file(owdir+'/'+difffiles[1])
5171
5172	# End of avoiding to repeat all the experiment search
5173
5174	print " For experiments '" + mod2+'@'+exp2 + "'-'" + mod+'@'+exp1+\
5175	"' is required to compute:", Nvar, "differences"
5176
5177	if dbg:
5178	print ' op differences to compute _______'
5179	gen.printing_dictionary(alldiffop)
5180	print ' var differences to compute _______'
5181	gen.printing_dictionary(alldiffvar)
5182
5183	# Computing differences
5184	##
5185	print " Computing operation differences ..."
5186	compute_op_diffs(cnf, alldiffop, owdir, moddiffscratch, dbg)
5187	print " Computing variable differences ..."
5188	compute_var_diffs(cnf, alldiffvar, owdir, moddiffscratch, dbg)
5189
5190	# Plotting operation differences
5191	##
5192	print " " + main + ": Plotting operation differences' figures ..."
5193	dirfigf = owdir + '/diffopplotsdraw.inf'
5194	if figmoddiffscratch:
5195	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5196
5197	objf = open(owdir+'/all_diffopfigures.inf','w')
5198	objf.write("## Drawing of all operation difference figures \n")
5199	objf.close()
5200
5201	if addmoddifffigures:
5202	if dbg:
5203	print ' ' + main + ": adding model differences' figures " + \
5204	"removing file '" + dirfigf + "' ..."
5205	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5206
5207	if not os.path.isfile(dirfigf):
5208	listplots, Nplt = plots_listconstruct(cnf, 'PLOTDIFFOP', dirfigf,\
5209	owdir, dbg)
5210	else:
5211	if dbg:
5212	print warnmsg
5213	print ' '+main+": getting plots to draw already from file !!"
5214	listplots, Nplt = read_plot_file(dirfigf)
5215
5216	# End of avoiding to repeat all the plots search
5217
5218	print " For experiment 'operation' differences '" + Sexps + "' is "+\
5219	"required to plot:" , Nplt, "plots"
5220
5221	if dbg:
5222	print 'Plots to draw _______'
5223	gen.printing_dictionary(listplots)
5224
5225	draw_diff_plots(cnf, listplots, owdir, alldiffop, 'diffop', \
5226	figmoddiffscratch, dbg)
5227
5228	# Plotting variable differences
5229	##
5230	print " " + main + ": Plotting variable differences' figures ..."
5231	dirfigf = owdir + '/diffvarplotsdraw.inf'
5232	if figscratch:
5233	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5234
5235	objf = open(owdir+'/all_diffvarfigures.inf','w')
5236	objf.write("## Drawing of all variables difference figures \n")
5237	objf.close()
5238
5239	if adddifffigures:
5240	if dbg:
5241	print ' ' + main + ": adding differences's figures " + \
5242	"removing file '" + dirfigf + "' ..."
5243	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5244
5245	if not os.path.isfile(dirfigf):
5246	listplots, Nplt = plots_listconstruct(cnf,'PLOTDIFFVAR',dirfigf, \
5247	owdir, dbg)
5248	else:
5249	if dbg:
5250	print warnmsg
5251	print ' '+main+": getting plots to draw already from file !!"
5252	listplots, Nplt = read_plot_file(dirfigf)
5253
5254	# End of avoiding to repeat all the plots search
5255
5256	print " For experiment 'variables' differences '" + Sexps+ "' is "+ \
5257	"required to plot:" , Nplt, "plots"
5258
5259	if dbg:
5260	print ' Plots to draw _______'
5261	gen.printing_dictionary(listplots)
5262
5263	draw_diff_plots(cnf, listplots, owdir, alldiffvar, 'diffvar', \
5264	figdiffscratch, dbg)
5265
5266	# end of exp2 loop
5267	# end of exp1 loop
5268	# end of mod2 loop
5269	# end of mod1 loop
5270
5271	### ## #
5272	# All model/exps linear plots
5273	### ## #
5274
5275	allmodexps = []
5276	for mod in mods:
5277	for exp in modexps[mod]:
5278	allmodexps.append(mod + '/' + exp)
5279
5280	print ' ***** ** * * ** * * ** * ** *****'
5281	print ' ** Multi models, multi experiments plots '
5282	print ' ***** ** * * ** * * ** * ** *****'
5283	print ' Model/experiments:', allmodexps
5284
5285	owdir = cnf['ofold'] + '/allmodexps'
5286	sout = sub.call('mkdir -p ' + owdir, shell=True)
5287
5288	# Plotting all model-experiments lines
5289	##
5290	print " " + main + ": Plotting all model-experiments variable figures ..."
5291	allf = owdir + '/allmodexp.inf'
5292	dirfigf = owdir + '/allmodexpfigures.inf'
5293	allfigf = owdir+'/all_figures.inf'
5294	if figallmodexpscratch:
5295	sout = sub.call('rm ' + allf + ' >& /dev/null', shell=True)
5296	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5297	sout = sub.call('rm ' + allfigf + ' >& /dev/null', shell=True)
5298
5299	objf = open(allfigf,'w')
5300	objf.write("## Drawing of all variables figures for all model-experiments\n")
5301	objf.close()
5302
5303	if addallmodexpfigures:
5304	if dbg:
5305	print ' ' + main + ": adding model-experiment differences removing " + \
5306	" file '" + allfigf + "' ..."
5307	print ' ' + main + ": adding model-experiment differences' figures " + \
5308	"removing file '" + dirfigf + "' ..."
5309	sout = sub.call('rm ' + allfigf + ' >& /dev/null', shell=True)
5310	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5311
5312	if not os.path.isfile(dirfigf):
5313	listplots, Nplt = plots_listconstruct(cnf, 'PLOTALLMODEXP', dirfigf, owdir, dbg)
5314	else:
5315	if dbg:
5316	print warnmsg
5317	print ' ' + main + ": getting plots to draw already from file !!"
5318	listplots, Nplt = read_plot_file(dirfigf)
5319
5320	# End of avoiding to repeat all the plots search
5321
5322	print " Is required to plot:", Nplt, " all model-experiment plots"
5323
5324	if dbg:
5325	print ' Plots to draw _______'
5326	gen.printing_dictionary(listplots)
5327
5328	if not os.path.isfile(allf):
5329	# Constructing the information of modexp / variable
5330	allmodexpvar, Nallmodexpvar = allmodexps_listconstruct(cnf, allmodexps, owdir, dbg)
5331	else:
5332	allmodexpvar, Nallmodexpvar = allmodexps_read(allf)
5333
5334	draw_allmodexp_plots(cnf, allmodexpvar, listplots, modexps, owdir, figallmodexpscratch, dbg)
5335
5336	print main + ': all files and figures have been properly done !!'
5337
5338	# Creation of a pdf with all figures
5339	##
5340	###
5341	owdir = cnf['ofold']
5342	os.chdir(owdir)
5343
5344	texout ='model_graphics-images'
5345	print ' ***** ** * * ** * * ** * ** *****'
5346	print ' ** Creation of a pdf with all figures '
5347	print ' ***** ** * * ** * * ** * ** *****'
5348	print ' ' + owdir + '/' + texout + '.pdf'
5349
5350	if dbg:
5351	print main + ": generation of a pdf file '" + owdir + '/' + texout + \
5352	"' with all the figures..."
5353
5354	otexf = open(owdir + '/' + texout + '.tex', 'w')
5355	otexf.write('\\documentclass{article}\n')
5356	otexf.write('\\usepackage{graphicx}\n')
5357	otexf.write('\\usepackage[colorlinks=true,urlcolor=blue]{hyperref}\n')
5358	otexf.write('\\textheight=23cm\n')
5359	otexf.write('\\textwidth=18cm\n')
5360	otexf.write('\\oddsidemargin=-1cm\n')
5361	otexf.write('\\evensidemargin=-1cm\n')
5362	otexf.write('\\topmargin=-1cm\n')
5363
5364	otexf.write('\n')
5365	otexf.write('\\begin{document}\n')
5366	otexf.write('\n')
5367	otexf.write('\\def\\fontBloc{\\fontfamily{\\sfdefault}\\large\\bfseries}\n')
5368	otexf.write('\\def\\linia{\\setlength{\\unitlength}{1mm}\n')
5369	otexf.write('\\line(1,0){80}}\n')
5370	otexf.write('\\newcommand{\\modexp}[1]{\\clearpage\n')
5371	otexf.write('\\noindent\\linia\n')
5372	otexf.write('\\section{#1}\n')
5373	otexf.write('\\linia}\n')
5374	otexf.write('\n')
5375
5376	otexf.write('\\title{model\_graphics: ' + gen.latex_text(owdir) + '}\n')
5377	otexf.write('\\maketitle\n')
5378	otexf.write('\n')
5379	otexf.write('\\tableofcontents\n')
5380	otexf.write('\\listoffigures\n')
5381	otexf.write('\\clearpage\n')
5382
5383	Nmods = len(mods)
5384	lmodexps = []
5385	ldiffmodexps = []
5386	# Figures from mod/exp pairs
5387	for mod in mods:
5388	exps = modexps[mod]
5389	for exp in exps:
5390	print ' direct plots:', mod, exp
5391	lmodexps.append(mod+'/'+exp)
5392	owdirme = owdir + '/' + mod + '/' + exp
5393	otexf.write('% ' + mod + ' ' + exp + '\n')
5394	otexf.write('\\modexp{' + gen.latex_text(mod+' '+exp) + '}\n')
5395	listplots, Nplt = read_plot_file(owdirme + '/directplotsdraw.inf')
5396	for plot in listplots.keys():
5397	plots = listplots[plot]
5398	ops = []
5399	for plt in plots:
5400	opn = plt.split('#')[0].split('\|')[1]
5401	if not gen.searchInlist(ops,opn): ops.append(opn)
5402	for op in ops:
5403	opS = op.replace('+','_')
5404	modexpimg = gen.files_folder_HMT(owdirme, head=plot, middle=opS, \
5405	tail=cnf['kindfig'])
5406	Nimages = len(modexpimg)
5407	if Nimages > 0:
5408	for img in range(Nimages): modexpimg[img] = owdirme + '/' + \
5409	modexpimg[img]
5410	caption = mod + ' ' + exp + ' ' + plot + ' ' + op
5411	flab = 'fig:' + caption.replace(' ', '_')
5412	try:
5413	with gen.Capturing() as output:
5414	gen.latex_fig_array(modexpimg, otexf, \
5415	gen.latex_text(caption), flab, refsize=0.5, dist='sqr',\
5416	dorest='center')
5417	except:
5418	print errmsg
5419	print 'gen.latex_fig_array(',modexpimg,', '+otexf+', '+ \
5420	'gen.latex_text(' + caption + '), ' + flab + \
5421	", refsize=0.5, dist='sqr', dorest='center')"
5422	for s1out in output: print s1out
5423	quit(-1)
5424	otexf.write('%\\clearpage\n')
5425
5426	# differences between experiments
5427	Nexps = len(exps)
5428	for iexp1 in range(0,Nexps-1):
5429	exp1 = exps[iexp1]
5430	for iexp2 in range(iexp1+1,Nexps):
5431	exp2 = exps[iexp2]
5432	Sexps = exp2 + '-' + exp1
5433	owdirme = owdir + '/' + mod + '/' + Sexps
5434	print ' diffs:', mod + '/' + Sexps + ' ...'
5435	ldiffmodexps.append(mod + '/' + Sexps)
5436	for diffn in ['op', 'var']:
5437	fileplotinf = owdirme + '/diff' + diffn + 'plotsdraw.inf'
5438	if os.path.isfile(fileplotinf):
5439	otexf.write('% ' + mod + ' ' + Sexps + ' diff' + diffn + '\n')
5440	otexf.write('\\modexp{' + gen.latex_text(mod+' '+Sexps+' diff'+ \
5441	diffn)+ '}\n')
5442	listplots, Nplt = read_plot_file(fileplotinf)
5443	for plot in listplots.keys():
5444	plots = listplots[plot]
5445	ops = []
5446	for plt in plots:
5447	opn = plt.split('#')[0].split('\|')[1]
5448	if not gen.searchInlist(ops,opn): ops.append(opn)
5449	for op in ops:
5450	opS = op.replace('+','-')
5451	modexpimg = gen.files_folder_HMT(owdirme, head=plot, \
5452	middle=opS, tail=cnf['kindfig'])
5453	Nimages = len(modexpimg)
5454	if Nimages > 0:
5455	for img in range(Nimages): modexpimg[img] = owdirme +\
5456	'/' + modexpimg[img]
5457	caption = mod + ' ' + Sexps + ' ' + plot + ' ' + op +\
5458	' diff' + diffn
5459	flab = 'fig:' + caption.replace(' ', '_')
5460	try:
5461	with gen.Capturing() as output:
5462	gen.latex_fig_array(modexpimg, otexf, \
5463	gen.latex_text(caption), flab, refsize=0.5,\
5464	dist='sqr', dorest='center')
5465	except:
5466	print errmsg
5467	print 'gen.latex_fig_array(',modexpimg,', '+ \
5468	otexf+', gen.latex_text(' + caption + '), ' + \
5469	flab + ", refsize=0.5, dist='sqr', " + \
5470	"dorest='center')"
5471	for s1out in output: print s1out
5472	quit(-1)
5473
5474	otexf.write('%\\clearpage\n')
5475	print ' mods-exps diffs ...'
5476	for imod1 in range(0,Nmods-1):
5477	mod1 = mods[imod1]
5478	exps1 = modexps[mod1]
5479	for imod2 in range(imod1+1,Nmods):
5480	mod2 = mods[imod2]
5481	exps2 = modexps[mod2]
5482
5483	Smods = mod2 + '-' + mod1
5484	Nexps1 = len(exps1)
5485	Nexps2 = len(exps2)
5486	for iexp1 in range(0,Nexps1):
5487	exp1 = exps1[iexp1]
5488	for iexp2 in range(0,Nexps2):
5489	exp2 = exps2[iexp2]
5490	Sexps = exp2 + '-' + exp1
5491	print ' ' + Smods + ' ' + Sexps + ' ...'
5492	ldiffmodexps.append(Smods + '/' + Sexps)
5493	owdirme = owdir + '/' + Smods + '/' + Sexps
5494	for diffn in ['op', 'var']:
5495	fileplotinf = owdirme + '/diff' + diffn + 'plotsdraw.inf'
5496	if os.path.isfile(fileplotinf):
5497	otexf.write('% '+Smods + ' ' + Sexps + ' diff'+diffn+'\n')
5498	otexf.write('\\modexp{' + gen.latex_text(Smods+' '+Sexps+\
5499	' diff'+diffn) + '}\n')
5500	listplots, Nplt = read_plot_file(fileplotinf)
5501	for plot in listplots.keys():
5502	plots = listplots[plot]
5503	ops = []
5504	for plt in plots:
5505	opn = plt.split('#')[0].split('\|')[1]
5506	if not gen.searchInlist(ops,opn): ops.append(opn)
5507	for op in ops:
5508	opS = op.replace('+','-')
5509	modexpimg = gen.files_folder_HMT(owdirme, \
5510	head=plot, middle=opS, tail=cnf['kindfig'])
5511	Nimages = len(modexpimg)
5512	if Nimages > 0:
5513	for img in range(Nimages): modexpimg[img] = \
5514	owdirme + '/' + modexpimg[img]
5515	caption = Smods + ' ' + Sexps + ' ' + plot + \
5516	' ' + op + ' diff' + diffn
5517	flab = 'fig:' + caption.replace(' ', '_')
5518	try:
5519	with gen.Capturing() as output:
5520	gen.latex_fig_array(modexpimg, otexf,\
5521	gen.latex_text(caption), flab, \
5522	refsize=0.5, dist='sqr', \
5523	dorest='center')
5524	except:
5525	print errmsg
5526	print 'gen.latex_fig_array(',modexpimg, \
5527	', '+otexf+', gen.latex_text(' + \
5528	caption + '), '+flab+ ", refsize=0.5,"+\
5529	" dist='sqr', dorest='center')"
5530	for s1out in output: print s1out
5531	quit(-1)
5532	otexf.write('%\\clearpage\n')
5533
5534	# allmodexp
5535	owdirme = owdir + '/allmodexps'
5536	print ' allmodexps...'
5537	fileplotinf = owdirme + '/allmodexpfigures.inf'
5538	if os.path.isfile(fileplotinf):
5539	otexf.write('% allmodexps\n')
5540	otexf.write('\\modexp{allmodexps}\n')
5541	listplots, Nplt = read_plot_file(fileplotinf)
5542	for plot in listplots.keys():
5543	plots = listplots[plot]
5544	ops = []
5545	for plt in plots:
5546	opn = plt.split('\|')[1]
5547	if not gen.searchInlist(ops,opn): ops.append(opn)
5548	for op in ops:
5549	opS = op.replace('+','-')
5550	modexpimg = gen.files_folder_HMT(owdirme, head=plot, middle=opS, \
5551	tail=cnf['kindfig'])
5552	Nimages = len(modexpimg)
5553	if Nimages > 0:
5554	for img in range(Nimages): modexpimg[img] = owdirme + '/' + \
5555	modexpimg[img]
5556	caption = 'allmodexps ' + plot + ' ' + op + ' diff' + diffn
5557	flab = 'fig:' + caption.replace(' ', '_')
5558	try:
5559	with gen.Capturing() as output:
5560	gen.latex_fig_array(modexpimg,otexf, gen.latex_text(caption),\
5561	flab, refsize=0.9, dist='sqr', dorest='center')
5562	except:
5563	print errmsg
5564	print 'gen.latex_fig_array(',modexpimg,', '+otexf+ \
5565	', gen.latex_text(' + caption + '), ' +flab+ ", refsize=0.9," +\
5566	" dist='sqr', dorest='center')"
5567	for s1out in output: print s1out
5568	quit(-1)
5569
5570	otexf.write('%\\clearpage\n')
5571	#
5572	# Grouping by type of figure
5573	#
5574	Nmods = len(mods)
5575	# Figures from mod/exp pairs
5576	owsearch = owdir + '/' + mods[0] + '/' + modexps[mods[0]][0]
5577	print " Looking in '" + owsearch + "' for figures by kind and variable..."
5578	otexf.write('% figures by kind and variable from:' + owsearch + '\n')
5579	modexpimg = gen.files_folder_HMT(owsearch, tail=cnf['kindfig'])
5580	# dictionary with kind of plots
5581	dplots = {}
5582	# list with kind of plots
5583	lplots = []
5584	# dictionary with variable-statistics of each kind of plot
5585	dvarstats = {}
5586	# list with variables of each kind of plot
5587	lvars = []
5588	for plot in modexpimg:
5589	Nvars, plotk, varn, statn = figinf_figname(plot,cnf)
5590	if not gen.searchInlist(lplots,plotk):
5591	lplots.append(plotk)
5592	if not dplots.has_key(plotk+'_'+varn):
5593	if not gen.searchInlist(lvars,varn): lvars.append(varn)
5594	dplots[plotk+'_'+varn] = [statn]
5595	else:
5596	vals = dplots[plotk+'_'+varn]
5597	dplots[plotk+'_'+varn] = vals + [statn]
5598	else:
5599	if not dplots.has_key(plotk+'_'+varn):
5600	if not gen.searchInlist(lvars,varn): lvars.append(varn)
5601	dplots[plotk+'_'+varn] = [statn]
5602	else:
5603	vals = dplots[plotk+'_'+varn]
5604	dplots[plotk+'_'+varn] = vals + [statn]
5605
5606	# Figures found
5607	print ' direct figures to group:', modexpimg
5608	print ' plots:', lplots
5609	print ' variables:', lvars
5610
5611	for dplot in dplots.keys():
5612	plot = dplot.split('_')[0]
5613	var = dplot.split('_')[1]
5614	otexf.write('% ' + var + '\n')
5615	otexf.write('\\modexp{' + gen.latex_text(var) + '}\n')
5616	varn1 = var.split('-')[0]
5617	varn2 = var.split('-')[1]
5618	varn = varn1+'-'+varn2
5619	for stn12 in dplots[dplot]:
5620	stn1 = stn12.split('@')[0]
5621	stn2 = stn12.split('@')[1]
5622	stn = gen.lstring_values(stn12, '@',' & ')
5623	plotvarfigs = []
5624	caption = varn.replace('-',' & ') + ' ' + plot + ' ' + stn
5625	flab = 'fig:' + varn + '_' + plot + '_' + stn12 + '_allmodexp'
5626	for me in lmodexps:
5627	modexpn = expGC[me].label
5628	modn = me.split('/')[0]
5629	imgns = gen.files_folder_HMT(owdir+'/'+me, head=plot+'_'+varn1, \
5630	middle=stn1+'-'+varn2, tail=stn2+'.'+cnf['kindfig'])
5631	caption = caption + ', ' + modexpn
5632	imgn = owdir+'/'+me+'/'+imgns[0]
5633	if os.path.isfile(imgn):
5634	plotvarfigs.append(imgn)
5635	if dbg:
5636	print ' Grouping figures:', plotvarfigs
5637	try:
5638	with gen.Capturing() as output:
5639	gen.latex_fig_array(plotvarfigs, otexf, gen.latex_text(caption), \
5640	flab, refsize=0.9, dist='sqr', dorest='center')
5641	except:
5642	print errmsg
5643	print 'gen.latex_fig_array(',plotvarfigs,', '+otexf+', gen.latex_text(' +\
5644	caption + '), ' + flab + ", refsize=0.9, dist='sqr', dorest='center')"
5645	for s1out in output: print s1out
5646	quit(-1)
5647
5648	Nmods = len(mods)
5649	# Figures from mod/exp differences
5650	owsearch = owdir + '/' + ldiffmodexps[0]
5651	print " Looking in '" + owsearch + "' for difference figures by kind and variable..."
5652	otexf.write('% difference figures by kind and variable from:' + owsearch + '\n')
5653	modexpimg = gen.files_folder_HMT(owsearch, tail=cnf['kindfig'])
5654	# dictionary with kind of plots
5655	dplots = {}
5656	# list with kind of plots
5657	lplots = []
5658	# dictionary with variable-statistics of each kind of plot
5659	dvarstats = {}
5660	# list with variables of each kind of plot
5661	lvars = []
5662	for plot in modexpimg:
5663	Nvars, plotk, varn, statn = figinf_figname(plot,cnf)
5664	if not gen.searchInlist(lplots,plotk):
5665	lplots.append(plotk)
5666	if not dplots.has_key(plotk+'_'+varn):
5667	if not gen.searchInlist(lvars,varn): lvars.append(varn)
5668	dplots[plotk+'_'+varn] = [statn]
5669	else:
5670	vals = dplots[plotk+'_'+varn]
5671	dplots[plotk+'_'+varn] = vals + [statn]
5672	else:
5673	if not dplots.has_key(plotk+'_'+varn):
5674	if not gen.searchInlist(lvars,varn): lvars.append(varn)
5675	dplots[plotk+'_'+varn] = [statn]
5676	else:
5677	vals = dplots[plotk+'_'+varn]
5678	dplots[plotk+'_'+varn] = vals + [statn]
5679
5680	# Figures found
5681	print ' difference figures to group:', modexpimg
5682	print ' plots:', lplots
5683	print ' variables:', lvars
5684
5685	for dplot in dplots.keys():
5686	plot = dplot.split('_')[0]
5687	var = dplot.split('_')[1]
5688	otexf.write('% ' + var + '\n')
5689	otexf.write('\\modexp{diff ' + gen.latex_text(var) + '}\n')
5690	varn1 = var.split('-')[0]
5691	varn2 = var.split('-')[1]
5692	varn = varn1+'-'+varn2
5693	for stn12 in dplots[dplot]:
5694	stn1 = stn12.split('@')[0]
5695	stn2 = stn12.split('@')[1]
5696	stn = gen.lstring_values(stn12, '@',' & ')
5697	plotvarfigs = []
5698	caption = varn.replace('-',' & ') + ' ' + plot + ' ' + stn
5699	flab = 'fig:' + varn + '_' + plot + '_' + stn12 + '_alldiffmodexp'
5700	for me in ldiffmodexps:
5701	mods = me.split('/')[0]
5702	exps = me.split('/')[1]
5703	if exps.find('-') != -1:
5704	expn1 = exps.split('-')[0]
5705	expn2 = exps.split('-')[1]
5706	else:
5707	expn1 = exps
5708	expn2 = exps
5709
5710	if mods.find('-') != -1:
5711	modn1 = mods.split('-')[0]
5712	modn2 = mods.split('-')[1]
5713	else:
5714	modn1 = mods
5715	modn2 = mods
5716	modexpn1 = expGC[modn1 + '/' + expn1].label
5717	modexpn2 = expGC[modn2 + '/' + expn2].label
5718	modexpn = modexpn1 + '-' + modexpn2
5719
5720	modn = me.split('/')[0]
5721	imgns = gen.files_folder_HMT(owdir+'/'+me, head=plot+'_'+varn1, \
5722	middle=stn1+'-'+varn2, tail=stn2+'.'+cnf['kindfig'])
5723	caption = caption + ', ' + modexpn
5724	imgn = owdir+'/'+me+'/'+imgns[0]
5725	if os.path.isfile(imgn):
5726	plotvarfigs.append(imgn)
5727	if dbg:
5728	print ' Grouping figures:', plotvarfigs
5729	try:
5730	with gen.Capturing() as output:
5731	gen.latex_fig_array(plotvarfigs, otexf, gen.latex_text(caption), \
5732	flab, refsize=0.9, dist='sqr', dorest='center')
5733	except:
5734	print errmsg
5735	print 'gen.latex_fig_array(',plotvarfigs,', '+otexf+', gen.latex_text('+ \
5736	caption + '), ' + flab + ", refsize=0.9, dist='sqr', dorest='center')"
5737	for s1out in output: print s1out
5738	quit(-1)
5739
5740	otexf.write('\\end{document}\n')
5741	otexf.close()
5742
5743	try:
5744	with gen.Capturing() as output:
5745	ins = 'pdflatex ' + owdir + '/' + texout
5746	sout = sub.call(ins, shell=True)
5747	except:
5748	print errormsg
5749	print ' ' + ins
5750	print sout
5751	for s1out in output: print s1out
5752	quit(-1)
5753
5754	with gen.Capturing() as output:
5755	sout = sub.call(ins + '>& /dev/null', shell=True)
5756	with gen.Capturing() as output:
5757	sout = sub.call(ins + '>& /dev/null', shell=True)
5758	with gen.Capturing() as output:
5759	sout = sub.call(ins + '>& /dev/null', shell=True)
5760	with gen.Capturing() as output:
5761	sout = sub.call(ins, shell=True)
5762	if dbg:
5763	print sout
5764	for s1out in output: print s1out
5765
5766	sub.call('evince ' + owdir + '/' + texout + '.pdf &', shell=True)
5767

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: lmdz_wrf/trunk/tools/model_graphics.py @ 2074

Download in other formats: