Context Navigation

source: lmdz_wrf/trunk/tools/model_graphics.py @ 1625

Last change on this file since 1625 was 1623, checked in by lfita, 8 years ago
Adding in 'diagnostics.py' 'WRFtimes' as variable-dimt on 'WRF' model
File size: 217.3 KB

Line
1	# Python script to generate the files and the plots for a sensitivity study with multiple modle configurations and models
2	# L. Fita
3	# LMD, Jussieu/Palaisseau, France
4	# Script configuration get from ASCII file 'model_graphics.dat'
5
6	# `Wheeler-Kiladis' diagram: https://github.com/UV-CDAT/wk
7	# $ cd /home/lluis/Downloads
8	# $ git clone https://github.com/UV-CDAT/wk
9	# $ cd wk
10	# $ su
11	# # python setup.py build >& run_pysetup.log
12	# # python setup.py install >& run_pyinstall.log
13
14	import numpy as np
15	import nc_var_tools as ncvar
16	import generic_tools as gen
17	import drawing_tools as drw
18	import time as tim
19	# To avoid errors like: '/bin/sh: 1: Syntax error: Bad fd number'
20	# Make sure that /bin/sh directs to bash and not to dash:
21	# http://stackoverflow.com/questions/15809060/sh-syntax-error-bad-fd-number
22	import subprocess as sub
23	import os
24
25	main = 'model_graphics.py'
26
27	errmsg = ncvar.errormsg
28	warnmsg = ncvar.warnmsg
29
30	def verify_configuration(ExpConf, debug):
31	""" Function to verify mandatory keys from experiment configuration
32	ExpConf= dictionary with the configuration
33	"""
34	fname = 'verify_configuration'
35
36	# List with the mandatory keys
37	mandatorykeys = ['pyHOME', 'cdoHOME', 'scratch', 'filescratch', 'figscratch', \
38	'diffscratch', 'figdiffscratch', 'figallmodexpscratch', 'addfiles', \
39	'addfigures', 'adddiffs', 'adddifffigures', 'addallmodexpfigures', 'debug', \
40	'models', 'modgraphchar', 'expgraphchar', 'ifold', 'ofold', 'warnmsg', \
41	'errmsg', 'titleoperations', 'kindfig', 'CFreftime', 'CFunitstime', 'mapval', \
42	'timekind', 'timefmt', 'timelabel' ]
43
44	# List with the optional keys
45	optionalkeys = ['specificvarplot', 'specificdiffopplot', 'specificdiffvarplot', \
46	'RefProj']
47
48	# Dictionary with the mandatory keys which are required as function of a given
49	# key from the dictionary
50	# if [key] must exist [keyB] = [values]
51	ifkeys = {'WRFexps': ['models','WRF'], 'LMDZexps': ['models','LMDZ'], \
52	'WRF_LMDZexps': ['models','WRF_LMDZ'], 'DYNAMICOexps': ['models','DYNAMICO'], \
53	'mthDYNAMICOexps': ['models','mthDYNAMICO'], \
54	'WRFheaders': ['models','WRF'], 'WRF_LMDZheaders': ['models','WRF_LMDZ'], \
55	'LMDZheaders': ['models','LMDZ'], 'DYNAMICOheaders': ['models','DYNAMICO'], \
56	'mthDYNAMICOheaders': ['models','mthDYNAMICO']}
57
58	# Dictionary with the optional keys (at least one) which are required as function
59	# of a given key from the dictionary
60	# if [key] must exist at least one of the [keyA, keyB, keyC, ...] with values
61	ifopkeys = {'RefProj': ['reprojectvar_remapbil', 'reprojectvar_remapbic', \
62	'reprojectvar_remapdis', 'reprojectvar_remapnn', 'reprojectvar_remapcon', \
63	'reprojectvar_remapcon2', 'reprojectvar_remaplaf', 'reprojectvar_dis', \
64	'reprojecvar_dis']}
65
66	for kdict in mandatorykeys:
67	if not ExpConf.has_key(kdict):
68	print errmsg
69	print ' ' + fname + "' configuration without required '" + kdict + "' !!"
70	quit(-1)
71	if debug:
72	print ExpConf[kdict]
73
74	for kdict in ifkeys.keys():
75	vals = ifkeys[kdict]
76	keyn = vals[0]
77	keyv = vals[1]
78	if type(ExpConf[keyn]) == type('A'):
79	if not ExpConf.has_key(kdict) and ExpConf[keyn] == keyv:
80	print errmsg
81	print ' ' + fname + "' configuration without '" + kdict + \
82	"' when it is required with configuration '" + keyn + '=' + keyv + \
83	"' !!"
84	quit(-1)
85	elif type(ExpConf[keyn]) == type(['A', 'B']):
86	keyvals = ExpConf[keyn]
87	if not ExpConf.has_key(kdict) and gen.searchInlist(keyvals, keyv):
88	print errmsg
89	print ' ' + fname + "' configuration without '" + kdict + \
90	"' when it is required with configuration '" + keyn + '= [..., ' + \
91	keyv + ", ...]' !!"
92	quit(-1)
93	else:
94	print errmsg
95	print ' ' +fname+ 'dictionary type ', type(ExpConf[keyn]), ' not ready!!'
96	quit(-1)
97	if debug:
98	print ' Experiments configuration _______'
99	gen.printing_dictionary(ExpConf)
100
101	for kdict in ifopkeys.keys():
102	if ExpConf.has_key(kdict):
103	opkeys = ifopkeys[kdict]
104	hasopkey = False
105	opdictvals = {}
106	for opkv in opkeys:
107	if ExpConf.has_key(opkv):
108	hasopkey = True
109	opdictvals[opkv] = ExpConf[opkv]
110
111	if not hasopkey:
112	print errmsg
113	print ' ' + fname + "' configuration wit '" + kdict + "' but " + \
114	"without any key value:", opkeys, "' when it is required !!"
115	quit(-1)
116
117	if debug:
118	print " Optional key '" + kdict + "' with values ________"
119	gen.printing_dictionary(opdictvals)
120
121	print fname + ': configuration seems to be fine'
122
123	return
124
125	def scratches(config):
126	""" Function to set-up if it is needed to start from the scratch
127	config: dictionary with the configuration
128	"""
129	fname = 'scratches'
130
131	if gen.Str_Bool(config['scratch']):
132	scr = True
133	print warnmsg
134	print " " + fname + ": starting from the SCRATCH !!"
135	print " 10 seconds left!!"
136	filescr = True
137	figscr = True
138	difscr = True
139	figdifscr = True
140	moddifscr = True
141	figmoddifscr = True
142	figallmodexpscr = True
143	tim.sleep(10)
144	else:
145	scr = False
146	if gen.Str_Bool(config['filescratch']):
147	filescr = True
148	print warnmsg
149	print " " + main + ": files starting from the SCRATCH !!"
150	print " 5 seconds left!!"
151	tim.sleep(5)
152	else:
153	filescr = False
154
155	if gen.Str_Bool(config['figscratch']):
156	figscr = True
157	print warnmsg
158	print " " + main + ": figures starting from the SCRATCH !!"
159	print " 5 seconds left!!"
160	tim.sleep(5)
161	else:
162	figscr = False
163
164	if gen.Str_Bool(config['diffscratch']):
165	difscr = True
166	print warnmsg
167	print " " + main + ": experiment differences starting from the SCRATCH !!"
168	print " 5 seconds left!!"
169	tim.sleep(5)
170	else:
171	difscr = False
172
173	if gen.Str_Bool(config['figdiffscratch']):
174	figdifscr = True
175	print warnmsg
176	print " " + main + ": figures experiment differences starting from the SCRATCH !!"
177	print " 5 seconds left!!"
178	tim.sleep(5)
179	else:
180	figdifscr = False
181
182	if gen.Str_Bool(config['moddiffscratch']):
183	moddifscr = True
184	print warnmsg
185	print " " + main + ": model differences starting from the SCRATCH !!"
186	print " 5 seconds left!!"
187	tim.sleep(5)
188	else:
189	moddifscr = False
190
191	if gen.Str_Bool(config['figmoddiffscratch']):
192	figmoddifscr = True
193	print warnmsg
194	print " " + main + ": figures model differences starting from the SCRATCH !!"
195	print " 5 seconds left!!"
196	tim.sleep(5)
197	else:
198	figmoddifscr = False
199
200	if gen.Str_Bool(config['figallmodexpscratch']):
201	figallmodexpscr = True
202	print warnmsg
203	print " " + main + ": figures all model-experiment starting from the SCRATCH !!"
204	print " 5 seconds left!!"
205	tim.sleep(5)
206	else:
207	figallmodexpscr = False
208
209	if gen.Str_Bool(config['addall']):
210	print warnmsg
211	print " " + fname + ": adding new values everywhere !!"
212	addfils = True
213	addfigs = True
214	adddiffs = True
215	adddifffigs = True
216	addmoddiffs = True
217	addmoddifffigs = True
218	addallmodexpfigs = True
219	else:
220	addfils = gen.Str_Bool(config['addfiles'])
221	addfigs = gen.Str_Bool(config['addfigures'])
222	adddiffs = gen.Str_Bool(config['adddiffs'])
223	adddifffigs = gen.Str_Bool(config['adddifffigures'])
224	addmoddiffs = gen.Str_Bool(config['addmoddiffs'])
225	addmoddifffigs = gen.Str_Bool(config['addmoddifffigures'])
226	addallmodexpfigs = gen.Str_Bool(config['addallmodexpfigures'])
227
228	debug = gen.Str_Bool(config['debug'])
229
230	return scr, filescr, figscr, difscr, figdifscr, moddifscr, figmoddifscr, \
231	figallmodexpscr, addfils, addfigs, adddiffs, adddifffigs, addmoddiffs, \
232	addmoddifffigs, addallmodexpfigs, debug
233
234
235	class gc(object):
236	""" Class which holds all the graphical characteristics
237	[label]: label as it should appear in the graphics
238	[color]: specific color of the model
239	[linetype]: specific type of line of the model
240	[marker]: specific marker of the model
241	[sizes]: line width and point size for the model
242	[tmod]: time modification to apply to the model files (None for nothing)
243	'setorigin',[YYYYMMDDHHMISS]: re-set origin of times at [YYYYMMDDHHMISS]
244	"""
245	def __init__( self, label, color, linetype, marker, sizes, Tmod):
246	self.label = None
247	self.color = None
248	self.linetype = None
249	self.marker = None
250	self.sizes = None
251	self.tmod = None
252	if label is not None:
253	self.label = label
254	self.color = color
255	self.linetype = linetype
256	self.marker = marker
257	self.sizes = sizes
258	if Tmod != 'None':
259	self.tmod = Tmod
260
261	def graphical_characteristics(config, debug):
262	""" Function to provide the graphical characteristics of models and experiments
263	config= configuration of the experiment
264	"""
265	fname = 'graphical_characteristics'
266
267	modgraphchar = {}
268	modvals = config['modgraphchar'].split(':')
269	for modval in modvals:
270	modv = modval.split('\|')
271	modgraphchar[modv[0]] = gc(modv[1], modv[2], modv[3], modv[4], modv[5], modv[6])
272
273	expgraphchar = {}
274	expvals = config['expgraphchar'].split(':')
275	for expval in expvals:
276	expvs = expval.split('\|')
277	modexp = expvs[0] + '/' + expvs[1]
278	if not modgraphchar.has_key(expvs[0]):
279	print errmsg
280	print ' ' + fname + ": a model called '" +expvs[0]+ "' does not exist !!"
281	print ' existing models with graphic characteristics:', \
282	modgraphchar.keys()
283	quit(-1)
284
285	modvs = modgraphchar[expvs[0]]
286	expv = expvs[2:7]
287	if expv[1] == 'asmodel': expv[1] = modvs.color
288	if expv[2] == 'asmodel': expv[2] = modvs.linetype
289	if expv[3] == 'asmodel': expv[3] = modvs.marker
290	if expv[4] == 'asmodel': expv[4] = modvs.sizes
291
292	expgraphchar[modexp] = gc(expv[0], expv[1], expv[2], expv[3], expv[4], 'None')
293
294	# Running some tests
295	mods = config['models'].split(':')
296	for mod in mods:
297	if not modgraphchar.has_key(mod):
298	print errmsg
299	print ' ' + fname + ": model called '" + mod + "' does not have " + \
300	"graphical characteristics !!"
301	print ' existing models with graphic characteristics:', \
302	modgraphchar.keys()
303	quit(-1)
304
305	exps = config[mod + 'exps'].split(':')
306	for exp in exps:
307	modexp = mod + '/' + exp
308	if not expgraphchar.has_key(modexp):
309	print errmsg
310	print ' ' + fname + ": model/experiment called '" + modexp + \
311	"' does not have graphical characteristics !!"
312	print ' existing model/experiments with graphic ' + \
313	'characteristics:', expgraphchar.keys()
314	quit(-1)
315
316	if debug:
317	print ' ' + fname + ': model graphical characteristics _______'
318	for modn in modgraphchar.keys():
319	with gen.Capturing() as output:
320	gen.printing_class(modgraphchar[modn])
321	print "'" + modn + "'; " + ', '.join(output)
322	print ' ' + fname + ': experiment graphical characteristics _______'
323	for modexpn in expgraphchar.keys():
324	with gen.Capturing() as output:
325	gen.printing_class(expgraphchar[modexpn])
326	print "'" + modexpn + "'; " + ', '.join(output)
327
328	if debug: print " End of '" + fname + "' "
329
330	return modgraphchar, expgraphchar
331
332	def exp_headers(mod,config):
333	""" Function to provide the headers and the experiments of a given model
334	model= model
335	config= configuration of the experiment
336	"""
337	fname = 'exp_headers'
338
339	# No CASE in python!
340	# case ${mod} in ...
341	if mod == 'WRF':
342	expers = config['WRFexps'].split(':')
343	fhs = config['WRFheaders'].split(':')
344	elif mod == 'LMDZ':
345	expers = config['LMDZexps'].split(':')
346	fhs = config['LMDZheaders'].split(':')
347	elif mod == 'WRF_LMDZ':
348	expers = config['WRF_LMDZexps'].split(':')
349	fhs = config['WRF_LMDZheaders'].split(':')
350	elif mod == 'DYNAMICO':
351	expers = config['DYNAMICOexps'].split(':')
352	fhs = config['DYNAMICOheaders'].split(':')
353	elif mod == 'mthDYNAMICO':
354	expers = config['mthDYNAMICOexps'].split(':')
355	fhs = config['mthDYNAMICOheaders'].split(':')
356	else:
357	print errmsg
358	print " " + fname + ": model '" + mod + "' not ready!!"
359	quit(-1)
360
361	return expers, fhs
362
363	class VariableInf(object):
364	""" Class which holds all information of a given variable
365	name= CF name of the variabgle
366	header= header of the file from which it can be computed
367	model= file-header variable from which it can be directly computed
368	diag= file-header diagnostics from which it can be computed
369	"""
370	def __init__( self, name, fheader, model, diag):
371	self.name= None
372	self.fheader= None
373	self.model= None
374	self.diag= None
375	if name is not None:
376	self.name = name
377	self.fheader= fheader
378	self.model= model
379	self.diag= diag
380
381	class ModelInf(object):
382	""" Class which holds all information from a given model
383	name= Acronym of the model
384	model= Long description of the model
385	dim[x/y/z/t/s]n= name of the dimensions of the model
386	vdim[x/y/z/t/s]n= name of the variable-dimensions of the model
387	dimensions= list of the dimensions
388	vardimensions= list of the variable-dimensions
389	timemodif= time modification to apply to the model files ('None' for nothing)
390	'setorigin',[YYYYMMDDHHMISS]: re-set origin of times at [YYYYMMDDHHMISS]
391	"""
392	def __init__( self, name, model, dx, dy, dz, dt, ds, vdx, vdy, vdz, vdt, vds, \
393	tmod):
394	self.name= None
395	self.model= None
396	self.dimxn= None
397	self.dimyn= None
398	self.dimzn= None
399	self.dimtn= None
400	self.dimsn= None
401	self.vardxn= None
402	self.vardyn= None
403	self.vardzn= None
404	self.vardtn= None
405	self.vardsn= None
406	self.timemodif = None
407	if name is not None:
408	self.name = name
409	self.model= model
410	self.dimxn= dx
411	self.dimyn= dy
412	self.dimzn= dz
413	self.dimtn= dt
414	self.dimsn= ds
415	self.vardxn= vdx
416	self.vardyn= vdy
417	self.vardzn= vdz
418	self.vardtn= vdt
419	self.vardsn= vds
420	if tmod is not None:
421	self.timemodif = tmod
422	self.dimensions = [dx, dy, dz, dt, ds]
423	self.vardimensions = [vdx, vdy, vdz, vdt, vds]
424
425	def variable_compute(idir,var,ftests,db):
426	""" Function to retrieve the computation way of a given variable using a series of test files
427	iwdir= directory with the test files
428	var= name of the variable to test
429	filtests= dictionary with the test files for each header
430	"""
431	fname='variable_compute'
432
433	cancompute = True
434	for headerf in ftests.keys():
435	filen=ftests[headerf]
436	try:
437	with gen.Capturing() as output:
438	vmod, vdiag = ncvar.computevar_model(var, idir + '/' + filen)
439	except:
440	print errmsg
441	print 'ncvar.computevar_model(' + var + ', ' + idir + '/' + filen + ')'
442	for s1out in output: print s1out
443	quit(-1)
444
445	if vmod is None and vdiag is None:
446	cancompute = False
447	else:
448	cancompute = True
449	# Should be considered that variable can also be computed by both ways?
450	break
451
452	if not cancompute:
453	print warnmsg
454	print ' ' + fname + ": there is no way to compute '" + var + \
455	"' for model '" + mod
456	# Too extrict!
457	# quit(-1)
458
459	# Getting only the first possibility of model and diagnostic
460	if vmod is not None:
461	modV = vmod[0]
462	else:
463	modV = None
464	if vdiag is not None:
465	diagV = vdiag[0]
466	else:
467	diagV = None
468
469	varcomp = VariableInf(var, headerf, modV, diagV)
470
471	# a ';' list 'varcompute' it is created for each variable giving:
472	# [var]\|[vark]\|[headerf][varmod]\|[vardiag]
473	# This list will be used to compute a new file for each variable
474
475	if db:
476	print 'Variable information _______'
477	gen.printing_class(varcomp)
478
479	if db: print " End of '" + fname + "' "
480
481	return varcomp
482
483	def get_operations_var(vc,db):
484	""" Function to provide the operations to make for each variable from 'VAR_' dictionary
485	vc= 'VAR_' dictionary, dictionary with all the parameters started with 'VAR_'
486	"""
487	fname = 'get_operations_var'
488
489	LH = len('VAR_')
490	iop = 1
491	# list of operations
492	doopers = {}
493	# operations by variable
494	operationsvar = {}
495	# individual operations by variable (not repeated)
496	indivoperationsvar = {}
497
498	# Variables with a calculation whic requires vertical interpolation to all the
499	# file (operations stating by 'VAR_pinterp')
500	LVp = len('VAR_pinterp')
501	varglobalp = []
502
503	for oper in vc.keys():
504	Loper = len(oper)
505	opn = oper[LH:Loper+1]
506	vns = vc[oper].split(':')
507	ops1 = opn.split('+')
508	doopers[opn] = ops1
509	for vn in vns:
510	if not operationsvar.has_key(vn):
511	operationsvar[vn] = [opn]
512	indivoperationsvar[vn] = ops1
513	else:
514	opers = operationsvar[vn]
515	opers.append(opn)
516	operationsvar[vn] = opers
517	opers1 = indivoperationsvar[vn]
518	for op1 in ops1:
519	if not gen.searchInlist(opers1, op1):
520	opers1.append(op1)
521	indivoperationsvar[vn] = opers1
522
523	if oper[0:LVp] == 'VAR_pinterp':
524	for vn in vns:
525	if not gen.searchInlist(varglobalp,vn): varglobalp.append(vn)
526
527	iop = iop + 1
528
529	if db:
530	print ' operations to make _______'
531	gen.printing_dictionary(doopers)
532	print ' operations by variable _______'
533	gen.printing_dictionary(operationsvar)
534	print ' individual operations by variable _______'
535	gen.printing_dictionary(indivoperationsvar)
536	print ' variables with the vertical interpolation for all the file _______'
537	print ' #', varglobalp
538
539	if db: print " End of '" + fname + "' "
540
541	return doopers, operationsvar, indivoperationsvar, varglobalp
542
543	def pinterp_var(oper):
544	""" Function to retrieve characteristics of the vertical interpolation for the operation
545	oper= operation
546	Wether vertical interpolation is:
547	# 'global': for all file
548	# 'local': at a given step of the process
549	# 'none': no vertical interpolation
550	>>> pinterp_var('last+pinterp+xmean')
551	local
552	>>> pinterp_var('pinterp+tmean+xmean')
553	global
554	>>> pinterp_var('xmean')
555	none
556	"""
557	fname = 'pinterp_var'
558
559	pinn = 'pinterp'
560
561	Lpinterp = len(pinn)
562	if oper[0:Lpinterp] == pinn:
563	pkind = 'global'
564	return pkind
565
566	if oper.find(pinn) != -1:
567	pkind = 'local'
568	else:
569	pkind = 'none'
570
571	return pkind
572
573	def compvars_listconstruct(config, minf, Files, TestFiles, idir, odir, debug):
574	""" Function to construct the list of variables to compute
575	config= dictionary with the configuration of the execution
576	variables to compute are get with all values started by `VAR_' in 'module_graphics.dat', and then
577	providing a consecutive number of calculations separated by '+'
578	VAR_[calc1]+[calc2] = tas:wss
579	will compute first [calc1] and then [calc2] for 'tas' and 'wss'
580	minf= class with information about the model
581	Files= dictionary of files for heach header
582	TestFiles= dictionary of files to test how to compute the variable
583	odir= output directory
584	"""
585	fname='compvars_listconstruct'
586
587	varcomp = gen.get_specdictionary_HMT(config,H='VAR_')
588
589	if debug:
590	print ' variables to compute ________'
591	gen.printing_dictionary(varcomp)
592
593	# Getting operations by variable
594	opers, varoper, indivaroper, varglobp = get_operations_var(varcomp,debug)
595
596	strfiles = gen.dictKeysVals_stringList(Files)
597	strtestfiles = gen.dictKeysVals_stringList(TestFiles)
598
599	Svarcompute = ''
600
601	# Main dictionary with all the variables and their calculation
602	allvarcomp = {}
603
604	ivop = 0
605	for vn in varoper:
606	vcomp = variable_compute(idir,vn,TestFiles,False)
607	for op in varoper[vn]:
608	# Creation of a String as: [CFname]\|[operation]\|[fheader]\|[vmodel]\|[vdiag]\|[globalP]
609	# [CFname]: CF-name of the variable (must appear in 'variables_values.dat')
610	# [operation]: operation to compute
611	# [fheader]: header of file with the required variables
612	# [vmodel]: Variable from model output
613	# [vdiag]: Varible as diagnostic from different variables of the model output
614	# [globalP]: Wether vertical interpolation is:
615	# 'global': for all file
616	# 'local': at a given step of the process
617	# 'none': no vertical interpolation
618	globalP = pinterp_var(op)
619	if vcomp.model is not None:
620	if type(vcomp.model) == type(list([1, 2])):
621	Smodel = ':'.join(vcomp.model)
622	elif type(vcomp.model) == type('A'):
623	Smodel = vcomp.model
624	else:
625	Smodel = 'None'
626	if vcomp.diag is not None:
627	if type(vcomp.diag) == type(list([1, 2])):
628	Sdiag = ':'.join(vcomp.diag)
629	elif type(vcomp.diag) == type('A'):
630	Sdiag = vcomp.diag
631	else:
632	Sdiag = 'None'
633
634	# To get a combination of operations, all the precedent steps are kept
635	# thus, they are also available !
636	seqops = op.split('+')
637	for seqop in seqops:
638	if seqop == seqops[0]: seqopS=seqop
639	else: seqopS = seqopS + '+' + seqop
640	allvarcomp[vn+'_'+seqopS] = [vcomp.fheader, vcomp.model, vcomp.diag, \
641	globalP]
642
643	Svc = vcomp.name + '\|' + seqopS + '\|' + vcomp.fheader + '\|' + \
644	Smodel + '\|' + Sdiag + '\|' + globalP
645	if ivop == 0:
646	Svarcompute = Svc
647	else:
648	Svarcompute = Svarcompute + ',' + Svc
649
650	ivop = ivop + 1
651
652	if debug:
653	print ' Variables to compute _______'
654	gen.printing_dictionary(allvarcomp)
655
656	# Outwritting the varcompute to avoid next time (if it is not filescratch!)
657	objf = open(odir + '/varcompute.inf', 'w')
658	objf.write('files: ' + strfiles + '\n')
659	objf.write('testfiles: ' + strtestfiles + '\n')
660	objf.write('varcompute: ' + Svarcompute + '\n')
661	objf.write('itotv: ' + str(ivop) + '\n')
662	objf.close()
663
664	if debug: print " End of '" + fname + "' "
665
666	return allvarcomp, ivop
667
668	def read_varcomp_file(filen):
669	""" Function to read 'varcompute.inf' and reconstruct the dictionaries
670	"""
671	fname = 'read_varcomp_file'
672
673	allvarcomp = {}
674
675	objf = open(filen, 'r')
676
677	for line in objf:
678	vals = line.replace('\n','').split(' ')
679	if vals[0] == 'files:':
680	Files = gen.stringList_dictKeysVals(vals[1])
681	elif vals[0] == 'testfiles:':
682	TestFiles = gen.stringList_dictKeysVals(vals[1])
683	elif vals[0] == 'varcompute:':
684	for VvalsS in vals[1].split(','):
685
686	Vvals = VvalsS.split('\|')
687	if Vvals[3] == 'None':
688	mod = None
689	else:
690	mod = Vvals[3].split(':')
691	if Vvals[4] == 'None':
692	diag = None
693	else:
694	diag = Vvals[4].split(':')
695
696	allvarcomp[Vvals[0]+'_'+Vvals[1]] = [Vvals[2], mod, diag, Vvals[5]]
697
698	# # To get a combination of operations, all the precedent steps are kept
699	# # thus, they are also available !
700	# seqops = Vvals[1].split('+')
701	# for seqop in seqops:
702	# if seqop == seqops[0]: seqopS=seqop
703	# else: seqopS = seqopS + '+' + seqop
704	# allvarcomp[Vvals[0]+'_'+seqopS] = [Vvals[2], mod, diag, Vvals[5]]
705	elif vals[0] == 'itotv:':
706	ivop = int(vals[1])
707
708	objf.close()
709
710	return Files, TestFiles, allvarcomp, ivop
711
712	def pinterpS(gP):
713	""" For that variables which require vertical interpolation 'p' suffix to the file header is added
714	gP= kind of pinterp: 'global', 'local', 'none'
715	"""
716	fname = 'pinterpS'
717
718	if gP != 'none':
719	gPS = 'p'
720	else:
721	gPS = ''
722
723	return gPS
724
725	def compute_variable(minf, idir, usefiles, odir, cvar, gP, scr, pyH, Tref, Tunits, db):
726	""" Function to compute a variable
727	minf= class with the information of the model
728	idir= directory with the input files
729	usefiles= dictionary of files as dict([headerf]) = [file1], ..., [fileN]
730	odir= directory to write the output files
731	cvar= class with the information of the variable: 'name', 'fheader', 'varmod', 'vardiag'
732	gP= kind of vertical interpolation ('global', 'local', 'none')
733	scr= should it be done from the scratch?
734	pyH= location of the python HOME
735	Tref= CF time reference
736	Tunits= CF time units
737	"""
738	fname='compute_variable'
739
740	CFvarn=cvar.name
741	headerf = cvar.fheader
742	modvar = cvar.model
743	diagvar = cvar.diag
744
745	cfiles = usefiles[headerf]
746
747	# dimensions
748	dnx = minf.dimxn
749	dny = minf.dimyn
750	# var-dimensions
751	vdnx = minf.vardxn
752	vdny = minf.vardyn
753
754	# Computing separately and then joinging for all files
755	Ntotfiles = len(cfiles)
756	Nzeros = len(str(Ntotfiles))
757	NStot = str(Ntotfiles).zfill(Nzeros)
758	# For that variables which require vertical interpolation 'p' suffix to the
759	# file header is added
760	SgP = pinterpS(gP)
761
762	# Getting in working dir
763	os.chdir(odir)
764
765	# File to keep track of all operations
766	otrackf = open( odir + '/all_computevars.inf', 'a')
767
768	# Computing variable
769	ifile=1
770	for cf in cfiles:
771	ifS = str(ifile).zfill(Nzeros)
772	ifilen = odir + '/' + CFvarn + '_' + headerf + SgP + '_' + ifS + '-' + \
773	NStot + '.nc'
774	fileon = odir + '/' + CFvarn + '_' + headerf + SgP + '.nc'
775
776	if scr:
777	sout = sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
778	sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
779
780	if not os.path.isfile(ifilen) and not os.path.isfile(fileon):
781	# Since model direct values are retrieved from `variables_valules.dat' which was
782	# initially coincived as a way to only give variable attributes, range and color
783	# bars, if a variable has a diagnostic way to be computed, the later one will be
784	# preferred
785	if db:
786	print ' ' + fname + ": creation of variable file '" + CFvarn + \
787	"' in file '" + ifilen + "' ..."
788	print ' model variable:', modvar
789	print ' diagostic variable:', diagvar
790
791	if diagvar is None:
792	# model variable
793	if db: print ' '+fname+ ": variable direct from model '" +modvar+ "'"
794	values = modvar + ',0,-1,-1'
795	vs = modvar + ',' + vdnx + ',' + vdny + ',' + vdnz + ',' + vdnt
796
797	try:
798	with gen.Capturing() as output:
799	ncvar.DataSetSection_multivars(values, idir + '/' + cf, vs)
800	except:
801	print errmsg
802	print 'ncvar.DataSetSection_multivars('+values+', '+idir+'/'+cf+ \
803	', '+vs+')'
804	for s1out in output: print s1out
805	quit(-1)
806
807	newfile, loc = gen.search_sec_list(output,'succesfull')
808	ofile = newfile[0].split(' ')[7]
809	sub.call('mv ' + ofile + ' ' + ifilen, shell=True)
810
811	# Keeping track of the operations
812	pyins = pyH + "/nc_var.py -f " + idir + '/' + cf + \
813	" -o DataSetSection_multivars -v " + vs + " -S '" + values + "'"
814	otrackf.write('\n')
815	otrackf.write('# ' + CFvarn + " " + modvar + '\n')
816	otrackf.write('python ' + pyins + '\n')
817
818	# CF renaming of variable
819	ncvar.chvarname(CFvarn,ifilen,modvar)
820	else:
821	# diagnostic variable
822	if db: print ' '+fname+ ": variable as diagnostic '" +diagvar[0]+ "'"
823	if minf.name == 'WRF':
824	dims = dnt+'@WRFtime,'+dnz+'@'+vdnz+','+dny+'@'+vdny+','+dnx+ \
825	'@'+vdnx
826	else:
827	dims = dnt+'@'+vdnt+','+dnz+'@'+vdnz+','+dny+'@'+vdny+','+dnx+ \
828	'@'+vdnx
829	diagn = diagvar[0]
830	Ndiagvars = len(diagvar)
831	diagc = '@'.join(diagvar[1:])
832
833	values = '-f ' + idir + '/' + cf + " -d '" + dims + "' -v '" + \
834	diagn + '\|' + diagc + "'"
835	try:
836	with gen.Capturing() as output:
837	sout = sub.call('python ' +pyH+ '/diagnostics.py ' + values, \
838	shell=True)
839	except:
840	print errmsg
841	print 'python ' + pyH + '/diagnostics.py ' + values
842	for s1out in output: print s1out
843	quit(-1)
844	if db:
845	for s1out in output: print s1out
846
847	sout = sub.call(' mv diagnostics.nc ' + ifilen, shell=True)
848
849	# Keeping track of the operations
850	pyins = 'python ' + pyH + '/diagnostics.py ' + values
851	otrackf.write('\n')
852	otrackf.write('# ' + CFvarn + " " + diagn + '\n')
853	otrackf.write(pyins + '\n')
854
855	# Attaching necessary variables for the pressure interpolation
856	if gP != 'none':
857	if minf.name == 'WRF' or minf.name == 'WRF_LMDZ':
858	requiredinterpvars = ['P', 'PB', 'PSFC', 'PH', 'PHB', 'HGT', 'T',\
859	'QVAPOR', 'XLONG', 'XLAT', 'Times']
860	elif minf.name == 'LMDZ':
861	requiredinterpvars = ['pres', 'psol', 'geop', 'phis', 'temp', \
862	'ovap', 'lon', 'lat', 'time_counter']
863	elif minf.name == 'DYNAMICO':
864	requiredinterpvars = ['pres', 'psol', 'geop', 'phis', 'temp', \
865	'ovap', 'lon', 'lat', 'time_counter']
866	elif minf.name == 'mthDYNAMICO':
867	requiredinterpvars = ['pres', 'psol', 'geop', 'phis', 'temp', \
868	'ovap', 'lon', 'lat', 'time_centered']
869	else:
870	print errmsg
871	print fname + ": for model '" + minf.name + "' required " + \
872	" variables for vertical interpolation are not known !!"
873	quit(-1)
874
875	print " " + fname + ": adding variables:", requiredinterpvars, \
876	' to allow pressure interpolation'
877	for rqv in requiredinterpvars:
878	try:
879	with gen.Capturing() as output:
880	ncvar.fvaradd(idir+'/'+cf+','+rqv,ifilen)
881	except:
882	print errmsg
883	print 'fvaradd('+idir+'/'+cf+','+rqv+', '+ifilen+')'
884	for s1out in output: print s1out
885	quit(-1)
886	if db:
887	for s1out in output: print s1out
888
889	# CFification of files
890	if minf.name == 'WRF' or minf.name == 'WRF_LMDZ':
891	values = vdnx + ':' + vdny + ':'+ Tref + ':' + Tunits
892	try:
893	with gen.Capturing() as output:
894	ncvar.WRF_toCF(values, ifilen)
895	except:
896	print errmsg
897	print 'WRF_toCF('+ values +', ' + ifilen + ')'
898	for s1out in output: print s1out
899	# Removing file in order to make sure that it will be redone
900	print ' ' + fname + " removing file '" + ifilen + "' ..."
901	sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
902	quit(-1)
903	if db:
904	for s1out in output: print s1out
905	elif minf.name == 'LMDZ':
906	try:
907	with gen.Capturing() as output:
908	ncvar.LMDZ_toCF(Tunits + '!since!' + Tref, ifilen)
909	except:
910	print errmsg
911	print 'LMDZ_toCF('+ Tunits +'!since!'+ Tref + ', ' + ifilen + ')'
912	for s1out in output: print s1out
913	# Removing file in order to make sure that it will be redone
914	print ' ' + fname + " removing file '" + ifilen + "' ..."
915	sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
916	quit(-1)
917	if db:
918	for s1out in output: print s1out
919	# At this stage, let's be simple. It might be necessary to complicate it! Since
920	# DYNAMICO might have 'time_counter' or 'time_instant'
921	elif minf.name == 'DYNAMICO':
922	try:
923	with gen.Capturing() as output:
924	ncvar.DYNAMICO_toCF(Tunits + '!since!' + Tref, ifilen)
925	except:
926	print errmsg
927	print 'DYNAMICO_toCF('+Tunits+'!since!'+Tref+ ', ' + ifilen + ')'
928	for s1out in output: print s1out
929	# Removing file in order to make sure that it will be redone
930	print ' ' + fname + " removing file '" + ifilen + "' ..."
931	sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
932	quit(-1)
933	if db:
934	for s1out in output: print s1out
935	elif minf.name == 'mthDYNAMICO':
936	try:
937	with gen.Capturing() as output:
938	ncvar.mthDYNAMICO_toCF(Tunits + '!since!' + Tref, ifilen)
939	except:
940	print errmsg
941	print 'mthDYNAMICO_toCF('+Tunits+'!since!'+Tref+ ', ' + ifilen + ')'
942	for s1out in output: print s1out
943	# Removing file in order to make sure that it will be redone
944	print ' ' + fname + " removing file '" + ifilen + "' ..."
945	sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
946	quit(-1)
947	if db:
948	for s1out in output: print s1out
949	else:
950	print errmsg
951	print ' ' + fname + ": no CFification for model '" +minf.name+ "' !!"
952	print " available ones: 'WRF', 'WRF_LMDZ', 'LMDZ', 'DYNAMICO'" + \
953	", 'mthDYNAMICO'"
954	quit(-1)
955
956	ifile = ifile + 1
957
958	otrackf.close()
959
960	# Joining variable files
961	if not os.path.isfile(fileon):
962	if Ntotfiles > 1:
963	if db:
964	print ' ' + fname + ": concatenating all variable files to " + \
965	"create '" + fileon + "..."
966	try:
967	with gen.Capturing() as output:
968	ncvar.netcdf_fold_concatenation_HMT('./,time', CFvarn + '_' + \
969	headerf + SgP +'_,-,.nc', 'all')
970	except:
971	print errmsg
972	print 'netcdf_fold_concatenation_HMT(./,time, ' + CFvarn + '_' + \
973	headerf +SgP + '_,-,.nc, all)'
974	for s1out in output: print s1out
975	quit(-1)
976	if db:
977	for s1out in output: print s1out
978
979	sout = sub.call('mv netcdf_fold_concatenated_HMT.nc '+fileon, shell=True)
980	if os.path.isfile(fileon):
981	sout = sub.call('rm '+CFvarn + '_' + headerf + SgP + '_-.nc >& ' + \
982	'/dev/null', shell=True)
983	else:
984	sout = sub.call('mv ' + CFvarn + '_' + headerf + SgP + '_1-1.nc ' + \
985	fileon, shell=True)
986
987	# Re-setting time of final concatenated file
988	if minf.timemodif is not None:
989	if db:
990	print ' ' + fname + ": Modifying times of the file by '" + \
991	minf.timemodif + "' !!"
992	try:
993	with gen.Capturing() as output:
994	ncvar.time_reset(minf.timemodif, fileon,'time')
995	except:
996	print errmsg
997	print 'time_reset(' + minf.timemodif + ', ' + fileon + ', time)'
998	for s1out in output: print s1out
999	# Removing file in order to make sure that it will be redone
1000	print ' ' + fname + " removing file '" + fileon + "' ..."
1001	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1002	quit(-1)
1003	if db:
1004	for s1out in output: print s1out
1005
1006	if db: print " End of '" + fname + "' "
1007
1008	return
1009
1010	def compute_statistics(minf, config, idir, usefiles, odir, cvar, gP, Opers, scr, db):
1011	""" Function to compute different statistics it will take previous steps if they
1012	are availale
1013	minf= class with the information of the model
1014	config= dictionary with the configuration of the experiment
1015	idir= directory with the input files
1016	usefiles= ',' list of files to use [file1],...,[fileN]
1017	odir= directory to write the output files
1018	cvar= class with the information of the variable: 'name', 'fheader', 'varmod', 'vardiag'
1019	gP= kind of vertical interpolation ('global', 'local', 'none')
1020	Opers= kind of operation: (as possible multiple consecutive combination of operations separated by '+'
1021	[calc1]+[calc2] will compute first [calc1] and then [calc2]
1022	acc: temporal accumulated values
1023	diff: differences between models
1024	direct: no statistics
1025	last: last temporal value
1026	Lmean: latitudinal mean values
1027	Lsec: latitudinal section (latitudinal value must be given, [var]@[lat])
1028	lmean: longitudinal mean values
1029	lsec: longitudinal section (longitudinal value must be given, [var]@[lat])
1030	pinterp: pressure interpolation (to the given $plevels)
1031	smean: spatial-weighted mean values
1032	tmean: temporal mean values
1033	tstd: temporal standard deviation values
1034	tturb: Taylor's turbulence decomposition value (x - <x>) for time
1035	tvar: temporal variance values
1036	xmean: x-axis mean values
1037	xvar: x-axis variance values
1038	ymean: y-axis mean values
1039	zsum: vertical aggregated values
1040	scr= should it be done from the scratch?
1041	"""
1042	fname='compute_statistics'
1043
1044	# Getting a previous file name to continue with(for combinations)
1045	CFvarn=cvar.name
1046	headerf = cvar.fheader
1047	modvar = cvar.model
1048	diagvar = cvar.diag
1049
1050	cfiles = usefiles[headerf]
1051
1052	# dimensions
1053	dnx = minf.dimxn
1054	dny = minf.dimyn
1055	# var-dimensions
1056	vdnx = minf.vardxn
1057	vdny = minf.vardyn
1058	# Model dimension-variable dictionary
1059	Modeldimvardict = {dnx: vdnx, dny: vdny, dnz: vdnz, dnt: vdnt}
1060
1061	# Some experiment configuration values
1062	# plevels= ':' separated list of pressures (in Pa) to use for the vertical interpolation
1063	plevels = config['plevels']
1064	# pyH= location of the python HOME
1065	pyH = config['pyHOME']
1066	# Tref= CF time reference
1067	Tref = config['CFreftime']
1068	# Tunits= CF time units
1069	Tunits = config['CFunitstime']
1070	# opsur = operation surnames
1071	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
1072	Opsurs = {}
1073	for opk in opsur:
1074	opn = opk.split('_')[1]
1075	vals = opsur[opk]
1076	Opsurs[opn] = vals.split(':')
1077	# if db:
1078	# print ' ' + fname + ' operation surnames _______'
1079	# gen.printing_dictionary(Opsurs)
1080
1081	# Getting in working dir
1082	os.chdir(odir)
1083
1084	# File to keep track of all operations
1085	otrackf = open( odir + '/all_statsvars.inf', 'a')
1086
1087	# For that variables which require vertical interpolation 'p' suffix to the file
1088	# header is added.
1089	# After some operations some CF dimension-variables might not be kept
1090
1091	if gP != 'none':
1092	SgP = 'p'
1093	varnCFs = ['lon', 'lat', 'pres', 'time']
1094	else:
1095	SgP = ''
1096	varnCFs = ['lon', 'lat', 'time']
1097
1098	# CF dimension-variable dictionary
1099	CFdimvardict = {'lon': 'lon', 'lat': 'lat', 'pres': 'pres', 'time': 'time'}
1100
1101	# Input file
1102	ifilen = odir + '/' + CFvarn + '_' + headerf + SgP + '.nc'
1103
1104	# Computing variable statisitcs
1105	istats=0
1106
1107	# List of not computed statistics
1108	wrongstats = []
1109
1110	opers = Opers.split('+')
1111	Fopers = ''
1112	if db: print " computing statistics of variable:'", CFvarn, "' operation '" + \
1113	Opers + "' using file '" + ifilen + "...'"
1114	for op in opers:
1115
1116	# File name from previous operations, and name of the variable within the
1117	# file (some operations change the name of the variable)
1118	if op == opers[0]:
1119	Fopers = op
1120	prevfile = ifilen
1121	vninF = CFvarn
1122	else:
1123	Fopers = Fopers + '_' + op
1124	prevfile = fileon
1125	fileon = odir + '/' + CFvarn + '_' + headerf + SgP + '_' + Fopers + '.nc'
1126
1127	# Adding required variables for the vertical interpolation in all operations
1128	SvarnCFs = ',' + ','.join(varnCFs)
1129
1130	if gP != 'none':
1131	if gP == 'local':
1132	CFvarnp = vninF + ',P,PB,PSFC,PH,PHB,HGT,T,QVAPOR,XLONG,XLAT,Times'+ \
1133	SvarnCFs
1134	else:
1135	CFvarnp = vninF + SvarnCFs
1136	else:
1137	CFvarnp = vninF + SvarnCFs
1138
1139	if scr:
1140	sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1141
1142	hprevfile = prevfile[0:len(prevfile)-3]
1143
1144	if db: print ' ', op, "using '" + prevfile + "' "
1145	# Just produce file in case output file does not exist
1146	if not os.path.isfile(fileon):
1147	if db:
1148	print ' ' + fname + ": creation of file '" + fileon + "' ..."
1149	dofile = True
1150	else:
1151	dofile = False
1152
1153	# Variables to be kept in the final file
1154	varkeep = []
1155
1156	if op == 'acc':
1157	# temporal accumulated values
1158	print " " + fname + ": kind '" + op + "' not ready !!"
1159	wrongstats.append(CFvarn + '_' + opers)
1160	break
1161
1162	elif op[0:11] == 'dimvarvalue':
1163	# Slicing along the index at the nearest given value of a dimension-variable
1164	dimvarn = op.split('~')[1]
1165	dimvalue = op.split('~')[2]
1166	vals = dimvarn + ',' + dimvalue + ',' + dimvalue + ',0'
1167	if dofile:
1168	try:
1169	with gen.Capturing() as output:
1170	ncvar.DataSetSection_multivars(vals,prevfile,CFvarnp)
1171	except:
1172	print errmsg
1173	print 'DataSetSection_multivars('+vals+',', prevfile, ',' + \
1174	CFvarnp + ')'
1175	for s1out in output: print s1out
1176	quit(-1)
1177
1178	if db:
1179	for s1out in output: print s1out
1180
1181	sout = sub.call('mv DataSetSection_multivars.nc '+fileon, shell=True)
1182
1183	# Keeping the operations
1184	pyins=pyH + "/nc_var.py -o DataSetSection_multivars -S '" + vals + \
1185	"' -f " + prevfile + " -v " + CFvarnp
1186	otrackf.write("\n")
1187	otrackf.write("# " + CFvarnp + " " + Fopers + "\n")
1188	otrackf.write(pyins + "\n")
1189
1190	# removing the given variable-dimension
1191	varnCFs.remove(dimvarn)
1192
1193	elif op == 'direct':
1194	# no statistics
1195	if dofile:
1196	sout = sub.call('mv ' + prevfile + ' ' + fileon, shell=True)
1197	elif op == 'last':
1198	# last temporal value
1199	vals='time,-9,0,0'
1200	if dofile:
1201	try:
1202	with gen.Capturing() as output:
1203	ncvar.DataSetSection(vals,prevfile)
1204	except:
1205	print errmsg
1206	print 'DataSetSection('+vals+',', prevfile, ')'
1207	for s1out in output: print s1out
1208	quit(-1)
1209
1210	if db:
1211	for s1out in output: print s1out
1212
1213	sout = sub.call('mv ' + hprevfile + '_time_B-9-E0-I0.nc ' + fileon, \
1214	shell=True)
1215
1216	# Keeping the operations
1217	pyins=pyH + "/nc_var.py -o DataSetSection -S '" + vals + "' -f " + \
1218	prevfile
1219	otrackf.write("\n")
1220	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1221	otrackf.write(pyins + "\n")
1222
1223	elif op =='Lmean':
1224	# latitudinal mean values
1225	print " " + fname + ": kind '" + op + "' not ready !!"
1226	wrongstats.append(CFvarn + '_' + opers)
1227	break
1228	elif op =='Lsec':
1229	# latitudinal section (latitudinal value must be given, [var]@[lat])
1230	print " " + fname + ": kind '" + op + "' not ready !!"
1231	wrongstats.append(CFvarn + '_' + opers)
1232	break
1233	elif op =='lmean':
1234	# longitudinal mean values
1235	print " " + fname + ": kind '" + op + "' not ready !!"
1236	wrongstats.append(CFvarn + '_' + opers)
1237	break
1238	elif op =='lsec':
1239	# longitudinal section (longitudinal value must be given, [var]@[lon])
1240	print " " + fname + ": kind '" + op + "' not ready !!"
1241	wrongstats.append(CFvarn + '_' + opers)
1242	break
1243	elif op == 'pinterp':
1244	# pinterp: pressure interpolation (to the given $plevels)
1245	vals=plevels + ',1,1'
1246	if dofile:
1247	try:
1248	with gen.Capturing() as output:
1249	ncvar.pinterp(vals,prevfile,vninF)
1250	except:
1251	print errmsg
1252	print 'pinterp('+vals+',', prevfile, ','+vninF+')'
1253	for s1out in output: print s1out
1254	ncvar.pinterp(vals,prevfile,vninF)
1255	quit(-1)
1256
1257	if db:
1258	for s1out in output: print s1out
1259
1260	sout = sub.call('mv pinterp.nc ' + fileon, shell=True)
1261
1262	# Keeping the operations
1263	pyins=pyH + "/nc_var.py -o pinterp -S '" + vals + "' -f " + \
1264	prevfile + "-v " + vninF
1265	otrackf.write("\n")
1266	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1267	otrackf.write(pyins + "\n")
1268
1269	# adding CF lon,lat,time in WRF files
1270	if minf.name == 'WRF' or minf.name == 'WRF_LMDZ':
1271	values = vdnx + ':' + vdny + ':'+ Tref + ':' + Tunits
1272	if db:
1273	print " CFification of '" + fileon + "' with:", values, "..."
1274	try:
1275	with gen.Capturing() as output:
1276	ncvar.WRF_toCF(values, fileon)
1277	except:
1278	print errmsg
1279	print 'ncvar.WRF_toCF(' + values+ ', ' + fileon + ')'
1280	for s1out in output: print s1out
1281	# Removing file in order to make sure that it will be redone
1282	print ' ' + fname + " removing file '" + fileon + "' ..."
1283	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1284	quit(-1)
1285
1286	# vertical interpolation variables are no more needed
1287	CFvarnp = vninF
1288	gP = 'none'
1289
1290	elif op == 'smean':
1291	# spatial-weighted mean values
1292	noLlvdims = list(varnCFs)
1293	noLlvdims.remove('lon')
1294	noLlvdims.remove('lat')
1295	Saddvars = ':'.join(noLlvdims)
1296	if len(Saddvars) > 1:
1297	vals = 'reglonlat,lon,lat,lon,lat,' + Saddvars
1298	else:
1299	vals = 'reglonlat,lon,lat,lon,lat,None'
1300
1301	if dofile:
1302	try:
1303	with gen.Capturing() as output:
1304	ncvar.SpatialWeightedMean(vals,prevfile,vninF)
1305	except:
1306	print errmsg
1307	print 'SpatialWeightedMean('+vals+',', prevfile, ','+vninF+')'
1308	for s1out in output: print s1out
1309	quit(-1)
1310
1311	if db:
1312	for s1out in output: print s1out
1313
1314	# renaming variable
1315	ncvar.chvarname(vninF+'mean', 'SpatialWeightedMean_reglonlat.nc', \
1316	vninF+'spaceweightmean')
1317	sout = sub.call('mv SpatialWeightedMean_reglonlat.nc '+fileon, \
1318	shell=True)
1319
1320	# Keeping the operations
1321	pyins=pyH + "/nc_var.py -o SpatialWeightedMean -S '" + vals + \
1322	"' -f " + prevfile + " -v " + vninF
1323	otrackf.write("\n")
1324	otrackf.write("# " + vninF + " " + Fopers + "\n")
1325	otrackf.write(pyins + "\n")
1326
1327	# removing dimension variable-dimension 'lon', 'lat'
1328	varnCFs.remove('lon')
1329	varnCFs.remove('lat')
1330
1331	varkeep.append(vninF + 'mean')
1332
1333	elif op == 'tmean':
1334	# temporal mean values
1335	vals='time\|-1,time,mean,' + ':'.join(varnCFs) + ':' + vdnz
1336	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1337	if dofile:
1338	try:
1339	with gen.Capturing() as output:
1340	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1341	except:
1342	print errmsg
1343	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1344	for s1out in output: print s1out
1345	quit(-1)
1346
1347	if db:
1348	for s1out in output: print s1out
1349
1350	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
1351
1352	# Keeping the operations
1353	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1354	"' -f " + prevfile + " -v " + CFvarnp
1355	otrackf.write("\n")
1356	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1357	otrackf.write(pyins + "\n")
1358
1359	# removing dimension variable-dimension 'time'
1360	varnCFs.remove('time')
1361
1362	varkeep.append('timestats')
1363
1364	elif op == 'tstd':
1365	# temporal standard deviation values
1366	vals='time\|-1,time,std,' + ':'.join(varnCFs) + ':' + vdnz
1367	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1368	if dofile:
1369	try:
1370	with gen.Capturing() as output:
1371	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1372	except:
1373	print errmsg
1374	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1375	for s1out in output: print s1out
1376	quit(-1)
1377
1378	if db:
1379	for s1out in output: print s1out
1380
1381	sout = sub.call('mv file_oper_alongdims_std.nc '+fileon, shell=True)
1382
1383	# Keeping the operations
1384	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1385	"' -f " + prevfile + " -v " + CFvarnp
1386	otrackf.write("\n")
1387	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1388	otrackf.write(pyins + "\n")
1389
1390	# removing dimension variable-dimension 'time'
1391	varnCFs.remove('time')
1392
1393	varkeep.append('timestats')
1394
1395	elif op == 'tturb':
1396	# temporal turbulent values
1397	vals='time\|-1,time,turb,' + ':'.join(varnCFs) + ':' + vdnz
1398	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1399	if dofile:
1400	try:
1401	with gen.Capturing() as output:
1402	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1403	except:
1404	print errmsg
1405	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1406	for s1out in output: print s1out
1407	quit(-1)
1408
1409	if db:
1410	for s1out in output: print s1out
1411
1412	sout = sub.call('mv file_oper_alongdims_turb.nc '+fileon, shell=True)
1413
1414	# Keeping the operations
1415	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1416	"' -f " + prevfile + " -v " + CFvarnp
1417	otrackf.write("\n")
1418	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1419	otrackf.write(pyins + "\n")
1420
1421	# removing dimension variable-dimension 'time'
1422	varkeep.append('timestats')
1423
1424	elif op == 'tvar':
1425	# temporal variance values (Taylor's turbulence)
1426	vals='time\|-1,time,var,' + ':'.join(varnCFs) + ':' + vdnz
1427	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1428	if dofile:
1429	try:
1430	with gen.Capturing() as output:
1431	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1432	except:
1433	print errmsg
1434	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1435	for s1out in output: print s1out
1436	quit(-1)
1437
1438	if db:
1439	for s1out in output: print s1out
1440
1441	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
1442
1443	# Keeping the operations
1444	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1445	"' -f " + prevfile + " -v " + CFvarnp
1446	otrackf.write("\n")
1447	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1448	otrackf.write(pyins + "\n")
1449
1450	# removing dimension variable-dimension 'time'
1451	varnCFs.remove('time')
1452
1453	varkeep.append('timestats')
1454
1455	elif op == 'xmean':
1456	# x-axis mean values
1457	vals='lon\|-1,lon,mean,' + ':'.join(varnCFs) + ':' + vdnz
1458	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1459	if dofile:
1460	try:
1461	with gen.Capturing() as output:
1462	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1463	except:
1464	print errmsg
1465	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1466	for s1out in output: print s1out
1467	quit(-1)
1468
1469	if db:
1470	for s1out in output: print s1out
1471
1472	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
1473
1474	# Keeping the operations
1475	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1476	"' -f " + prevfile + " -v " + CFvarnp
1477	otrackf.write("\n")
1478	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1479	otrackf.write(pyins + "\n")
1480
1481	# removing dimension variable-dimension 'lon'
1482	varnCFs.remove('lon')
1483
1484	varkeep.append('lonstats')
1485
1486	elif op == 'xvar':
1487	# x-axis variance values
1488	vals='lon\|-1,lon,var,' + ':'.join(varnCFs) + ':' + vdnz
1489	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1490	if dofile:
1491	try:
1492	with gen.Capturing() as output:
1493	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1494	except:
1495	print errmsg
1496	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1497	for s1out in output: print s1out
1498	quit(-1)
1499
1500	if db:
1501	for s1out in output: print s1out
1502
1503	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
1504
1505	# Keeping the operations
1506	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1507	"' -f " + prevfile + " -v " + CFvarnp
1508	otrackf.write("\n")
1509	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1510	otrackf.write(pyins + "\n")
1511
1512	# removing dimension variable-dimension 'lon'
1513	varnCFs.remove('lon')
1514
1515	varkeep.append('lonstats')
1516
1517	elif op == 'ymean':
1518	# y-axis mean values
1519	vals='lat\|-1,lat,mean,' + ':'.join(varnCFs) + ':' + vdnz
1520	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1521	if dofile:
1522	try:
1523	with gen.Capturing() as output:
1524	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1525	except:
1526	print errmsg
1527	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1528	for s1out in output: print s1out
1529	quit(-1)
1530
1531	if db:
1532	for s1out in output: print s1out
1533
1534	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
1535
1536	# Keeping the operations
1537	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1538	"' -f " + prevfile + " -v " + CFvarnp
1539	otrackf.write("\n")
1540	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1541	otrackf.write(pyins + "\n")
1542
1543	# removing dimension variable-dimension 'lat'
1544	varnCFs.remove('lat')
1545
1546	varkeep.append('latstats')
1547
1548	elif op == 'zsum':
1549	# vertical aggregated values
1550	print " " + fname + ": kind '" + op + "' not ready !!"
1551	wrongstats.append(CFvarn + '_' + opers)
1552	break
1553	else:
1554	print errmsg
1555	print ' ' + fname + ": operation '" + op + "' not ready !!"
1556	quit(-1)
1557
1558	# End of kind of operation
1559
1560	# Variable name in file (vninF) changed due to operation
1561	# but only if previous operation does not the same 'statistic'
1562	chvn = gen.dictionary_key_list(Opsurs, op)
1563	if chvn is not None:
1564	oldvninF = vninF
1565	vninF = vninF + chvn
1566	CFvarnp = CFvarnp.replace(oldvninF,vninF)
1567
1568	if dofile:
1569	if len(varkeep) > 0:
1570	varkeepS = ',' + ','.join(varkeep)
1571	else:
1572	varkeepS = ''
1573
1574	# Adding such CF dimension variables which might not be in the
1575	# post-operation file
1576	try:
1577	with gen.Capturing() as output:
1578	varinfile = ncvar.ivars(fileon)
1579	except:
1580	print errmsg
1581	print 'ivars('+fileon+')'
1582	for s1out in output: print s1out
1583	# Removing file in order to make sure that it will be redone
1584	print ' ' + fname + " removing file '" + fileon + "' ..."
1585	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1586	quit(-1)
1587
1588	for CFvn in varnCFs:
1589	if not gen.searchInlist(varinfile, CFvn):
1590	if db:
1591	print ' ' + fname + ": recupering CF variable '" + CFvn + "'"
1592	try:
1593	with gen.Capturing() as output:
1594	ncvar.fvaradd(prevfile+','+CFvn, fileon)
1595	except:
1596	print errmsg
1597	print 'fvaradd(' + prevfile + ',' + CFvn +', '+fileon+')'
1598	ncvar.fvaradd(prevfile+','+CFvn, fileon)
1599	for s1out in output: print s1out
1600	# Removing file in order to make sure that it will be redone
1601	print ' ' + fname + " removing file '" + fileon + "' ..."
1602	#sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1603	quit(-1)
1604
1605	if db:
1606	for s1out in output: print s1out
1607
1608	totalvarkeeps = CFvarnp + ',' + ','.join(varnCFs) + varkeepS
1609	try:
1610	with gen.Capturing() as output:
1611	oclean = ncvar.cleaning_varsfile(totalvarkeeps,fileon)
1612	except:
1613	print errmsg
1614	print 'cleaning_varsfile('+totalvarkeeps+','+fileon+')'
1615	ncvar.cleaning_varsfile(totalvarkeeps,fileon)
1616	for s1out in output: print s1out
1617	# Removing file in order to make sure that it will be redone
1618	print ' ' + fname + " removing file '" + fileon + "' ..."
1619	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1620	quit(-1)
1621
1622	if db:
1623	for s1out in output: print s1out
1624
1625	# End of operations
1626	if db:
1627	print fname + ": successful calculation of '" + vninF + "' from '" + \
1628	CFvarn + "' '" + Opers + "' !!"
1629
1630	if len(wrongstats) > 1:
1631	print warnmsg
1632	print ' ' + fname + ": statisitcs not possible to compute:", wrongstats
1633
1634	if db: print " End of '" + fname + "' "
1635
1636	return
1637
1638	def compute_vars(config, modinf, idir, odir, Files, allvarcomp, fscr, debug):
1639	""" Function to compute the variables
1640	config= Configuration of the experiment
1641	modinf= class with information about the model
1642	idir= input experiment folder
1643	odir= output experiment folder
1644	Files= dictionary with the header and the files' correspondence
1645	allvarcomp= dictionary with all the variables to compute and their information
1646	fscr= whether files should be done from the scratch or not
1647	"""
1648	fname = 'compute_vars'
1649
1650	for vopn in allvarcomp.keys():
1651	# variable & operation
1652	vn = vopn.split('_')[0]
1653	op = vopn.split('_')[1]
1654	vopnV = allvarcomp[vopn]
1655	fheader = vopnV[0]
1656	if type(vopnV[1]) == type([1, 2]) and len(vopnV[1]) == 1:
1657	vals = vopnV[1]
1658	model = vals[0]
1659	else:
1660	model = vopnV[1]
1661	diag = vopnV[2]
1662	globalP = vopnV[3]
1663
1664	if debug:
1665	print ' ' + vn + ': ' + op
1666
1667	# Computing CF variable
1668	if model is not None or diag is not None:
1669	vinf = VariableInf(vn,fheader,model,diag)
1670	if op.find('pinterp') != -1:
1671	SP = 'p'
1672	else:
1673	SP = ''
1674
1675	# Comppute variable
1676	finalfilen = odir + '/' + vn + '_' + fheader + SP + '.nc'
1677	if fscr:
1678	sout = sub.call('rm ' + finalfilen + ' >& /dev/null', shell=True)
1679
1680	if not os.path.isfile(finalfilen):
1681	compute_variable(modinf, idir, Files, odir, vinf, globalP, fscr, \
1682	config['pyHOME'], config['CFreftime'], config['CFunitstime'], debug)
1683
1684	# Compute variable statistics
1685	finalfilen = odir + '/' + vn + '_' + fheader + SP + '_' + \
1686	op.replace('+','_') + '.nc'
1687	if fscr:
1688	sout = sub.call('rm ' + finalfilen + ' >& /dev/null', shell=True)
1689
1690	if not os.path.isfile(finalfilen):
1691	compute_statistics(modinf, cnf, iwdir, Files, owdir, vinf, globalP, \
1692	op, fscr, debug)
1693
1694	else:
1695	print errmsg
1696	print ' ' + fname + ": neither 'model' or 'diag' variables for '" + vn \
1697	+ "' !!"
1698	quit(-1)
1699
1700	return
1701
1702	def get_plots_var(vc,pltk,db):
1703	""" Function to provide the plots to make for each variable from pltk ('DIRPLT_', or 'DIFFPLT_') dictionary
1704	vc= pltk dictionary, dictionary with all the parameters started with pltk
1705	pltk= kind of plots. Values started by 'DIRPLT_', or 'DIFFPLT_' in model configuration
1706	"""
1707	fname = 'get_plots_var'
1708
1709	LH = len(pltk)
1710	# list of plots
1711	doplots = {}
1712	# plots by variable
1713	plotsvar = {}
1714	# individual plots by variable (not repeated)
1715	indivplotsvar = {}
1716
1717	# Variables with a plot which requires vertical interpolation
1718	varplotp = []
1719
1720	ipl = 0
1721	for plot in vc.keys():
1722	Lplot = len(plot)
1723	pln = plot[LH:Lplot+1]
1724	varops = vc[plot].split(':')
1725	for vn in varops:
1726	VOn = vn.split('#')
1727	# Variables and operations in plot
1728	vplt = []
1729	oplt = []
1730	voplt = []
1731	for vop in VOn:
1732	vv = vop.split('\|')[0]
1733	oo = vop.split('\|')[1]
1734	vplt.append(vv)
1735	oplt.append(oo)
1736	if not gen.searchInlist(voplt,vop): voplt.append(vop)
1737	vpltS = '#'.join(vplt)
1738	opltS = '#'.join(oplt)
1739
1740	if not doplots.has_key(pln):
1741	doplots[pln] = [vn]
1742	else:
1743	plots = doplots[pln]
1744	plots.append(vn)
1745	doplots[pln] = plots
1746
1747	if not plotsvar.has_key(vn):
1748	plotsvar[vn] = [pln]
1749	else:
1750	plots = plotsvar[vn]
1751	plots.append(pln)
1752	plotsvar[vn] = plots
1753
1754	for VOv in voplt:
1755	if not indivplotsvar.has_key(VOv):
1756	indivplotsvar[VOv] = [pln]
1757	else:
1758	plots = indivplotsvar[VOv]
1759	plots.append(pln)
1760	indivplotsvar[VOv] = plots
1761
1762	if vc[plot].find('pinterp') != -1:
1763	if not gen.searchInlist(varplotp,vn): varplotp.append(pln)
1764
1765	ipl = ipl + 1
1766
1767	if db:
1768	print ' plots to make _______'
1769	gen.printing_dictionary(doplots)
1770	print ' plots by variable\|operation _______'
1771	gen.printing_dictionary(plotsvar)
1772	print ' individual plots by variable\|operation _______'
1773	gen.printing_dictionary(indivplotsvar)
1774	print ' plots with the vertical interpolation _______'
1775	print ' #', varplotp
1776
1777	if db: print " End of '" + fname + "' "
1778
1779	return doplots, plotsvar, indivplotsvar, varplotp
1780
1781	def plots_listconstruct(config, pkind, finf, odir, debug):
1782	""" Function to create the list of plots to draw
1783	config= Configuration of the experiment
1784	pkind= kind of plots ('DIRPLT', 'DIFFPLT')
1785	finf= file with the instructions of the plots
1786	odir= output experiment folder
1787	"""
1788	fname='plots_listconstruct'
1789
1790	dirplot = gen.get_specdictionary_HMT(config, H=pkind+'_')
1791
1792	# No plots because there are not plots of this kind
1793	if dirplot is None:
1794	print warnmsg
1795	print ' ' + fname + ": no plots for type '" + pkind + "' !!"
1796	return {}, 0
1797
1798	if debug:
1799	print " '" + pkind + "' plots to draw ________"
1800	gen.printing_dictionary(dirplot)
1801
1802	# Getting plots by variable
1803	plots, varplot, indivarplot, plotp = get_plots_var(dirplot, pkind+'_', debug)
1804
1805	Svarplot = gen.dictKeysVals_stringList(plots,cV=':')
1806
1807	Nplots = 0
1808	for pl in Svarplot.split(','):
1809	Nplots = Nplots + len(pl.split(':'))
1810
1811	# Outwritting the varcompute to avoid next time (if it is not filescratch!)
1812	objf = open(finf, 'w')
1813	objf.write('plots: ' + Svarplot + '\n')
1814	objf.write('itotp: ' + str(Nplots) + '\n')
1815	objf.close()
1816
1817	if debug: print " End of '" + fname + "' "
1818
1819	return plots, Nplots
1820
1821	def read_plot_file(figfile):
1822	""" Function to read the file with the information about the plots to draw
1823	figfile= file with the information
1824	"""
1825	fname = 'read_plot_file'
1826
1827	if not os.path.isfile(figfile):
1828	print errormsg
1829	print ' ' + fname + ": figures file '" + figfile + "' does not exist !!"
1830	quit(-1)
1831
1832	objf = open(figfile, 'r')
1833	for line in objf:
1834	if line[0:1] != '#' and len(line) > 1:
1835	values = line.replace('\\n','').split(' ')
1836	if values[0] == 'plots:':
1837	if values[1] != 'none':
1838	plots = gen.stringList_dictKeysVals(values[1],cV=':')
1839	else:
1840	pots = {}
1841	elif values[0] == 'itotp:':
1842	Nplots = int(values[1])
1843
1844	objf.close()
1845
1846	return plots, Nplots
1847
1848	def varnoper(vn, oper, opsurdict):
1849	""" Function to provide name of the variable after operation as result of addition of the 'surnames'
1850	vn= variable name
1851	oper= '+' separated list of operations
1852	opsurdict= dictionary with the surname of operations
1853	>>> OpSurDict = {'mean': ['tmean', 'xmean', 'ymean']}
1854	>>> varnoper('tas', 'pinterp+tmean+xmean', OpSurDict)
1855	tasmeanmean
1856	"""
1857	fname = 'varnoper'
1858
1859	newvn = vn
1860
1861	for op in oper.split('+'):
1862	surname = gen.dictionary_key_list(opsurdict,op)
1863	if surname is not None:
1864	newvn = newvn + surname
1865
1866	return newvn
1867
1868	def draw_plot(kplot, CFvplot, fplot, vplot, dplot, pplot, finame, tfig, kfig, mapval,\
1869	tvals, expgraphc, od, pyH, fscr, db):
1870	""" Function to draw a plot
1871	kplot= kind of plot
1872	CFvplot= list of the CFnames of the variables
1873	fplot= list with the files in the plot
1874	vplot= list with the name of the variables in each file
1875	dplot= list of the dimensions in each file
1876	pplot= list of pictoric characteristics for each variable
1877	finame= name of the figure
1878	tfig= title of the figure
1879	kfig= kind of the figure
1880	mapval= value of the map
1881	tvals= list with time values
1882	tunits: units of time ('!' for spaces)
1883	timekind: kind of time ticks in the plot
1884	timefmt: format of time ticks in the plot
1885	timelabel: label of time-axis in the plot
1886	expgraphc= dictionary with expriment graphical characteristics
1887	od= output directory
1888	pyH= python HOME
1889	fscr= whether figure should be done from scratch
1890	"""
1891	fname = 'draw_plot'
1892
1893	if fscr:
1894	sout = sub.call('rm ' + finame + ' >& /dev/null', shell=True)
1895
1896	os.chdir(od)
1897
1898	# CF variable-dimensions
1899	CFvardims = {'lon':'lon', 'lat': 'lat', 'pres': 'pres', 'time': 'time'}
1900
1901	# Tracking file with all the figures
1902	trkobjf = open('all_figures.inf', 'a')
1903
1904	# Time-characteristics
1905	tunits = tvals[0]
1906	timekind = tvals[1]
1907	timefmt = tvals[2]
1908	timelabel = tvals[3]
1909
1910	# Graphical labels and configuration
1911
1912
1913	if not os.path.isfile(finame):
1914	if db:
1915	print " drawing '" + finame + "' ..."
1916	print ' plot kind:', kplot
1917	print ' CFvars:', CFvplot
1918	print ' files:', fplot
1919	print ' in file vars:', vplot
1920	print ' dims:', dplot
1921	print ' pictoric values:', pplot
1922	print ' figure name:', finame
1923	print ' title:', tfig.replace('!',' ')
1924	print ' kind:', kfig
1925	print ' map:', mapval
1926	print ' time units:', tunits
1927
1928	if kplot == 'diffmap2Dsfc':
1929	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1930	quit(-1)
1931	elif kplot == 'diffmap2Dz':
1932	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1933	quit(-1)
1934	elif kplot == 'map2Dsfc':
1935	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1936	quit(-1)
1937	elif kplot == 'map3D':
1938	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1939	quit(-1)
1940	elif kplot == '2lines':
1941	figtit = tfig.replace('!','\|')
1942
1943	lineAstdn = CFvplot[0]
1944	lineBstdn = CFvplot[1]
1945	figfs = ','.join(fplot)
1946
1947	lineA = pplot[0]
1948	Arange = str(lineA[0]) + ',' + str(lineA[1])
1949	# Values are changed from specific values in `model_graphics.dat'
1950	if lineA[3].find('%') == -1:
1951	Aline = lineA[2]
1952	Akind = lineA[3]
1953	Amark = lineA[4]
1954	Asize = lineA[5]
1955	else:
1956	Aline = 'red'
1957	Akind = '-'
1958	Amark = ','
1959	Asize = 2.
1960
1961	lineB = pplot[1]
1962	Brange = str(lineB[0]) + ',' + str(lineB[1])
1963	# Values are changed from specific values in `model_graphics.dat'
1964	if lineB[3].find('%') == -1:
1965	Bline = lineB[2]
1966	Bkind = lineB[3]
1967	Bmark = lineB[4]
1968	Bsize = lineB[5]
1969	else:
1970	Bline = 'blue'
1971	Bkind = '-'
1972	Bmark = ','
1973	Bsize = '2.'
1974
1975	# It is assumed that if the space variable is 'lon': is desired a
1976	# (lon, vals) plot if it is 'lat': then (vals, lat) plot
1977	if gen.searchInlist(dplot[0],'lon'):
1978	spacedim ='lon'
1979	axisvals = 'y'
1980	figmid = 'longitudinal'
1981	elif gen.searchInlist(dplot[0],'lat'):
1982	spacedim = 'lat'
1983	axisvals = 'x'
1984	figmid = 'meridional'
1985	elif gen.searchInlist(dplot[0],'pres'):
1986	spacedim = 'pres'
1987	axisvals = 'x'
1988	figmid = 'vertical'
1989	else:
1990	print errmsg
1991	print ' ' + fname + ": in '2lines' only ready for: 'lon', 'lat'," + \
1992	" 'pres' as common dimension !!"
1993	print ' dimensions in plot:', dplot[0]
1994	quit(-1)
1995
1996	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
1997	'!'.join(tfig.split('!')[2:])
1998
1999	graphvals = spacedim + ':' + Arange + ':' + Brange + ':Extrs:' + \
2000	axisvals + ':' + ','.join(CFvplot) + ':' + Aline + ',' + Bline + \
2001	':2.,2.:' + Akind + ',' + Bkind + ':2.,2.:,:' + figtit + ':' + \
2002	spacedim + ':0\|auto:' + finame.replace('.'+kfig, '') + ':' + kfig + \
2003	':True'
2004
2005	fvarS = ','.join(vplot)
2006
2007	drwins = 'draw_2lines'
2008	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2009	" -S '" + graphvals + "' -v " + fvarS
2010
2011	try:
2012	with gen.Capturing() as output:
2013	sout = sub.call(plotins, shell=True)
2014	except:
2015	print errmsg
2016	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2017	print sout
2018	for s1out in output: print s1out
2019	quit(-1)
2020
2021	# keeping all figures
2022	trkobjf.write('\n')
2023	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2024	trkobjf.write(plotins + '\n')
2025
2026	elif kplot == '2linesTime':
2027	figtit = tfig.replace('!','\|')
2028
2029	lineAstdn = CFvplot[0]
2030	lineBstdn = CFvplot[1]
2031	figfs = ','.join(fplot)
2032
2033	lineA = pplot[0]
2034	Arange = str(lineA[0]) + ',' + str(lineA[1])
2035	# Values are changed from specific values in `model_graphics.dat'
2036	if lineA[3].find('%') == -1:
2037	Aline = lineA[2]
2038	Akind = lineA[3]
2039	Amark = lineA[4]
2040	Asize = lineA[5]
2041	else:
2042	Aline = 'red'
2043	Akind = '-'
2044	Amark = ','
2045	Asize = '2.'
2046
2047	lineB = pplot[1]
2048	Brange = str(lineB[0]) + ',' + str(lineB[1])
2049	# Values are changed from specific values in `model_graphics.dat'
2050	if lineB[3].find('%') == -1:
2051	Bline = lineB[2]
2052	Bkind = lineB[3]
2053	Bmark = lineB[4]
2054	Bsize = lineB[5]
2055	else:
2056	Bline = 'blue'
2057	Bkind = '-'
2058	Bmark = ','
2059	Bsize = '2.'
2060
2061	timeaxis = 'x'
2062	figmid = 'evolution'
2063
2064	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
2065	'!'.join(tfig.split('!')[2:])
2066
2067	graphvals = 'time:' + Arange + ':' + Brange + ':' + timekind + ';' + \
2068	timefmt + ':' + timeaxis + ':' + ','.join(CFvplot) + ':' + Aline + ',' \
2069	+ Bline + ':2.,2.:' + Akind + ',' + Bkind + ':2.,2.:,:' + figtit + ':' \
2070	+ timelabel + ':0\|auto:' + finame.replace('.'+kfig, '') + ':' + \
2071	kfig + ':True'
2072
2073	fvarS = ','.join(vplot)
2074
2075	drwins = 'draw_2lines_time'
2076	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2077	" -S '" + graphvals + "' -v " + fvarS
2078
2079	try:
2080	with gen.Capturing() as output:
2081	sout = sub.call(plotins, shell=True)
2082	except:
2083	print errmsg
2084	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2085	print sout
2086	for s1out in output: print s1out
2087	quit(-1)
2088
2089	# keeping all figures
2090	trkobjf.write('\n')
2091	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2092	trkobjf.write(plotins + '\n')
2093
2094	elif kplot == 'Nlines':
2095	figtit = tfig.replace('!','\|')
2096
2097	linestdn = CFvplot[0]
2098	figfs = ','.join(fplot)
2099
2100	Nlines = len(fplot)
2101
2102	lablines = []
2103	for il in range(Nlines):
2104	linev = pplot[il]
2105	# Values are changed from specific values in `model_graphics.dat'
2106	if linev[3].find('%') == -1:
2107	line = linev[2]
2108	kind = linev[3]
2109	mark = linev[4]
2110	size = linev[5]
2111	if il == 0:
2112	vrange = str(linev[0]) + ',' + str(linev[1])
2113	kinds = kind
2114	lines = line
2115	marks = mark
2116	sizes = str(size)
2117	else:
2118	kinds = kinds + ',' + kind
2119	lines = lines + ',' + line
2120	marks = marks + '@' + mark
2121	sizes = sizes + ',' + str(size)
2122	else:
2123	lines = 'None'
2124	kinds = '-'
2125	marks = ','
2126	sizes = '2.'
2127
2128	secsf = fplot[il].split('/')
2129	Nsecsf = len(secsf)
2130	expvs = expgraphc[secsf[Nsecsf-3] + '/' + secsf[Nsecsf-2]]
2131	lablines.append(expvs.label)
2132
2133	# It is assumed that if the space variable is 'lon': is desired a
2134	# (lon, vals) plot if it is 'lat': then (vals, lat) plot
2135	if gen.searchInlist(dplot[0],'lon'):
2136	spacedim ='lon'
2137	sdlab ='lon ($degrees\ East$)'
2138	axisvals = 'y'
2139	figmid = 'longitudinal'
2140	elif gen.searchInlist(dplot[0],'lat'):
2141	spacedim = 'lat'
2142	sdlab = 'lat ($degrees\ North$)'
2143	axisvals = 'x'
2144	figmid = 'meridional'
2145	elif gen.searchInlist(dplot[0],'pres'):
2146	spacedim = 'pres'
2147	sdlab = 'pres ($Pa$)'
2148	axisvals = 'x'
2149	figmid = 'vertical'
2150	else:
2151	print errmsg
2152	print ' ' + fname + ": in 'Nlines' only ready for: 'lon', 'lat'," + \
2153	" 'pres' as common dimension !!"
2154	print ' dimensions in plot:', dplot[0]
2155	quit(-1)
2156
2157	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
2158	'!'.join(tfig.split('!')[2:])
2159	figtit = figtit.replace('!',' ')
2160	leglabels = ','.join(lablines)
2161
2162	graphvals = spacedim + ':' + axisvals + ':' + sdlab + ':auto:' + \
2163	leglabels + ':' + CFvplot[0] +':'+ figtit + ':0\|auto:' + lines + ':' + \
2164	kinds + ':' + marks +':'+ sizes +':'+ sizes +':all:'+ \
2165	finame.replace('.'+kfig, '') + ':' + kfig + ':True'
2166
2167	fvarS = vplot[0]
2168
2169	drwins = 'draw_lines'
2170	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2171	" -S '" + graphvals + "' -v " + fvarS
2172
2173	try:
2174	with gen.Capturing() as output:
2175	sout = sub.call(plotins, shell=True)
2176	except:
2177	print errmsg
2178	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2179	print sout
2180	for s1out in output: print s1out
2181	quit(-1)
2182
2183	# keeping all figures
2184	trkobjf.write('\n')
2185	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2186	trkobjf.write(plotins + '\n')
2187
2188	elif kplot == 'Nlines_time':
2189	figtit = tfig.replace('!','\|')
2190
2191	linestdn = CFvplot[0]
2192	figfs = ','.join(fplot)
2193
2194	Nlines = len(fplot)
2195
2196	lablines = []
2197	for il in range(Nlines):
2198	linev = pplot[il]
2199	# Values are changed from specific values in `model_graphics.dat'
2200	if linev[3].find('%') == -1:
2201	line = linev[2]
2202	kind = linev[3]
2203	mark = linev[4]
2204	size = linev[5]
2205	if il == 0:
2206	vrange = str(linev[0]) + ',' + str(linev[1])
2207	kinds = kind
2208	lines = line
2209	marks = mark
2210	sizes = str(size)
2211	else:
2212	kinds = kinds + ',' + kind
2213	lines = lines + ',' + line
2214	marks = marks + '@' + mark
2215	sizes = sizes + ',' + str(size)
2216	else:
2217	lines = 'None'
2218	kinds = '-'
2219	marks = ','
2220	sizes = '2.'
2221
2222	secsf = fplot[il].split('/')
2223	Nsecsf = len(secsf)
2224	expvs = expgraphc[secsf[Nsecsf-3] + '/' + secsf[Nsecsf-2]]
2225	lablines.append(expvs.label)
2226
2227	valsaxis = 'y'
2228	figmid = 'evolution'
2229
2230	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
2231	'!'.join(tfig.split('!')[2:])
2232	figtit = figtit.replace('!',' ')
2233	leglabels = ','.join(lablines)
2234
2235	graphvals = 'time;' + valsaxis + ';time;' + leglabels + ';' + \
2236	CFvplot[0] + ';' + figtit + ';None;time\|' + '\|'.join(tvals) + \
2237	';0\|auto;' + kfig + ';' + kinds + ';' + lines + ';' + marks + ';' + \
2238	sizes + ';' + sizes + ';all;-1;True'
2239
2240	fvarS = vplot[0]
2241
2242	drwins = 'draw_lines_time'
2243	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2244	" -S '" + graphvals + "' -v " + fvarS
2245
2246	try:
2247	with gen.Capturing() as output:
2248	sout = sub.call(plotins, shell=True)
2249	except:
2250	print errmsg
2251	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2252	print sout
2253	for s1out in output: print s1out
2254	quit(-1)
2255
2256	sout = sub.call('mv lines_time.' + kfig + ' ' + finame, shell=True)
2257
2258	# keeping all figures
2259	trkobjf.write('\n')
2260	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2261	trkobjf.write(plotins + '\n')
2262
2263	elif kplot == 'shadcont2Dsfc':
2264	figtit = tfig.replace('!','\|')
2265	shdstdn = CFvplot[0]
2266	cntstdn = CFvplot[1]
2267	figfs = ','.join(fplot)
2268
2269	shad = pplot[0]
2270	srange = str(shad[0]) + ',' + str(shad[1])
2271	cbar = shad[2]
2272	cnt = pplot[1]
2273	crange = str(cnt[0]) + ',' + str(cnt[1])
2274	cfmt = cnt[3]
2275	ckind = cnt[4]
2276	cline = cnt[5]
2277
2278	graphvals = ','.join(CFvplot)
2279	graphvals = graphvals + ':lon\|-1,lat\|-1:lon\|-1,lat\|-1:lon:lat:auto:' + \
2280	cbar + ',auto,auto:' + ckind + ',' + cline + ':' + cfmt + ':' + \
2281	srange + ':' + crange + ',9:' + figtit + ':' + kfig + ':None:' + \
2282	mapval + ':True'
2283
2284	fvarS = ','.join(vplot)
2285
2286	drwins = 'draw_2D_shad_cont'
2287	plotins = "python " + pyH + "/drawing.py -f " + figfs + " -o " + drwins +\
2288	" -S '" + graphvals + "' -v " + fvarS
2289	try:
2290	with gen.Capturing() as output:
2291	sout = sub.call(plotins, shell=True)
2292	except:
2293	print errmsg
2294	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2295	print sout
2296	for s1out in output: print s1out
2297	quit(-1)
2298
2299	sout = sub.call('mv 2Dfields_shadow-contour.'+kfig+' '+finame, shell=True)
2300
2301	# keeping all figures
2302	trkobjf.write('\n')
2303	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2304	trkobjf.write(plotins + '\n')
2305
2306	elif kplot == 'shadconthovmsfc':
2307	figtit = tfig.replace('!','\|')
2308	shdstdn = CFvplot[0]
2309	cntstdn = CFvplot[1]
2310	figfs = ','.join(fplot)
2311
2312	shad = pplot[0]
2313	srange = str(shad[0]) + ',' + str(shad[1])
2314	cbar = shad[2]
2315	cnt = pplot[1]
2316	crange = str(cnt[0]) + ',' + str(cnt[1])
2317	cfmt = cnt[3]
2318	ckind = cnt[4]
2319	cline = cnt[5]
2320	# It is assumed that if the space variable is 'lon': is desired a
2321	# (lon, time) plot if it is 'lat': then (time, lat) plot
2322	if gen.searchInlist(dplot,'lon'):
2323	spacedim ='lon'
2324	figmid = 'longitudinal\|evolution\|of'
2325	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2326	'\|'.join(tfig.split('!')[2:])
2327	reverse= 'None'
2328	dims= spacedim+'\|-1,time\|-1;'+spacedim+'\|-1,time\|-1;'+spacedim+';time'
2329	else:
2330	spacedim='lat'
2331	figmid = 'meridional\|evolution\|of'
2332	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2333	'\|'.join(tfig.split('!')[2:])
2334	reverse='None'
2335	dims= spacedim+'\|-1,time\|-1;'+spacedim+'\|-1,time\|-1;time;'+spacedim
2336
2337	graphvals = ','.join(CFvplot)
2338	graphvals = graphvals + ';' + dims +';auto;' + cbar + ',auto,auto' + ';' \
2339	+ ckind + ',' + cline + ';' + cfmt + ';' + srange + ';' + crange + \
2340	',9;' + figtit + ';' + kfig + ';' + reverse + ';' + 'time\|' + tunits + \
2341	'\|'+ timekind + '\|' + timefmt + '\|' + timelabel + ';True'
2342
2343	fvarS = ','.join(vplot)
2344
2345	drwins = 'draw_2D_shad_cont_time'
2346	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2347	" -S '" + graphvals + "' -v " + fvarS
2348
2349	try:
2350	with gen.Capturing() as output:
2351	sout = sub.call(plotins, shell=True)
2352	except:
2353	print errmsg
2354	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2355	print sout
2356	for s1out in output: print s1out
2357	quit(-1)
2358
2359	sout = sub.call('mv 2Dfields_shadow-contour.'+kfig+' '+finame, shell=True)
2360
2361	# keeping all figures
2362	trkobjf.write('\n')
2363	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2364	trkobjf.write(plotins + '\n')
2365
2366	elif kplot == 'shadcont2Dzsec':
2367	figtit = tfig.replace('!','\|')
2368	shdstdn = CFvplot[0]
2369	cntstdn = CFvplot[1]
2370	figfs = ','.join(fplot)
2371
2372	shad = pplot[0]
2373	srange = str(shad[0]) + ',' + str(shad[1])
2374	cbar = shad[2]
2375	cnt = pplot[1]
2376	crange = str(cnt[0]) + ',' + str(cnt[1])
2377	cfmt = cnt[3]
2378	ckind = cnt[4]
2379	cline = cnt[5]
2380
2381	# It is assumed that if the space variable is 'lon': is desired a
2382	# (lon, pres) plot it it is 'lat': then (pres, lat) plot
2383	if gen.searchInlist(dplot, 'lon'):
2384	spacedim='lon'
2385	figmid = 'long.\|vert.\|cross\|sec.\|of'
2386	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2387	'\|'.join(tfig.split('!')[2:])
2388	dims= spacedim+'\|-1,pres\|-1:'+spacedim+'\|-1,pres\|-1:'+spacedim+':pres'
2389	else:
2390	spacedim='lat'
2391	figmid = 'mer.\|vert.\|cross\|sec.\|of'
2392	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2393	'\|'.join(tfig.split('!')[2:])
2394	# NOT WORKING? dims= spacedim+'\|-1,pres\|-1:'+spacedim+'\|-1,pres\|-1:'+'pres:'+spacedim
2395	dims= spacedim+'\|-1,pres\|-1:'+spacedim+'\|-1,pres\|-1:'+spacedim+':pres'
2396
2397	reverse = 'None'
2398
2399	graphvals = ','.join(CFvplot)
2400	graphvals = graphvals + ':' + dims + ':auto:' + cbar + ',auto,auto:' + \
2401	ckind + ',' + cline + ':' + cfmt + ':' + srange + ':' + crange + ',9:' \
2402	+ figtit + ':' + kfig + ':' + reverse + ':None:True'
2403
2404	fvarS = ','.join(vplot)
2405
2406	drwins = 'draw_2D_shad_cont'
2407	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs + ' -o ' + drwins + \
2408	" -S '" + graphvals + "' -v " + fvarS
2409	try:
2410	with gen.Capturing() as output:
2411	sout0 = sub.call(plotins, shell=True)
2412	except:
2413	print errmsg
2414	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2415	for s1out in output: print s1out
2416	quit(-1)
2417
2418	sout = sub.call('mv 2Dfields_shadow-contour.'+kfig+' '+finame, shell=True)
2419
2420	# keeping all figures
2421	trkobjf.write('\n')
2422	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2423	trkobjf.write(plotins + '\n')
2424
2425	else:
2426	print errmsg
2427	print " " + fname + ": plot kind '" + kplot + "' not ready !!"
2428	quit(-1)
2429
2430	trkobjf.close()
2431
2432	if db: print " End of '" + fname + "' "
2433
2434	return
2435
2436	def opexplained(Opn,Exp):
2437	""" Function to provide the final explanation for the title in the graph. Some operations have
2438	values separated by `~' which need to be processed
2439	Opn= full operation string
2440	Exp= explanation from dictionary
2441	>>> opexplained('dimvarvalue~pres~85000','@')
2442	pres@85000
2443	>>> opexplained('xmean','xmean')
2444	xmean
2445	"""
2446	fname = 'opexplained'
2447
2448	if Opn.find('~') == -1:
2449	explanation = Exp
2450	else:
2451	Opnv = Opn.split('~')
2452	if Opnv[0] == 'dimvarvalue':
2453	explanation = Opnv[1] + Exp + Opnv[2]
2454	else:
2455	print errormsg
2456	print ' ' + fname + ": unknow '~' separated operation '" + Opnv[0] + \
2457	"' !!"
2458	quit(-1)
2459	return explanation
2460
2461	def optitle(op,opexp):
2462	""" Function to creat the operation section in the title of a figure ('!' for spaces)
2463	op= '+' separated list of operations
2464	opexp= dictionary with the `explanation'(text in title) to appear for each operation
2465	>>> optitle('pinterp+xmean+tmean',{'pinterp':'pinterp', 'xmean':'xmean', 'tmean':'tmean'})
2466	$_{[pinterp\!xmean\!tmean]}$
2467	>>> optitle('pinterp+dimvarvalue~pres~85000+tmean',{'dimvarvalue':'@', 'pinterp':'pinterp', 'xmean':'xmean', 'tmean':'tmean'})
2468	$_{[pinterp\!pres@85000\!tmean]}$
2469	"""
2470	fname = 'optitle'
2471
2472	opvals = op.split('+')
2473	for op1 in opvals:
2474	op1n = op1.split('~')[0]
2475	if not opexp.has_key(op1n):
2476	print errmsg
2477	print ' ' + fname + ": no explanation for operation '" + op1n + "' !!"
2478	print ' provided:', opexp.keys()
2479	quit(-1)
2480	# Final explanation
2481	op1en = opexplained(op1,opexp[op1n]).replace(' ','!')
2482
2483	if op1n == opvals[0].split('~')[0]:
2484	titop = '$_{[' + op1en
2485	if len(opvals) == 1: titop = titop + ']}$'
2486	elif op1n == opvals[len(opvals)-1].split('~')[0]:
2487	titop = titop + '\!' + op1en + ']}$'
2488	else:
2489	titop = titop + '\!' + op1en
2490
2491	return titop
2492
2493	def create_figure_title(mod,exp,varops,explop):
2494	""" Function to create the title of a figure ('!' for spaces)
2495	mod: model name
2496	exp: experiment name
2497	varops: list with [var]\|[op] values from wich create the title
2498	explop: dictionary with the `explanation'(text in title) to appear for each operation
2499	>>> expops = {'pinterp':'pinterp', 'xmean':'xmean', 'tmean':'tmean', 'last':'last'}
2500	>>> create_figure_title('WRF','current',['ua\|pinterp+xmean+tmean', 'va\|pinterp+xmean+tmean'], expops)
2501	WRF!current!ua!&!va$_{[pinterp\!xmean\!tmean]}$
2502	>>> create_figure_title('WRF','current',['ua\|pinterp+xmean+tmean', 'ua\|pinterp+xmean+last'], expops)
2503	WRF!current!ua$_{[pinterp\!xmean\!tmean]}$!&!!$_{[pinterp\!xmean\!last]}$
2504	>>> create_figure_title('WRF','current',['ua\|pinterp+xmean+tmean', 'va\|pinterp+xmean+last'], expops)
2505	WRF!current!ua$_{[pinterp\!xmean\!tmean]}$!&!va$_{[pinterp\!xmean\!last]}$
2506	>>> create_figure_title('WRF','current',['uas\|dimvarvalue~lon~23.23', 'vas\|dimvarvalue~lon~23.23'], expops)
2507	WRF!current!uas!&!vas$_{[lon@23.23]}$
2508	>>> create_figure_title('WRF','current',['hus\|pinterp+tmean+dimvarvalue~pres~85000.', 'ta\|pinterp+tmean+dimvarvalue~pres~85000.'], expops)
2509	WRF!current!hus!&!ta$_{[pinterp\!tmean\!pres@85000.]}$
2510	"""
2511	fname = 'create_figure_title'
2512
2513	titfig = mod + '!' + exp + '!'
2514
2515	varns = []
2516	opns = []
2517	for varop in varops:
2518	vn = varop.split('\|')[0]
2519	op = varop.split('\|')[1]
2520	varns.append(vn)
2521	opns.append(op)
2522	if varop == varops[0]:
2523	allvarequal = True
2524	allopequal = True
2525	varinfig = vn
2526	opinfig = op
2527	else:
2528	if vn != varinfig: allvarequal = False
2529	if op != opinfig: allopequal = False
2530
2531	Nvars = len(varns)
2532	Nops = len(opns)
2533
2534	if allvarequal:
2535	titfig = titfig + '!' + varns[0]
2536	if allopequal:
2537	opS = optitle(op,explop)
2538	titfig = titfig + opS
2539	else:
2540	for opn in opns:
2541	opS = optitle(opn,explop)
2542	if opn == opns[0]:
2543	titfig = titfig + opS
2544	elif opn == opns[Nops-1]:
2545	titfig = titfig + '!&!' + opS
2546	else:
2547	titfig = titfig + ',!' + opS
2548	else:
2549	if allopequal:
2550	opS = optitle(op,explop)
2551	for vn in varns:
2552	if vn == varns[0]:
2553	titfig = titfig + '!' + vn
2554	elif vn == varns[Nvars-1]:
2555	titfig = titfig + '!&!' + vn
2556	else:
2557	titfig = titfig + ',!' + vn
2558	titfig = titfig + opS
2559	else:
2560	for ivop in range(Nvars):
2561	vn = varns[ivop]
2562	op = opns[ivop]
2563	vnop = vn + optitle(op,explop)
2564
2565	if vn == varns[0]:
2566	titfig = titfig + '!' + vnop
2567	elif vn == varns[Nvars-1]:
2568	titfig = titfig + '!&!' + vnop
2569	else:
2570	titfig = titfig + ',!' + vnop
2571
2572	return titfig.replace('!!','!')
2573	#expops = {'dimvarvalue':'@', 'last':'last', 'pinterp':'pinterp', 'xmean':'xmean', 'tmean':'tmean'}
2574
2575	def draw_plots(config, plots, mod, exp, odir, allvarcomp, figscr, debug):
2576	""" Function to draw all plots
2577	config= Configuration of the experiment
2578	plots= dictionary with the plots
2579	mod= model of the plots
2580	exp= experiment of the plots
2581	odir= output experiment folder
2582	allvarcomp= dictionary with all the variables to compute and their information
2583	figscr= whether figures should be done from the scratch or not
2584
2585	* Plot as
2586	{[kplot]} = [varn1]\|[op1]#[varn2]\|[op2]#[...[varnN]\|[opN]], ...
2587	[kplot] ___
2588	diffmap2Dsfc: 2D map of surface differences values of 1 variable
2589	diffmap2Dz: 2D map of 3D differences values of 1 variable
2590	map2Dsfc: 2D map of surface values of 1 variable
2591	map3D: 2D map of 3D values of 1 variable
2592	shadconthovmsfc: Hovmoeller diagrams of 2 variable at the surface in shadow and the other in contourn
2593	shadcont2Dsfc: 2D map of shadow (1st variable) and countour (2nd variable) [stvar1]#[stvar2]
2594	shadcont2Dzsec: 2D map of vertical section of 2 variables one in shadow and the other in contourn
2595	[varn]
2596	variable
2597	[op]
2598	'+' separated list of operations
2599	in figures with more than 1 variable, use '#' to separate the [varn]\|[op]
2600	"""
2601	fname = 'draw_plots'
2602
2603	os.chdir(odir)
2604
2605	# Dictionary with the operations with surnames for the operated variable
2606	opersurnames = {}
2607	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
2608	for opsr in opsur.keys():
2609	opn = opsr.split('_')[1]
2610	vls = opsur[opsr].split(':')
2611	opersurnames[opn] = vls
2612
2613	# time values
2614	# Units time for the plots
2615	rd = config['CFreftime']
2616	tunits = config['CFunitstime'] + '!since!' + rd[0:4] + '-' + rd[4:6] + '-' + \
2617	rd[6:8] + '!' + rd[8:10] + ':' + rd[10:12] + ':' + rd[12:14]
2618	# time ticks kind
2619	tkind = config['timekind']
2620	# time ticks format
2621	tfmt = config['timefmt']
2622	# time axis label
2623	tlab = config['timelabel']
2624	timevals = [tunits, tkind, tfmt, tlab]
2625
2626	# Dictionary of plot specificities
2627	specplotkeyn = 'specificvarplot'
2628	plotspecifics = {}
2629	if config.has_key(specplotkeyn):
2630	# [minval]: minimum value
2631	# [maxval]: minimum value
2632	# [colorbar]: name of the colorbar (from matplotlib) to use
2633	# [cntformat]: format of the contour labels
2634	# [colorcnt]: color for the countor lines
2635	plotspecs = config[specplotkeyn].split(':')
2636	for pltspc in plotspecs:
2637	pltvls = pltspc.split('\|')
2638	vn = pltvls[0]
2639	op = pltvls[1]
2640	fn = pltvls[2]
2641	plotspecifics[fn + '_' + vn + '_' + op] = pltvls[3:]
2642	if debug:
2643	print 'Specific values for plots _______'
2644	gen.printing_dictionary(plotspecifics)
2645
2646	# Kind of figures
2647	kindfigure = config['kindfig']
2648
2649	# Graphical labels and configuration
2650	modgc, expgc = graphical_characteristics(config, debug)
2651	modv = modgc[mod]
2652	modlab = modv.label
2653	expv = expgc[mod+'/'+exp]
2654	explab = expv.label
2655
2656	# Map value
2657	mapvalue = config['mapval']
2658
2659	# pythone scripts HOME
2660	pyHOME = config['pyHOME']
2661
2662	# Title-text of operations
2663	opexplained = {}
2664	optits = config['titleoperations'].split(':')
2665	for optit in optits:
2666	opn = optit.split('\|')[0]
2667	opt = optit.split('\|')[1]
2668	opexplained[opn] = opt
2669	if debug:
2670	print 'Titles for operations _______'
2671	gen.printing_dictionary(opexplained)
2672
2673	for kplot in plots.keys():
2674	varsplt = plots[kplot]
2675	for varplt in varsplt:
2676	if debug:
2677	print " printing '" + kplot + "' var ':" + varplt + "'..."
2678	varops = varplt.split('#')
2679
2680	# CF variables in plot
2681	CFvarsplot = []
2682	# Files in plot
2683	filesplot = []
2684	# Variables in plot within the files
2685	varsplot = []
2686	# Dims in figure
2687	dimsplot = []
2688	# pictoric values in figure
2689	pictplot = []
2690	# Name of the figure
2691	figname = ''
2692	# Title of the figure
2693	titfigure = ''
2694
2695	ivp = 0
2696	for varop in varops:
2697	vn = varop.split('\|')[0]
2698	op = varop.split('\|')[1]
2699
2700	# CF variables in plot
2701	CFvarsplot.append(vn)
2702
2703	vnopS = vn + '_' + op
2704	if not allvarcomp.has_key(vnopS):
2705	print errmsg
2706	print ' ' + fname + ": no file for variable-operation '" + \
2707	vnopS + "' !!"
2708	vopvals = allvarcomp[vnopS]
2709	headf = vopvals[0]
2710	globalP = vopvals[3]
2711	gP = pinterpS(globalP)
2712
2713	filen = vn + '_' + headf + gP + '_' + op.replace('+','_') + '.nc'
2714	filesplot.append(filen)
2715	# Do we have processed the given variable?
2716	if not os.path.isfile(filen):
2717	print warnmsg
2718	print " " + fname + ": there is no file for variable '" + varop \
2719	+ "' skiping it !!"
2720	break
2721
2722	# Name of the variable inside the file
2723	vnsur = varnoper(vn, op, opersurnames)
2724	varsplot.append(vnsur)
2725
2726	# Dimensions in file
2727	try:
2728	with gen.Capturing() as output:
2729	dims = ncvar.idims(filen)
2730	except:
2731	print errmsg
2732	print 'ncvar.idims('+filen+')'
2733	for s1out in output: print s1out
2734	quit(-1)
2735
2736	dimsplot.append(dims)
2737
2738	# pictoric values for the figure
2739	Sfivaop = kplot + '_' + vn + '_' + op
2740	if plotspecifics.has_key(Sfivaop):
2741	pictvals = plotspecifics[Sfivaop]
2742	else:
2743	Vvals = gen.variables_values(vn)
2744	pictvals = [Vvals[2], Vvals[3], Vvals[6], '%g', 'fixc', 'black']
2745
2746	pictplot.append(pictvals)
2747
2748	# Header of the name of the figure
2749	if ivp == 0:
2750	figname = kplot +'_'+ vn + '_' + headf + '_' + op.replace('+','-')
2751	else:
2752	figname = figname + '-'+vn+'_'+headf+'_'+op.replace('+','-')
2753
2754	ivp = ivp + 1
2755	# End of variable-operation
2756	figname = figname + '.' + kindfigure
2757
2758	# Removig double '..' (from `dimvarvalue')
2759	figname = figname.replace('..', '.')
2760
2761	# Title of figure
2762	titfigure = create_figure_title(modlab, explab, varops, opexplained)
2763
2764	draw_plot(kplot, CFvarsplot, filesplot, varsplot, dimsplot, pictplot, \
2765	figname, titfigure, kindfigure, mapvalue, timevals, expgc, odir, \
2766	pyHOME, figscr, debug)
2767
2768	# End of variables-operations
2769
2770	# End of kind of plots
2771
2772	if debug: print " End of '" + fname + "' "
2773
2774	return
2775
2776	def get_differences_var(vc,kdiff,db):
2777	""" Function to provide the operations to make for each variable from kdiff ('DIFFOP_' or 'DIFFVAR_') dictionary
2778	vc= kdiff dictionary, dictionary with all the parameters started with kdiff
2779	kdiff= kind of differences 'op', 'var'
2780	"""
2781	fname = 'get_differences_var'
2782
2783	if kdiff == 'op':
2784	diffn = 'DIFFOP'
2785	elif kdiff == 'var':
2786	diffn = 'DIFFVAR'
2787	else:
2788	print errmsg
2789	print ' ' + fname + ": differences '" + kdiff + "' not ready !!"
2790	quit(-1)
2791	LD = len(diffn + '_')
2792
2793	iop = 1
2794	# list of operations by difference
2795	doopers = {}
2796	# operations by variable
2797	operationsvar = {}
2798	# individual operations by variable (not repeated)
2799	indivoperationsvar = {}
2800
2801	# Variables with a calculation whic requires vertical interpolation to all the
2802	# file (operations stating by [kdiff]'_pinterp')
2803	LVp = len(diffn + '_pinterp')
2804	varglobalp = []
2805
2806	for oper in vc.keys():
2807	Loper = len(oper)
2808	opn = oper[LD:Loper+1]
2809	vns = vc[oper].split(':')
2810	ops1 = opn.split('+')
2811	doopers[opn] = ops1
2812	for vn in vns:
2813	if not operationsvar.has_key(vn):
2814	operationsvar[vn] = [opn]
2815	indivoperationsvar[vn] = ops1
2816	else:
2817	opers = operationsvar[vn]
2818	opers.append(opn)
2819	operationsvar[vn] = opers
2820	opers1 = indivoperationsvar[vn]
2821	for op1 in ops1:
2822	if not gen.searchInlist(opers1, op1):
2823	opers1.append(op1)
2824	indivoperationsvar[vn] = opers1
2825
2826	if oper[0:LVp] == diffn + '_pinterp':
2827	for vn in vns:
2828	if not gen.searchInlist(varglobalp,vn): varglobalp.append(vn)
2829
2830	iop = iop + 1
2831
2832	if db:
2833	print ' operations to make _______'
2834	gen.printing_dictionary(doopers)
2835	print ' operations by variable _______'
2836	gen.printing_dictionary(operationsvar)
2837	print ' individual operations by variable _______'
2838	gen.printing_dictionary(indivoperationsvar)
2839	print ' variables with the vertical interpolation for all the file _______'
2840	print ' #', varglobalp
2841
2842	if db: print " End of '" + fname + "' "
2843
2844	return doopers, operationsvar, indivoperationsvar, varglobalp
2845
2846	def diffvarop_listconstruct(config, mods, exps, reprj, odir, debug):
2847	""" Function to create the list of differences to compute and draw
2848	config= Configuration of the experiment
2849	mods= list with the models to use
2850	exps= list with the experiments to use
2851	reprj= list with whether re-rpojection needs to be done
2852	odir= output differences folder
2853	"""
2854	fname='diffvarop_listconstruct'
2855
2856	diffop = gen.get_specdictionary_HMT(config, H='DIFFOP_')
2857	diffvar = gen.get_specdictionary_HMT(config, H='DIFFVAR_')
2858
2859	if debug:
2860	if diffop is not None:
2861	print " 'op' differences ________"
2862	gen.printing_dictionary(diffop)
2863	if diffvar is not None:
2864	print " 'var' differences ________"
2865	gen.printing_dictionary(diffvar)
2866
2867	# Getting variable characteristics from each model/experiment
2868	idir1 = config['ofold'] + '/' + mods[0] + '/' + exps[0]
2869	idir2 = config['ofold'] + '/' + mods[1] + '/' + exps[1]
2870
2871	fvarcomp = 'varcompute.inf'
2872	if reprj[0]: fvc1 = 'reproj_' + fvarcomp
2873	else: fvc1 = fvarcomp
2874
2875	if reprj[1]: fvc2 = 'reproj_' + fvarcomp
2876	else: fvc2 = fvarcomp
2877
2878	files1, testfiles1, allcompvar1, Nvar1= read_varcomp_file(idir1 + '/' + fvc1)
2879	files2, testfiles2, allcompvar2, Nvar2= read_varcomp_file(idir2 + '/' + fvc2)
2880
2881	diffops = {}
2882	if diffop is not None:
2883	# Getting 'op' differences characteristics
2884	doops, opvar, indivopvar, varglobalp = get_differences_var(diffop,'op',debug)
2885
2886	Ndiffop = 0
2887	for vn in opvar.keys():
2888	vnops = opvar[vn]
2889	for op in vnops:
2890	vnop = vn + '_' + op
2891	if not allcompvar1.has_key(vnop):
2892	print warnmsg
2893	print ' ' + fname + ": DIFFOP variable+operation '" + vnop + \
2894	"' in '" + mods[0] + '/' + exps[0] + "' was not computed !!"
2895	print ' skipping it!'
2896	print ' variables computed:', allcompvar1
2897	break
2898	if not allcompvar2.has_key(vnop):
2899	print warnmsg
2900	print ' ' + fname + ": DIFFOP variable+operation '" + vnop + \
2901	"' in '" + mods[1] + '/' + exps[1] + "' was not computed !!"
2902	print ' skipping it!'
2903	print ' variables computed:', allcompvar2
2904	break
2905	vals1 = allcompvar1[vnop]
2906	vals2 = allcompvar2[vnop]
2907
2908	headerf1 = vals1[0]
2909	headerf2 = vals2[0]
2910
2911	diffops[vnop] = [mods[0],mods[1],exps[0],exps[1],headerf1,headerf2]
2912
2913	Ndiffop = Ndiffop + 1
2914	else:
2915	Ndiffop = 0
2916
2917	diffvars = {}
2918	if diffvar is not None:
2919	# Getting 'var' differences characteristics
2920	doops,opvar,indivopvar,varglobalp = get_differences_var(diffvar,'var',debug)
2921
2922	Ndiffvar = 0
2923	for vn in opvar.keys():
2924	vnops = opvar[vn]
2925	for op in vnops:
2926	vnop = vn + '_' + op
2927	if not allcompvar1.has_key(vnop):
2928	print warnmsg
2929	print ' ' + fname + ": DIFFVAR variable+operation '" + vnop + \
2930	"' in '" + mods[0] + '/' + exps[0] + "' was not computed !!"
2931	print ' skipping it!'
2932	print ' variables computed:', allcompvar1
2933	break
2934	if not allcompvar2.has_key(vnop):
2935	print warnmsg
2936	print ' ' + fname + ": DIFFVAR variable+operation '" + vnop + \
2937	"' in '" + mods[1] + '/' + exps[1] + "' was not computed !!"
2938	print ' skipping it!'
2939	print ' variables computed:', allcompvar2
2940	break
2941	vals1 = allcompvar1[vnop]
2942	vals2 = allcompvar2[vnop]
2943
2944	headerf1 = vals1[0]
2945	headerf2 = vals2[0]
2946
2947	diffvars[vnop] = [mods[0],mods[1],exps[0],exps[1],headerf1,headerf2]
2948
2949	Ndiffvar = Ndiffvar + 1
2950	else:
2951	Ndiffvar = 0
2952
2953	if diffop is not None:
2954	Sopdiffs = gen.dictKeysVals_stringList(diffops)
2955	else:
2956	Sopdiffs = 'none'
2957	# Outwritting the op diffferences file to avoid next time
2958	objf = open(odir + '/diffop.inf', 'w')
2959	objf.write('differences: ' + Sopdiffs + '\n')
2960	objf.write('Ndiff: ' + str(Ndiffop) + '\n')
2961	objf.close()
2962
2963	if diffvar is not None:
2964	Svardiffs = gen.dictKeysVals_stringList(diffvars)
2965	else:
2966	Svardiffs = 'none'
2967	# Outwritting the var diffferences file to avoid next time
2968	objf = open(odir + '/diffvar.inf', 'w')
2969	objf.write('differences: ' + Svardiffs + '\n')
2970	objf.write('Ndiff: ' + str(Ndiffvar) + '\n')
2971	objf.close()
2972
2973	if debug: print " End of '" + fname + "' "
2974
2975	return diffops, diffvars, Ndiffop, Ndiffvar
2976
2977	def read_diff_file(difffile):
2978	""" Function to read the file with the information about the differences
2979	difffile= file with the information
2980	"""
2981	fname = 'read_diff_file'
2982
2983	if not os.path.isfile(difffile):
2984	print errormsg
2985	print ' ' + fname + ": differences file '" + difffile + "' does not exist !!"
2986	quit(-1)
2987
2988	objf = open(difffile, 'r')
2989	for line in objf:
2990	if line[0:1] != '#' and len(line) > 1:
2991	values = line.replace('\n','').split(' ')
2992	if values[0] == 'differences:':
2993	if values[1] != 'none':
2994	diffs = gen.stringList_dictKeysVals(values[1])
2995	else:
2996	diffs = {}
2997	elif values[0] == 'Ndiff:':
2998	Ndiffs = int(values[1])
2999
3000	objf.close()
3001
3002	return diffs, Ndiffs
3003
3004	def compute_op_diffs(config, diffallop, odir, diffscr, debug):
3005	""" Function to compute operation differences
3006	config= experiment configuration
3007	diffallop= dictonary with the differences to compute
3008	odir= output folder
3009	diffscr= wether it should be done from the scratch
3010	"""
3011	fname = 'compute_op_diffs'
3012
3013	# output folder
3014	ofold = config['ofold']
3015
3016	# Home of the python scripts
3017	pyH = config['pyHOME']
3018
3019	# opsur = operation surnames
3020	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
3021	Opsurs = {}
3022	for opk in opsur:
3023	opn = opk.split('_')[1]
3024	vals = opsur[opk]
3025	Opsurs[opn] = vals.split(':')
3026
3027	# Getting in working dir
3028	os.chdir(odir)
3029
3030	# CF dimensions
3031	CFdims = ['lon', 'lat', 'pres', 'time']
3032
3033	# Trakcing file
3034	trkobjf = open(odir + '/all_diffop.inf', 'a')
3035
3036	for vnop in diffallop.keys():
3037	vn = vnop.split('_')[0]
3038	op = vnop.split('_')[1]
3039	opS = op.replace('+','_')
3040
3041	# `p' header add related to 'pinterp'
3042	if opS.find('pinterp') != -1: SgP = 'p'
3043	else: SgP = ''
3044
3045	# Variable name within the file
3046	vninF = varnoper(vn, op, Opsurs)
3047
3048	vals = diffallop[vnop]
3049	mod1 = vals[0]
3050	mod2 = vals[1]
3051	exp1 = vals[2]
3052	exp2 = vals[3]
3053	headerf1 = vals[4]
3054	headerf2 = vals[5]
3055	modexpdiff = mod2 + '/' + exp2 + ' - ' + mod1 + '/' + exp1
3056	modexpdiffS = mod2 + '-' + exp2 + '_' + mod1 + '-' + exp1
3057
3058	ifile1 = ofold + '/' + mod1 + '/' + exp1 + '/' + vn + '_' + headerf1 + SgP + \
3059	'_' + opS + '.nc'
3060	ifile2 = ofold + '/' + mod2 + '/' + exp2 + '/' + vn + '_' + headerf2 + SgP + \
3061	'_' + opS + '.nc'
3062	difffilen = odir + '/' + vn + '_diffop_' + modexpdiffS + '_' + opS + '.nc'
3063
3064	if diffscr:
3065	sout = sub.call('rm ' + difffilen + ' >& /dev/null', shell=True)
3066
3067	if not os.path.isfile(difffilen):
3068	if debug:
3069	print " Computing operation difference '" + op + "' of '" + vn + \
3070	"' for: '" + modexpdiff
3071	print " var in file: '" + vninF + "'"
3072
3073	# Range of the dimensions
3074	for CFd in CFdims:
3075	if CFd == CFdims[0]:
3076	CFdimS = CFd + '\|' + CFd + '\|-1'
3077	else:
3078	CFdimS = CFdimS + ';' + CFd + '\|' + CFd + '\|-1'
3079
3080	ncvar.ivattrs(ifile2,vninF)
3081
3082	# Attributes of the variable
3083	try:
3084	with gen.Capturing() as output:
3085	varattrs = ncvar.ivattrs(ifile2,vninF)
3086	except:
3087	print errmsg
3088	print 'ncvar.ivarattrs(' + ifile2 + ',' + vninF + ')'
3089	for s1out in output: print s1out
3090	quit(-1)
3091
3092	if debug:
3093	for s1out in output: print s1out
3094	varvals = vninF + ',' + varattrs['long_name'][0] + ',' + \
3095	varattrs['units'][0]
3096
3097	values = CFdimS + '@' + 'add\|' + ifile2 + '\|' + vninF + '%' + CFdimS + \
3098	'@' + 'sub\|' + ifile1 + '\|' + vninF
3099	pyins = 'python ' + pyH + "/nc_var.py -o compute_opersvarsfiles -S '" + \
3100	values + "' -v '" + varvals + "'"
3101
3102	try:
3103	with gen.Capturing() as output:
3104	ncvar.compute_opersvarsfiles(values,varvals)
3105	except:
3106	print errmsg
3107	print 'ncvar.compute_opersvarsfiles(' + values + ',' + varvals + ')'
3108	for s1out in output: print s1out
3109	quit(-1)
3110
3111	sout = sub.call('mv opersvarsfiles_'+vninF+'.nc ' + difffilen, shell=True)
3112
3113	# keeping all differences
3114	trkobjf.write('\n')
3115	trkobjf.write("#" + vn + ' ' + op + '\n')
3116	trkobjf.write( pyins + '\n')
3117
3118	trkobjf.close()
3119	if debug: print " End of '" + fname + "' "
3120
3121	return
3122
3123	def compute_diff_statistics(Ecnf, filen, ofold, varn, Opers, scr, db):
3124	""" Function to compute statistics from var diff files
3125	Ecnf= dictionary with the configuration of the experiment
3126	filen= variable difference file to use
3127	ofold= directory to write the output files
3128	varn= name of the variable
3129	Opers= kind of operation: (as possible multiple consecutive combination of operations separated by '+'
3130	[calc1]+[calc2] will compute first [calc1] and then [calc2]
3131	acc: temporal accumulated values
3132	diff: differences between models
3133	dimvarvalue~[dimvarn]~[value]: Slicing along the index at the nearest given value of a dimension-variable
3134	direct: no statistics
3135	last: last temporal value
3136	Lmean: latitudinal mean values
3137	Lsec: latitudinal section (latitudinal value must be given, [var]@[lat])
3138	lmean: longitudinal mean values
3139	lsec: longitudinal section (longitudinal value must be given, [var]@[lat])
3140	pinterp: pressure interpolation (to the given $plevels)
3141	tmean: temporal mean values
3142	tstd: temporal standard deviation values
3143	tturb: Taylor's turbulence decomposition value (x - <x>) for time
3144	tvar: temporal variance values
3145	xmean: x-axis mean values
3146	xvar: x-axis variance values
3147	ymean: y-axis mean values
3148	zsum: vertical aggregated values
3149	scr= should it be done from the scratch?
3150	"""
3151	fname='compute_diff_statistics'
3152
3153	# Full header of file
3154	Lfilen = len(filen)
3155	Hfile = filen[0:Lfilen-3]
3156
3157	# CF dimension-variables name
3158	OpersS = '+'.join(Opers)
3159	if OpersS.find('pinterp') != -1 or filen.find('pinterp') != -1:
3160	varnCFs = ['lon', 'lat', 'pres', 'time']
3161	else:
3162	varnCFs = ['lon', 'lat', 'time']
3163
3164	# Variable-dimension CF dictionary
3165	CFdimvardict = {'lon': 'lon', 'lat': 'lat', 'pres': 'pres', 'time': 'time'}
3166
3167	# Python scripts HOME
3168	pyH = Ecnf['pyHOME']
3169
3170	# opsur = operation surnames
3171	opsur = gen.get_specdictionary_HMT(Ecnf, H='opsur_',M='',T='')
3172	Opsurs = {}
3173	for opk in opsur:
3174	opn = opk.split('_')[1]
3175	vals = opsur[opk]
3176	Opsurs[opn] = vals.split(':')
3177
3178	# Getting in working dir
3179	os.chdir(ofold)
3180
3181	# File to keep track of all the statistics of var differences
3182	otrackf = open(ofold + '/all_vardiffstatistics.inf', 'a')
3183
3184	if db:
3185	print " computing statistics of diff variable: '", varn ,"' operation '"+ \
3186	OpersS + "' using file '" + filen + " ...'"
3187
3188	for op in Opers:
3189	# File name from previous operations, and name of the variable within the
3190	# file (some operations change the name of the variable)
3191	if op == Opers[0]:
3192	Fopers = op
3193	prevfile = filen
3194	vninF = varn
3195	else:
3196	Fopers = Fopers + '_' + op
3197	prevfile = fileon
3198	fileon = Hfile + '_' + Fopers + '.nc'
3199
3200	SvarnCFs = ',' + ','.join(varnCFs)
3201	CFvarnp = vninF + SvarnCFs
3202
3203	if scr:
3204	sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
3205
3206	# Just produce file in case output file does not exist
3207	if not os.path.isfile(fileon):
3208	if db:
3209	print ' ' + fname + ": creation of file '" + fileon + "' ..."
3210	dofile = True
3211	else:
3212	dofile = False
3213
3214	# Variables to be kept in the final file
3215	varkeep = []
3216
3217	if op == 'acc':
3218	# temporal accumulated values
3219	print " " + fname + ": kind '" + op + "' not ready !!"
3220	wrongstats.append(varn + '_' + opers)
3221	break
3222
3223	elif op[0:11] == 'dimvarvalue':
3224	# Slicing along the index at the nearest given value of a dimension-variable
3225	dimvarn = op.split('~')[1]
3226	dimvalue = op.split('~')[2]
3227	vals = dimvarn + ',' + dimvalue + ',' + dimvalue + ',0'
3228	if dofile:
3229	try:
3230	with gen.Capturing() as output:
3231	ncvar.DataSetSection_multivars(vals,prevfile,CFvarnp)
3232	except:
3233	print errmsg
3234	print 'DataSetSection_multivars('+vals+',', prevfile, ',' + \
3235	CFvarnp + ')'
3236	for s1out in output: print s1out
3237	quit(-1)
3238
3239	if db:
3240	for s1out in output: print s1out
3241
3242	sout = sub.call('mv DataSetSection_multivars.nc '+fileon, shell=True)
3243
3244	# Keeping the operations
3245	pyins=pyH + "/nc_var.py -o DataSetSection_multivars -S '" + vals + \
3246	"' -f " + prevfile + " -v " + CFvarnp
3247	otrackf.write("\n")
3248	otrackf.write("# " + CFvarnp + " " + Fopers + "\n")
3249	otrackf.write(pyins + "\n")
3250
3251	# removing the given variable-dimension
3252	varnCFs.remove(dimvarn)
3253
3254	elif op == 'direct':
3255	# no statistics
3256	if dofile:
3257	sout = sub.call('mv ' + prevfile + ' ' + fileon, shell=True)
3258	elif op == 'last':
3259	# last temporal value
3260	vals='time,-9,0,0'
3261	if dofile:
3262	try:
3263	with gen.Capturing() as output:
3264	ncvar.DataSetSection(vals,prevfile)
3265	except:
3266	print errmsg
3267	print 'DataSetSection('+vals+',', prevfile, ')'
3268	for s1out in output: print s1out
3269	quit(-1)
3270
3271	if db:
3272	for s1out in output: print s1out
3273
3274	hprevfile = prevfile[0:len(prevfile)-3]
3275
3276	sout = sub.call('mv ' + hprevfile + '_time_B-9-E0-I0.nc ' + fileon, \
3277	shell=True)
3278
3279	# Keeping the operations
3280	pyins=pyH + "/nc_var.py -o DataSetSection -S '" + vals + "' -f " + \
3281	prevfile
3282	otrackf.write("\n")
3283	otrackf.write("# " + varn + " " + Fopers + "\n")
3284	otrackf.write(pyins + "\n")
3285
3286	elif op =='Lmean':
3287	# latitudinal mean values
3288	print " " + fname + ": kind '" + op + "' not ready !!"
3289	wrongstats.append(varn + '_' + opers)
3290	break
3291	elif op =='Lsec':
3292	# latitudinal section (latitudinal value must be given, [var]@[lat])
3293	print " " + fname + ": kind '" + op + "' not ready !!"
3294	wrongstats.append(varn + '_' + opers)
3295	break
3296	elif op =='lmean':
3297	# longitudinal mean values
3298	print " " + fname + ": kind '" + op + "' not ready !!"
3299	wrongstats.append(varn + '_' + opers)
3300	break
3301	elif op =='lsec':
3302	# longitudinal section (longitudinal value must be given, [var]@[lon])
3303	print " " + fname + ": kind '" + op + "' not ready !!"
3304	wrongstats.append(varn + '_' + opers)
3305	break
3306	elif op == 'tmean':
3307	# temporal mean values
3308	vals='time\|-1,time,mean,' + ':'.join(varnCFs)
3309	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3310	if dofile:
3311	try:
3312	with gen.Capturing() as output:
3313	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3314	except:
3315	print errmsg
3316	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3317	for s1out in output: print s1out
3318	quit(-1)
3319
3320	if db:
3321	for s1out in output: print s1out
3322
3323	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
3324
3325	# Keeping the operations
3326	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3327	"' -f " + prevfile + " -v " + CFvarnp
3328	otrackf.write("\n")
3329	otrackf.write("# " + varn + " " + Fopers + "\n")
3330	otrackf.write(pyins + "\n")
3331
3332	# removing dimension variable-dimension 'time'
3333	varnCFs.remove('time')
3334
3335	varkeep.append('timestats')
3336
3337	elif op == 'tstd':
3338	# temporal standard deviation values
3339	vals='time\|-1,time,std,' + ':'.join(varnCFs) + ':' + vdnz
3340	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
3341	if dofile:
3342	try:
3343	with gen.Capturing() as output:
3344	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3345	except:
3346	print errmsg
3347	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3348	for s1out in output: print s1out
3349	quit(-1)
3350
3351	if db:
3352	for s1out in output: print s1out
3353
3354	sout = sub.call('mv file_oper_alongdims_std.nc '+fileon, shell=True)
3355
3356	# Keeping the operations
3357	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3358	"' -f " + prevfile + " -v " + CFvarnp
3359	otrackf.write("\n")
3360	otrackf.write("# " + varn + " " + Fopers + "\n")
3361	otrackf.write(pyins + "\n")
3362
3363	# removing dimension variable-dimension 'time'
3364	varnCFs.remove('time')
3365
3366	varkeep.append('timestats')
3367
3368	elif op == 'tturb':
3369	# temporal turbulent values
3370	vals='time\|-1,time,turb,' + ':'.join(varnCFs)
3371	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3372	if dofile:
3373	try:
3374	with gen.Capturing() as output:
3375	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3376	except:
3377	print errmsg
3378	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3379	for s1out in output: print s1out
3380	quit(-1)
3381
3382	if db:
3383	for s1out in output: print s1out
3384
3385	sout = sub.call('mv file_oper_alongdims_turb.nc '+fileon, shell=True)
3386
3387	# Keeping the operations
3388	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3389	"' -f " + prevfile + " -v " + CFvarnp
3390	otrackf.write("\n")
3391	otrackf.write("# " + varn + " " + Fopers + "\n")
3392	otrackf.write(pyins + "\n")
3393
3394	varkeep.append('timestats')
3395
3396	elif op == 'tvar':
3397	# temporal variance values (Taylor's turbulence)
3398	vals='time\|-1,time,var,' + ':'.join(varnCFs) + ':' + vdnz
3399	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
3400	if dofile:
3401	try:
3402	with gen.Capturing() as output:
3403	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3404	except:
3405	print errmsg
3406	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3407	for s1out in output: print s1out
3408	quit(-1)
3409
3410	if db:
3411	for s1out in output: print s1out
3412
3413	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
3414
3415	# Keeping the operations
3416	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3417	"' -f " + prevfile + " -v " + CFvarnp
3418	otrackf.write("\n")
3419	otrackf.write("# " + varn + " " + Fopers + "\n")
3420	otrackf.write(pyins + "\n")
3421
3422	# removing dimension variable-dimension 'time'
3423	varnCFs.remove('time')
3424
3425	varkeep.append('timestats')
3426
3427	elif op == 'xmean':
3428	# x-axis mean values
3429	vals='lon\|-1,lon,mean,' + ':'.join(varnCFs)
3430	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3431	if dofile:
3432	try:
3433	with gen.Capturing() as output:
3434	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3435	except:
3436	print errmsg
3437	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3438	for s1out in output: print s1out
3439	quit(-1)
3440
3441	if db:
3442	for s1out in output: print s1out
3443
3444	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
3445
3446	# Keeping the operations
3447	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3448	"' -f " + prevfile + " -v " + CFvarnp
3449	otrackf.write("\n")
3450	otrackf.write("# " + varn + " " + Fopers + "\n")
3451	otrackf.write(pyins + "\n")
3452
3453	# removing dimension variable-dimension 'lon'
3454	varnCFs.remove('lon')
3455
3456	varkeep.append('lonstats')
3457
3458	elif op == 'xvar':
3459	# x-axis variance values
3460	vals='lon\|-1,lon,var,' + ':'.join(varnCFs)
3461	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3462	if dofile:
3463	try:
3464	with gen.Capturing() as output:
3465	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3466	except:
3467	print errmsg
3468	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3469	for s1out in output: print s1out
3470	quit(-1)
3471
3472	if db:
3473	for s1out in output: print s1out
3474
3475	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
3476
3477	# Keeping the operations
3478	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3479	"' -f " + prevfile + " -v " + CFvarnp
3480	otrackf.write("\n")
3481	otrackf.write("# " + varn + " " + Fopers + "\n")
3482	otrackf.write(pyins + "\n")
3483
3484	# removing dimension variable-dimension 'lon'
3485	varnCFs.remove('lon')
3486
3487	varkeep.append('lonstats')
3488
3489	elif op == 'ymean':
3490	# y-axis mean values
3491	vals='lat\|-1,lat,mean,' + ':'.join(varnCFs)
3492	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3493	if dofile:
3494	try:
3495	with gen.Capturing() as output:
3496	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3497	except:
3498	print errmsg
3499	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3500	for s1out in output: print s1out
3501	quit(-1)
3502
3503	if db:
3504	for s1out in output: print s1out
3505
3506	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
3507
3508	# Keeping the operations
3509	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3510	"' -f " + prevfile + " -v " + CFvarnp
3511	otrackf.write("\n")
3512	otrackf.write("# " + varn + " " + Fopers + "\n")
3513	otrackf.write(pyins + "\n")
3514
3515	# removing dimension variable-dimension 'lat'
3516	varnCFs.remove('lat')
3517
3518	varkeep.append('latstats')
3519
3520	elif op == 'zsum':
3521	# vertical aggregated values
3522	print " " + fname + ": kind '" + op + "' not ready !!"
3523	wrongstats.append(varn + '_' + opers)
3524	break
3525	else:
3526	print errmsg
3527	print ' ' + fname + ": operation '" + op + "' not ready !!"
3528	quit(-1)
3529
3530	# End of kind of operation
3531
3532	# Variable name in file (vninF) changed due to operation
3533	# but only if previous operation does not the same 'statistic'
3534	chvn = gen.dictionary_key_list(Opsurs, op)
3535	if chvn is not None:
3536	oldvninF = vninF
3537	vninF = vninF + chvn
3538	CFvarnp = CFvarnp.replace(oldvninF,vninF)
3539
3540	if dofile:
3541	if len(varkeep) > 0:
3542	varkeepS = ',' + ','.join(varkeep)
3543	else:
3544	varkeepS = ''
3545
3546	# Adding such CF dimension variables which might not be in the
3547	# post-operation file
3548	try:
3549	with gen.Capturing() as output:
3550	varinfile = ncvar.ivars(fileon)
3551	except:
3552	print errmsg
3553	print 'ivars('+fileon+')'
3554	for s1out in output: print s1out
3555	# Removing file in order to make sure that it will be redone
3556	print ' ' + fname + " removing file '" + fileon + "' ..."
3557	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
3558	quit(-1)
3559
3560	for CFvn in varnCFs:
3561	if not gen.searchInlist(varinfile, CFvn):
3562	if db:
3563	print ' ' + fname + ": recupering CF variable '" + CFvn + "'"
3564	try:
3565	with gen.Capturing() as output:
3566	ncvar.fvaradd(prevfile+','+CFvn, fileon)
3567	except:
3568	print errmsg
3569	print 'fvaradd(' + prevfile + ',' + CFvn +', ' + fileon + ')'
3570	for s1out in output: print s1out
3571	# Removing file in order to make sure that it will be redone
3572	print ' ' + fname + " removing file '" + fileon + "' ..."
3573	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
3574	quit(-1)
3575
3576	if db:
3577	for s1out in output: print s1out
3578
3579	totalvarkeeps = CFvarnp + ',' + ','.join(varnCFs) + varkeepS
3580	try:
3581	with gen.Capturing() as output:
3582	oclean = ncvar.cleaning_varsfile(totalvarkeeps,fileon)
3583	except:
3584	print errmsg
3585	print 'cleaning_varsfile('+totalvarkeeps+','+fileon+')'
3586	for s1out in output: print s1out
3587	# Removing file in order to make sure that it will be redone
3588	print ' ' + fname + " removing file '" + fileon + "' ..."
3589	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
3590	quit(-1)
3591
3592	if db:
3593	for s1out in output: print s1out
3594
3595	# End of operations
3596	otrackf.close()
3597
3598	if db: print " End of '" + fname + "' "
3599
3600	return
3601
3602	def compute_var_diffs(config, diffallvar, odir, diffscr, debug):
3603	""" Function to compute variable differences
3604	config= experiment configuration
3605	diffallvar= dictonary with the differences to compute
3606	odir= output folder
3607	diffscr= wether it should be done from the scratch
3608	"""
3609	fname = 'compute_var_diffs'
3610
3611	# output folder
3612	ofold = config['ofold']
3613
3614	# Home of the python scripts
3615	pyH = config['pyHOME']
3616
3617	# opsur = operation surnames
3618	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
3619	Opsurs = {}
3620	for opk in opsur:
3621	opn = opk.split('_')[1]
3622	vals = opsur[opk]
3623	Opsurs[opn] = vals.split(':')
3624
3625	# Getting in working dir
3626	os.chdir(odir)
3627
3628	# CF dimensions
3629	CFdims = ['lon', 'lat', 'pres', 'time']
3630
3631	# Trakcing file
3632	trkobjf = open(odir + '/all_diffvar.inf', 'a')
3633
3634	for vnop in diffallvar.keys():
3635	vn = vnop.split('_')[0]
3636	op = vnop.split('_')[1]
3637	opSfinal = op.replace('+','_')
3638
3639	lop = op.split('+')
3640	# `p' header add related to 'pinterp'
3641	if op.find('pinterp') != -1:
3642	print warnmsg
3643	print ' ' + fname + ': there is the vertical interpolation as operation'
3644	print ' variable differences will be computed from that files where '
3645	print ' vertical interpolation was just done'
3646	ipop = lop.index('pinterp')
3647	opS = '_' + '_'.join(lop[0:ipop+1])
3648	doops = lop[ipop+1:]
3649	SgP = 'p'
3650	vninF = varnoper(vn, '+'.join(lop[0:ipop+1]), Opsurs)
3651	print ' operations to perform:', doops
3652	else:
3653	SgP = ''
3654	opS = ''
3655	doops = lop
3656	vninF = vn
3657
3658	Sdoops = '+'.join(doops)
3659
3660	# Variable name within the file
3661
3662	vals = diffallvar[vnop]
3663	mod1 = vals[0]
3664	mod2 = vals[1]
3665	exp1 = vals[2]
3666	exp2 = vals[3]
3667	headerf1 = vals[4]
3668	headerf2 = vals[5]
3669	modexpdiff = mod2 + '/' + exp2 + ' - ' + mod1 + '/' + exp1
3670	modexpdiffS = mod2 + '-' + exp2 + '_' + mod1 + '-' + exp1
3671
3672	ifile1 = ofold + '/' + mod1 + '/' + exp1 + '/' + vn + '_' + headerf1 + SgP + \
3673	opS + '.nc'
3674	ifile2 = ofold + '/' + mod2 + '/' + exp2 + '/' + vn + '_' + headerf2 + SgP + \
3675	opS + '.nc'
3676	difffilen = odir + '/' + vn + '_diffvar_' + modexpdiffS + opS + '.nc'
3677	difffinalfilen = odir + '/' + vn + '_diffvar_' + modexpdiffS + opS + '_' + \
3678	Sdoops + '.nc'
3679
3680	if diffscr:
3681	sout = sub.call('rm ' + difffilen + ' >& /dev/null', shell=True)
3682	sout = sub.call('rm ' + difffinalfilen + ' >& /dev/null', shell=True)
3683
3684	if not os.path.isfile(difffinalfilen):
3685	if debug:
3686	print " Computing variable difference of '" + vn + "' for: '" + \
3687	modexpdiff + "' and performing operation '" + Sdoops + "'"
3688	print " var in file: '" + vninF + "'"
3689
3690	# Range of the dimensions
3691	for CFd in CFdims:
3692	if CFd == CFdims[0]:
3693	CFdimS = CFd + '\|' + CFd + '\|-1'
3694	else:
3695	CFdimS = CFdimS + ';' + CFd + '\|' + CFd + '\|-1'
3696
3697	# Attributes of the variable
3698	with gen.Capturing() as output:
3699	varattrs = ncvar.ivattrs(ifile2,vninF)
3700	if debug:
3701	for s1out in output: print s1out
3702	varvals = vninF + ',' + varattrs['long_name'][0] + ',' + \
3703	varattrs['units'][0]
3704
3705	values = CFdimS + '@' + 'add\|' + ifile2 + '\|' + vninF + '%' + CFdimS + \
3706	'@' + 'sub\|' + ifile1 + '\|' + vninF
3707	pyins = 'python ' + pyH + "/nc_var.py -o compute_opersvarsfiles -S '" + \
3708	values + "' -v '" + varvals + "'"
3709
3710	try:
3711	with gen.Capturing() as output:
3712	ncvar.compute_opersvarsfiles(values,varvals)
3713	except:
3714	print errmsg
3715	print 'ncvar.compute_opersvarsfiles(' + values + ', ' + varvals + ')'
3716	for s1out in output: print s1out
3717	quit(-1)
3718
3719	sout = sub.call('mv opersvarsfiles_'+vninF+'.nc ' + difffilen, shell=True)
3720
3721	# keeping all differences
3722	trkobjf.write('\n')
3723	trkobjf.write("#" + vn + ' ' + op + '\n')
3724	trkobjf.write( pyins + '\n')
3725
3726	# Computing statisitcs
3727	compute_diff_statistics(config, difffilen, odir, vninF, doops, diffscr, \
3728	debug)
3729
3730	trkobjf.close()
3731
3732	if debug: print " End of '" + fname + "' "
3733
3734	return
3735
3736
3737	def draw_diff_plots(config, plots, odir, allvarcomp, kdiff, figscr, debug):
3738	""" Function to draw all plots
3739	config= Configuration of the experiment
3740	plots= dictionary with the plots
3741	odir= output experiment folder
3742	allvarcomp= dictionary with all the variables to compute and their information
3743	kdiff= kind of differences:
3744	'diffop': plots from operation differences
3745	'diffvar': plots from variable differences
3746	figscr= whether figures should be done from the scratch or not
3747
3748	* Plot as
3749	{[kplot]} = [varn1]\|[op1]#[varn2]\|[op2]#[...[varnN]\|[opN]], ...
3750	[kplot] ___
3751	diffmap2Dsfc: 2D map of surface differences values of 1 variable
3752	diffmap2Dz: 2D map of 3D differences values of 1 variable
3753	map2Dsfc: 2D map of surface values of 1 variable
3754	map3D: 2D map of 3D values of 1 variable
3755	shadconthovmsfc: Hovmoeller diagrams of 2 variable at the surface in shadow and the other in contourn
3756	shadcont2Dsfc: 2D map of shadow (1st variable) and countour (2nd variable) [stvar1]#[stvar2]
3757	shadcont2Dzsec: 2D map of vertical section of 2 variables one in shadow and the other in contourn
3758	[varn]
3759	variable
3760	[op]
3761	'+' separated list of operations
3762	in figures with more than 1 variable, use '#' to separate the [varn]\|[op]
3763	"""
3764	fname = 'draw_diff_plots'
3765
3766	os.chdir(odir)
3767
3768	# Dictionary with the operations with surnames for the operated variable
3769	opersurnames = {}
3770	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
3771	for opsr in opsur.keys():
3772	opn = opsr.split('_')[1]
3773	vls = opsur[opsr].split(':')
3774	opersurnames[opn] = vls
3775
3776	# time values
3777	# Units time for the plots
3778	rd = config['CFreftime']
3779	tunits = config['CFunitstime'] + '!since!' + rd[0:4] + '-' + rd[4:6] + '-' + \
3780	rd[6:8] + '!' + rd[8:10] + ':' + rd[10:12] + ':' + rd[12:14]
3781	# time ticks kind
3782	tkind = config['timekind']
3783	# time ticks format
3784	tfmt = config['timefmt']
3785	# time axis label
3786	tlab = config['timelabel']
3787	timevals = [tunits, tkind, tfmt, tlab]
3788
3789	if kdiff == 'diffop':
3790	specplotkeyn = 'specificdiffopplot'
3791	elif kdiff == 'diffvar':
3792	specplotkeyn = 'specificdiffvarplot'
3793	else:
3794	print errmsg
3795	print ' ' + fname + ": differences kind '" + kdiff + "' not ready !!"
3796	quit(-1)
3797
3798	# Dictionary of plot specificities
3799	plotspecifics = {}
3800	if config.has_key(specplotkeyn):
3801	# [minval]: minimum value
3802	# [maxval]: minimum value
3803	# [colorbar]: name of the colorbar (from matplotlib) to use
3804	# [cntformat]: format of the contour labels
3805	# [colorcnt]: color for the countor lines
3806	plotspecs = config[specplotkeyn].split(':')
3807	for pltspc in plotspecs:
3808	pltvls = pltspc.split('\|')
3809	vn = pltvls[0]
3810	op = pltvls[1]
3811	fn = pltvls[2]
3812	plotspecifics[fn + '_' + vn + '_' + op] = pltvls[3:]
3813	if debug:
3814	print 'Specific values for plots _______'
3815	gen.printing_dictionary(plotspecifics)
3816
3817	# Kind of figures
3818	kindfigure = config['kindfig']
3819
3820	# Graphical labels and configuration
3821	modgc, expgc = graphical_characteristics(config, debug)
3822
3823	# Map value
3824	mapvalue = config['mapval']
3825
3826	# pythone scripts HOME
3827	pyHOME = config['pyHOME']
3828
3829	# Title-text of operations
3830	opexplained = {}
3831	optits = config['titleoperations'].split(':')
3832	for optit in optits:
3833	opn = optit.split('\|')[0]
3834	opt = optit.split('\|')[1]
3835	opexplained[opn] = opt
3836	if debug:
3837	print 'Titles for operations _______'
3838	gen.printing_dictionary(opexplained)
3839
3840	for kplot in plots.keys():
3841	varsplt = plots[kplot]
3842	for varplt in varsplt:
3843	if debug:
3844	print " printing '" + kplot + "' var ':" + varplt + "'..."
3845	varops = varplt.split('#')
3846
3847	# CF variables in plot
3848	CFvarsplot = []
3849	# Files in plot
3850	filesplot = []
3851	# Variables in plot within the files
3852	varsplot = []
3853	# Dims in figure
3854	dimsplot = []
3855	# pictoric values in figure
3856	pictplot = []
3857	# Name of the figure
3858	figname = ''
3859	# Title of the figure
3860	titfigure = ''
3861
3862	ivp = 0
3863	for varop in varops:
3864	vn = varop.split('\|')[0]
3865	op = varop.split('\|')[1]
3866
3867	# CF variables in plot
3868	CFvarsplot.append(vn)
3869
3870	vnopS = vn + '_' + op
3871	if not allvarcomp.has_key(vnopS):
3872	print errmsg
3873	print ' ' + fname + ": no file for variable-operation '" + \
3874	vnopS + "' !!"
3875	print ' available ones:', allvarcomp.keys()
3876	quit(-1)
3877	vopvals = allvarcomp[vnopS]
3878	mod1 = vopvals[0]
3879	mod2 = vopvals[1]
3880	mod3 = vopvals[2]
3881	mod4 = vopvals[3]
3882	headf = vopvals[4]
3883
3884	modv = modgc[mod1]
3885	expv = expgc[mod1+'/'+exp1]
3886	modlab1 = modv.label
3887	explab1 = expv.label
3888	modv = modgc[mod2]
3889	expv = expgc[mod2+'/'+exp2]
3890	modlab2 = modv.label
3891	explab2 = expv.label
3892
3893	modexpdiff = explab2 + '-' + explab1
3894	modexpdiffS = mod2 + '-' + exp2 + '_' + mod1 + '-' + exp1
3895
3896	difffilen = odir + '/' + vn + '_' + kdiff + '_' + modexpdiffS + \
3897	'_' + op.replace('+','_') + '.nc'
3898
3899	filesplot.append(difffilen)
3900	# Do we have processed the given difference?
3901	if not os.path.isfile(difffilen):
3902	print warnmsg
3903	print " " + fname + ": there is no file for '" + kdiff + \
3904	"' difference '" + varop + "' skiping it !!"
3905	break
3906
3907	# Name of the variable inside the file
3908	vnsur = varnoper(vn, op, opersurnames)
3909	varsplot.append(vnsur)
3910
3911	# Dimensions in file
3912	try:
3913	with gen.Capturing() as output:
3914	dims = ncvar.idims(difffilen)
3915	except:
3916	print errmsg
3917	print 'ncvar.idims('+difffilen+')'
3918	for s1out in output: print s1out
3919	quit(-1)
3920
3921	dimsplot.append(dims)
3922
3923	# pictoric values for the figure
3924	Sfivaop = kplot + '_' + vn + '_' + op
3925	if plotspecifics.has_key(Sfivaop):
3926	pictvals = plotspecifics[Sfivaop]
3927	else:
3928	Vvals = gen.variables_values(vn)
3929	pictvals = [Vvals[2], Vvals[3], Vvals[6], '%g', 'fixsigc','black']
3930
3931	pictplot.append(pictvals)
3932
3933	# Header of the name of the figure
3934	if ivp == 0:
3935	figname = kplot + '_' + vn + '_' + kdiff + '_' + modexpdiffS + \
3936	'_' + op.replace('+','-')
3937	else:
3938	figname = figname + '-' + vn + '_' + op.replace('+','-')
3939
3940	# Title of the figure:
3941	if mod1 == mod2:
3942	modexptit = mod1 + '!' + exp2 + '-' + exp1
3943	else:
3944	modexptit = modexpdiff
3945
3946	ivp = ivp + 1
3947	# End of variable-operation
3948	figname = figname + '.' + kindfigure
3949
3950	# Title of figure
3951	titfigure = create_figure_title(modexptit, kdiff, varops, opexplained)
3952
3953	if len(titfigure) > 80:
3954	print warnmsg
3955	print ' ' + fname + ": figure title '" + titfigure.replace('!', ' ')+\
3956	"' larger than 80 characters (actually", len(titfigure), ') !!'
3957	print " simplifying it to '" + modexptit.replace('!',' ') + '!' + \
3958	kdiff + '!' + vnS + "'"
3959	titfigure = modexptit.replace(' ','!') + '!' + kdiff + '!' + vnS
3960
3961	draw_plot(kplot, CFvarsplot, filesplot, varsplot, dimsplot, pictplot, \
3962	figname, titfigure, kindfigure, mapvalue, timevals, expgc, odir, \
3963	pyHOME, figscr, debug)
3964
3965	# End of variables-operations
3966
3967	# End of kind of plots
3968
3969	if debug: print " End of '" + fname + "' "
3970
3971	return
3972
3973	def reproject_modexp(mod,exp,config,REPRJops,fscr,fadd,debug):
3974	""" Function to re-project and re-compute files from a given model/experiment
3975	mod= model
3976	exp= experiment
3977	config= configuration
3978	REPRJops= dictionary with the reprojection operators to use for reproject and for which variable
3979	# remapbil CDO: Bilinear interpolation
3980	# remapbic CDO: Bicubic interpolation
3981	# remapdis CDO: Distance-weighted average remapping
3982	# remapnn CDO: Nearest neighbor remapping
3983	# Operators only available if projections have the corners of the grid points.
3984	# remapcon CDO: First order conservative remapping
3985	# remapcon2 CDO: Second order conservative remapping
3986	# remaplaf CDO: Largest area fraction remapping
3987	####### ####### ####### or additionally ####### ####### #######
3988	# python remapping
3989	# dis python: Distance-weighted average remapping
3990	# pnn python: Nearest neighbor remapping
3991
3992	fscr= re-projected files to compute from the scratch
3993	fadd= re-projected files to be added
3994	"""
3995	fname = 'reproject_modexp'
3996
3997	odir = config['ofold'] + '/' + mod + '/' + exp
3998
3999	# NON re-projectable operations
4000	opNOreproj = config['NOreprojops'].split(':')
4001
4002	# Need to pass to analyze all the data?
4003	inffiles = ['reproj_varcompute.inf', 'all_reproj_computevars.inf']
4004	if fscr:
4005	for inff in inffiles:
4006	if debug: print " removing information file '" + inff + "' ..."
4007	ins = 'rm ' + odir + '/' + inff + ' >& /dev/null'
4008	sout = sub.call(ins, shell=True)
4009
4010	objf = open(odir+'/all_reproj_computevars.inf','w')
4011	objf.write("## Computation of reprojected variables \n")
4012	objf.close()
4013
4014	varcompf = odir + '/reproj_varcompute.inf'
4015	if fadd:
4016	sub.call('rm ' + varcompf +' >& /dev/null', shell=True)
4017
4018	# CF dims-vardims pairs:
4019	CFvardims = ['long@lon', 'lat@lat', 'pres@pres', 'time@time']
4020
4021	# Dictionary with the operations with surnames for the operated variable
4022	opersurnames = {}
4023	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
4024	for opsr in opsur.keys():
4025	opn = opsr.split('_')[1]
4026	vls = opsur[opsr].split(':')
4027	opersurnames[opn] = vls
4028
4029	if not os.path.isfile(varcompf):
4030	# Getting original computing information
4031	origvarcompf = odir + '/varcompute.inf'
4032	origfiles,origtestfiles,origallcompvar, Nvar = read_varcomp_file(origvarcompf)
4033
4034	Svarcompute = ''
4035	allcompvar = dict(origallcompvar)
4036	ivop = 0
4037	for vnop in origallcompvar.keys():
4038	vn = vnop.split('_')[0]
4039	op = vnop.split('_')[1]
4040	values = allcompvar[vnop]
4041	values[0] = 'reproj-' + values[0]
4042	allcompvar[vnop] = values
4043
4044	# Building strings from list values
4045	model = values[1]
4046	diag = values[2]
4047	if model is not None:
4048	if type(model) == type(list([1, 2])):
4049	Smodel = ':'.join(model)
4050	elif type(model) == type('A'):
4051	Smodel = model
4052	else:
4053	Smodel = 'None'
4054	if diag is not None:
4055	if type(diag) == type(list([1, 2])):
4056	Sdiag = ':'.join(diag)
4057	elif type(diag) == type('A'):
4058	Sdiag = diag
4059	else:
4060	Sdiag = 'None'
4061
4062	Svc = vn + '\|' + op + '\|' + values[0] + '\|' + Smodel + '\|' + Sdiag + \
4063	'\|' + values[3]
4064	if ivop == 0:
4065	Svarcompute = Svc
4066	else:
4067	Svarcompute = Svarcompute + ',' + Svc
4068	ivop = ivop + 1
4069
4070	origstrfiles = gen.dictKeysVals_stringList(origfiles)
4071	origstrtestfiles = gen.dictKeysVals_stringList(origtestfiles)
4072
4073	# Outwritting the varcompute to avoid next time (if it is not filescratch!)
4074	objf = open(odir + '/reproj_varcompute.inf', 'w')
4075	objf.write('files: ' + origstrfiles + '\n')
4076	objf.write('testfiles: ' + origstrtestfiles + '\n')
4077	objf.write('varcompute: ' + Svarcompute + '\n')
4078	objf.write('itotv: ' + str(ivop) + '\n')
4079	objf.close()
4080	else:
4081	if dbg:
4082	print warnmsg
4083	print ' ' + main + ": getting variables to compute already from file !!"
4084	origfiles, origtestfiles, allcompvar, Nvar = read_varcomp_file(varcompf)
4085
4086	# End of avoiding to repeat all the experiment search
4087
4088	print " For experiment '" + exp + "' is required to re-project:", Nvar, \
4089	"variables"
4090
4091	if dbg:
4092	print 'Variables to re-project _______'
4093	gen.printing_dictionary(allcompvar)
4094
4095	# Re-projection variables
4096	print " Reprojecting variables ..."
4097	os.chdir(odir)
4098
4099	# Keeping track of reprojection
4100	trckobjf = open(odir + '/all_reproj_computevars.inf','a')
4101
4102	for vnop in allcompvar.keys():
4103	vn = vnop.split('_')[0]
4104	op = vnop.split('_')[1]
4105	values = allcompvar[vnop]
4106	VnOpS = varnoper(vn, op, opersurnames)
4107
4108	# Only are reprojected that files which do contain 'lon' and 'lat',
4109	# otherways, it is only re-projected the original file
4110	inNO, iopNO, iop = gen.list_coincidences(opNOreproj,op.split('+'))
4111
4112	reprjop = gen.dictionary_key_list(REPRJops,vn).split('_')[1]
4113	if reprjop is None:
4114	print errmsg
4115	print ' ' + fname + ": no reprojecting operator found for " + \
4116	" variable '" + vn + "' !!"
4117	print ' available values _______'
4118	gen.printing_dictionary(REPRJops)
4119	quit(-1)
4120
4121	# Re-projecting also the 'original' (at least with 'pinterp') file for varsdif
4122	if op.find('pinterp') != -1:
4123	ifileorig = vn +'_'+ values[0].replace('reproj-','') + 'p_pinterp.nc'
4124	ofileorig = vn +'_'+ values[0] + 'p_pinterp.nc'
4125	ofilen = vn +'_'+ values[0] + 'p_pinterp.nc'
4126	ifileop = vn +'_'+ values[0].replace('reproj-','') + 'p_' + \
4127	op.replace('+','_') + '.nc'
4128	else:
4129	ifileorig = vn +'_'+ values[0].replace('reproj-','') + '.nc'
4130	ofileorig = vn +'_'+ values[0] + '.nc'
4131	ifileop = vn +'_'+ values[0].replace('reproj-','') + '_' + \
4132	op.replace('+','_') + '.nc'
4133
4134	ofileop = vn + '_' + values[0] + '_' + op.replace('+','_') + '.nc'
4135
4136	if fscr:
4137	sout = sub.call('rm ' + odir+'/'+ofileorig + ' >& /dev/null', shell=True)
4138	sout = sub.call('rm ' + odir+'/'+ofileop + ' >& /dev/null', shell=True)
4139
4140	# List of files to re-project (first without operation)
4141	ifiles = [odir+'/'+ifileorig, odir+'/'+ifileop]
4142	# List of output files after re-projection (first without operation)
4143	ofiles = [odir+'/'+ofileorig, odir+'/'+ofileop]
4144
4145	for iif in range(2):
4146	ifile = ifiles[iif]
4147	ofile = ofiles[iif]
4148
4149	# Only projecting original file-variable if there are not 'lon' and 'lat'
4150	doreproj = True
4151	if iif == 1 and len(inNO) > 0:
4152	doreproj = False
4153	if dbg:
4154	print warnmsg
4155	print ' ' + fname + ": operation '" + op + "' can not be " + \
4156	"reprojected due to the lack of 'lon' or 'lat' skipping it !!"
4157	print " NOTE: difference can only be computed as:"
4158	print " - 1st: compute variable's differences between models"
4159	print " - 2nd: Perform the operations"
4160
4161	# Building up resources to compute re-projected statistics
4162	if dbg:
4163	print ' '+fname+': computing from the original reprojected file'
4164	ModI = ModelInf(mod, mod, 'lon', 'lat', 'pres', 'time', 'depth', \
4165	'lon', 'lat', 'pres', 'time', 'depth', None)
4166	if type(values) == type([1]):
4167	fileh = values[0]
4168	else:
4169	fileh = values
4170
4171	if type(values) == type([1]):
4172	fileh = values[0]
4173	else:
4174	fileh = values
4175
4176	VarI = VariableInf(vn, values[0], values[2], values[3])
4177	usefs = {values[0]: ofileorig}
4178
4179	compute_statistics(ModI, config, config['ifold'], usefs, \
4180	config['ofold']+'/'+mod+'/'+exp,VarI,pinterp_var(op), op, fscr, dbg)
4181
4182	if not os.path.isfile(ofile) and doreproj:
4183	if debug and ofile == ofiles[0]:
4184	print " reprojecting '" + vn + "' '" + op + "' into '" + \
4185	ofile + "'"
4186
4187	# Reprojecting...
4188	if reprjop[0:6] == 'remap':
4189	ins = config['cdoHOME'] + '/cdo ' + reprjop + ',' + \
4190	config['RefProjfile'] + ' ' + ifile + ' ' + ofile
4191	try:
4192	with gen.Capturing() as output:
4193	sout = sub.call(ins, shell=True)
4194	except:
4195	print errmsg
4196	print ' ' + fname + ': $ ' + ins
4197	for s1out in output: print s1out
4198	print sout
4199	quit(-1)
4200	if sout != 0:
4201	print errmsg
4202	print ' ' + fname + ': $ ' + ins
4203	sub.call(ins, shell=True)
4204	quit(-1)
4205	else:
4206	# Name of the variable to reproject
4207	if iif == 0:
4208	reprjvn = vn
4209	else:
4210	reprjvn = VnOpS
4211
4212	reprojvalues = 'lon,lat,' + config['RefProjfile'] + ',lon,lat,'+ \
4213	reprjop + ',' + ':'.join(CFvardims)
4214	ins= 'ncvar.reproject('+reprojvalues + ', ' + ifile + ', '+ \
4215	reprjvn + ')'
4216	try:
4217	with gen.Capturing() as output:
4218	sout=ncvar.reproject(reprojvalues, ifile, reprjvn)
4219	except:
4220	print errmsg
4221	print ' ' + fname + ': ' + ins
4222	for s1out in output: print s1out
4223	quit(-1)
4224
4225	sub.call('mv reproject.nc ' + ofile, shell=True)
4226
4227	if debug:
4228	for s1out in output: print s1out
4229	print sout
4230
4231	# Keeping track of re-projection
4232	trckobjf.write('\n')
4233	if iif == 0:
4234	trckobjf.write('# ' + vn + '\n')
4235	else:
4236	trckobjf.write('# ' + vn + ' ' + op + '\n')
4237
4238	trckobjf.write(ins + '\n')
4239
4240	if reprjop[0:6] == 'remap':
4241	# CFing the reprojected file
4242	try:
4243	with gen.Capturing() as output:
4244	ncvar.CDO_toCF(ofile)
4245	except:
4246	print errmsg
4247	print 'ncvar.CDO_toCF(' + ofile + ')'
4248	for s1out in output: print s1out
4249	# Removing file in order to make sure that it will be redone
4250	print ' ' + fname + " removing file '" + ofile + "' ..."
4251	sub.call('rm ' + ofile + ' >& /dev/null', shell=True)
4252	quit(-1)
4253
4254	if debug:
4255	for s1out in output: print s1out
4256
4257	trckobjf.close()
4258
4259	if debug: print " End of '" + fname + "' "
4260
4261	return
4262
4263	def allmodexps_listconstruct(config, modexps, odir, debug):
4264	""" Function to create the list of all model-experiments to draw
4265	config= Configuration of the experiment
4266	modexps= list with the models-experiments pairs to use
4267	odir= output differences folder
4268	"""
4269	fname='allmodexps_listconstruct'
4270
4271	allplts = gen.get_specdictionary_HMT(config, H='PLOTALLMODEXP_')
4272
4273	if debug:
4274	if allplts is not None:
4275	print " 'all mod-exps' plots ________"
4276	gen.printing_dictionary(allplts)
4277
4278	# Getting Model-Reference Projection
4279	ModComProj = cnf['RefProj'].split(':')
4280
4281	allmodexp = {}
4282	Nallmodexp = 0
4283	for modexp in modexps:
4284	if debug:
4285	print " " + fname + ": '" + modexp + "' ..."
4286	mod = modexp.split('/')[0]
4287	exp = modexp.split('/')[1]
4288
4289	# Getting variable characteristics from each model/experiment
4290	idir = config['ofold'] + '/' + modexp
4291
4292	reproj = not gen.searchInlist(ModComProj,mod)
4293
4294	fvarcomp = 'varcompute.inf'
4295	if reproj: fvc = 'reproj_' + fvarcomp
4296	else: fvc = fvarcomp
4297
4298	files, testfiles, allcompvar, Nvar= read_varcomp_file(idir + '/' + fvc)
4299
4300	if allplts is not None:
4301	# Getting all model-experiments plots characteristics
4302
4303	for allplt in allplts.keys():
4304	pltn = allplt.split('_')[1]
4305	if debug and modexp == modexps[0]:
4306	print ' ' + fname + ": plot '" + pltn + "' "
4307	varops = allplts[allplt].split(':')
4308	for varop in varops:
4309	vn = varop.split('\|')[0]
4310	op = varop.split('\|')[1]
4311	vnop = vn + '_' + op
4312	if debug and modexp == modexps[0]:
4313	print ' ' + fname + ": varop '" + vnop + "' "
4314	if not allcompvar.has_key(vnop):
4315	print warnmsg
4316	print ' ' + fname + ": variable+operation '" + vnop + \
4317	"' in '" + modexp + "' was not computed !!"
4318	print ' skipping it!'
4319	print ' computed values:', allcompvar.keys()
4320	break
4321
4322	vals = allcompvar[vnop]
4323
4324	headerf = vals[0]
4325	if modexp == modexps[0]:
4326	allmodexp[vnop] = [modexp,headerf]
4327	Nallmodexp = Nallmodexp + 1
4328	else:
4329	prevals = allmodexp[vnop]
4330	prevals = prevals + [modexp,headerf]
4331	allmodexp[vnop] = prevals
4332
4333	else:
4334	Nallmodexp = 0
4335
4336	if allplts is not None:
4337	Sallmodexps = gen.dictKeysVals_stringList(allmodexp)
4338	else:
4339	Sallmodexps = 'none'
4340
4341	# Outwritting the all model-experiment plots file to avoid next time
4342	objf = open(odir + '/allmodexp.inf', 'w')
4343	objf.write('allmodexps: ' + Sallmodexps + '\n')
4344	objf.write('Nallmodexp: ' + str(Nallmodexp) + '\n')
4345	objf.close()
4346
4347	if debug: print " End of '" + fname + "' "
4348
4349	return allmodexp, Nallmodexp
4350
4351	def allmodexps_read(filen):
4352	""" Function to read the file with the information about the all model-experiment plots
4353	filen= file with the information
4354	"""
4355	fname = 'allmodexps_read'
4356
4357	if not os.path.isfile(filen):
4358	print errormsg
4359	print ' ' + fname + ": all model-experiment file '" + filen + \
4360	"' does not exist !!"
4361	quit(-1)
4362
4363	objf = open(filen, 'r')
4364	for line in objf:
4365	if line[0:1] != '#' and len(line) > 1:
4366	values = line.replace('\n','').split(' ')
4367	if values[0] == 'allmodexps:':
4368	allplts = gen.stringList_dictKeysVals(values[1])
4369	elif values[0] == 'Nallmodexp:':
4370	Nallmodexp = int(values[1])
4371
4372	objf.close()
4373
4374	return allplts, Nallmodexp
4375
4376	def draw_allmodexp_plots(config, allvarcomp, plots, modexp, odir, figscr, debug):
4377	""" Function to draw all model-experiment plots
4378	config= Configuration of the experiment
4379	allvarcomp = dictionary with the file headers for each variable
4380	plots= dictionary with the plots
4381	modexps= list with the names of the model-experiment pairs
4382	odir= output experiment folder
4383	figscr= whether figures should be done from the scratch or not
4384
4385	* Plot as
4386	{[kplot]} = [varn1]\|[op1]#[varn2]\|[op2]#[...[varnN]\|[opN]], ...
4387	[kplot] ___
4388	Nlines: Multiple lines with one line by model-experiment pairs
4389	[varn]
4390	variable
4391	[op]
4392	'+' separated list of operations
4393	in figures with more than 1 variable, use '#' to separate the [varn]\|[op]
4394	"""
4395	fname = 'draw_allmodexp_plots'
4396
4397	os.chdir(odir)
4398
4399	# Folder with the processed files
4400	infold = config['ofold']
4401
4402	# Dictionary with the operations with surnames for the operated variable
4403	opersurnames = {}
4404	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
4405	for opsr in opsur.keys():
4406	opn = opsr.split('_')[1]
4407	vls = opsur[opsr].split(':')
4408	opersurnames[opn] = vls
4409
4410	# time values
4411	# Units time for the plots
4412	rd = config['CFreftime']
4413	tunits = config['CFunitstime'] + '!since!' + rd[0:4] + '-' + rd[4:6] + '-' + \
4414	rd[6:8] + '!' + rd[8:10] + ':' + rd[10:12] + ':' + rd[12:14]
4415	# time ticks kind
4416	tkind = config['timekind']
4417	# time ticks format
4418	tfmt = config['timefmt']
4419	# time axis label
4420	tlab = config['timelabel']
4421	timevals = [tunits, tkind, tfmt, tlab]
4422
4423	# Dictionary of plot specificities
4424	specplotkeyn = 'specificvarplot'
4425	plotspecifics = {}
4426	if config.has_key(specplotkeyn):
4427	# [minval]: minimum value
4428	# [maxval]: minimum value
4429	# [colorbar]: name of the colorbar (from matplotlib) to use
4430	# [cntformat]: format of the contour labels
4431	# [colorcnt]: color for the countor lines
4432	plotspecs = config[specplotkeyn].split(':')
4433	for pltspc in plotspecs:
4434	pltvls = pltspc.split('\|')
4435	vn = pltvls[0]
4436	op = pltvls[1]
4437	fn = pltvls[2]
4438	plotspecifics[fn + '_' + vn + '_' + op] = pltvls[3:]
4439	if debug:
4440	print 'Specific values for plots _______'
4441	gen.printing_dictionary(plotspecifics)
4442
4443	# Graphical labels and configuration
4444	modgc, expgc = graphical_characteristics(config, debug)
4445
4446	# Kind of figures
4447	kindfigure = config['kindfig']
4448
4449	# Map value
4450	mapvalue = config['mapval']
4451
4452	# pythone scripts HOME
4453	pyHOME = config['pyHOME']
4454
4455	# Title-text of operations
4456	opexplained = {}
4457	optits = config['titleoperations'].split(':')
4458	for optit in optits:
4459	opn = optit.split('\|')[0]
4460	opt = optit.split('\|')[1]
4461	opexplained[opn] = opt
4462	if debug:
4463	print 'Titles for operations _______'
4464	gen.printing_dictionary(opexplained)
4465
4466	for kplot in plots.keys():
4467	varsplt = plots[kplot]
4468	for varplt in varsplt:
4469	if debug:
4470	print " printing '" + kplot + "' var: '" + varplt + "'..."
4471	varops = varplt.split('#')
4472
4473	# CF variables in plot
4474	CFvarsplot = []
4475	# Files in plot
4476	filesplot = []
4477	# Variables in plot within the files
4478	varsplot = []
4479	# Dims in figure
4480	dimsplot = []
4481	# pictoric values in figure
4482	pictplot = []
4483	# Name of the figure
4484	figname = ''
4485	# Title of the figure
4486	titfigure = ''
4487
4488	ivp = 0
4489	for varop in varops:
4490	vn = varop.split('\|')[0]
4491	op = varop.split('\|')[1]
4492
4493	# CF variables in plot
4494	CFvarsplot.append(vn)
4495
4496	if op.find('pinterp') != -1: gP = 'p'
4497	else: gP = ''
4498
4499	vnopS = vn + '_' + op
4500	if not allvarcomp.has_key(vnopS):
4501	print errmsg
4502	print ' ' + fname + ": no file for variable-operation '" + \
4503	vnopS + "' !!"
4504	modexpvals = allvarcomp[vnopS]
4505	Nmodexps = len(modexpvals)
4506
4507	for imodexp in range(0,Nmodexps,2):
4508	modexp = modexpvals[imodexp]
4509	headf = modexpvals[imodexp+1]
4510	if debug:
4511	print ' ' + fname + " model/experiment: '" + modexp + "'"
4512
4513	filen = infold + '/' + modexp + '/' + vn +'_'+ headf+gP + '_' + \
4514	op.replace('+','_') + '.nc'
4515
4516	filesplot.append(filen)
4517	# Do we have processed the given variable?
4518	if not os.path.isfile(filen):
4519	print warnmsg
4520	print " " + fname + ": there is no file for variable '" + \
4521	varop + "' skiping it !!"
4522	break
4523
4524	if imodexp == 0:
4525	# Name of the variable inside the file
4526	vnsur = varnoper(vn, op, opersurnames)
4527	varsplot.append(vnsur)
4528
4529	# Dimensions in file
4530	try:
4531	with gen.Capturing() as output:
4532	dims = ncvar.idims(filen)
4533	except:
4534	print errmsg
4535	print 'ncvar.idims('+filen+')'
4536	for s1out in output: print s1out
4537	quit(-1)
4538
4539	if imodexp == 0:
4540	# Dimensions in plot
4541	dimsplot.append(dims)
4542
4543	# Header of the name of the figure
4544	if ivp == 0:
4545	figname = kplot+'_'+vn+'_'+headf+'_'+ op.replace('+','-')
4546	else:
4547	figname = figname+'-'+vn+'_'+headf+'_'+op.replace('+','-')
4548
4549	# pictoric values for the figure
4550	Sfivaop = kplot + '_' + vn + '_' + op
4551	if plotspecifics.has_key(Sfivaop):
4552	pictvals = plotspecifics[Sfivaop]
4553	else:
4554	Vvals = gen.variables_values(vn)
4555	pictvals = [Vvals[2],Vvals[3],Vvals[6], '%g', 'fixc', 'black']
4556
4557	pictplot.append(pictvals)
4558
4559	ivp = ivp + 1
4560	# End of variable-operation
4561	figname = figname + '.' + kindfigure
4562
4563	# Title of figure
4564	titfigure = 'all!model-experiments!' + vn + optitle(op,opexplained)
4565
4566	draw_plot(kplot, CFvarsplot, filesplot, varsplot, dimsplot, pictplot, \
4567	figname, titfigure, kindfigure, mapvalue, timevals, expgc, odir, \
4568	pyHOME, figscr, debug)
4569
4570	# End of variables-operations
4571
4572	# End of kind of plots
4573
4574	if debug: print " End of '" + fname + "' "
4575
4576	return
4577
4578	def figinf_figname(figname,config):
4579	""" Function to retrieve figure information from its name
4580	figname= name of the figure
4581	config= configuration of `model_graphics.py'
4582	>>> figinf_figname('2linesTime_wss_wrfout_xvar-ymean-tas_wrfout_xvar-ymean.pdf', cnf)
4583	(2, '2linesTime', 'wss-tas', 'xvar-ymean@xvar-ymean')
4584	>>> figinf_figname('Nlines_tas_wrfout_tturb-xmean-tmean.pdf', cnf)
4585	(1, 'Nlines', 'tas', 'tturb-xmean-tmean')
4586	>>> figinf_figname('2lines_wss_diffvar_WRF-micro1_WRF-current_tturb-xmean-last-tas_tturb-xmean-last.pdf',cnf)
4587	(2, '2lines', 'wss-tas', 'tturb-xmean-last@tturb-xmean-last')
4588	"""
4589	fname = 'figinf_figname'
4590	plotsecs = figname.split('_')
4591	plotk = plotsecs[0]
4592
4593	if gen.searchInlist(config['pairfigures'].split(':'), plotk):
4594	Nvars = 2
4595	varn1 = plotsecs[1]
4596	if plotsecs[2] == 'diffvar' or plotsecs[2] == 'diffop':
4597	statn1varn2 = plotsecs[5].split('-')
4598	statn2 = plotsecs[6].split('.')[0]
4599	Lstatn1varn2 = len(statn1varn2)
4600	statn1 = '-'.join(statn1varn2[0:Lstatn1varn2-1])
4601	varn2 = statn1varn2[Lstatn1varn2-1]
4602	else:
4603	statn1varn2 = plotsecs[3].split('-')
4604	statn2 = plotsecs[5].split('.')[0]
4605	Lstatn1varn2 = len(statn1varn2)
4606	statn1 = '-'.join(statn1varn2[0:Lstatn1varn2-1])
4607	varn2 = statn1varn2[Lstatn1varn2-1]
4608	return Nvars, plotk, varn1+'-'+varn2, statn1+'@'+statn2
4609	elif gen.searchInlist(config['Nsourcesfigures'].split(':'), plotk):
4610	Nvars = 1
4611	varn = plotsecs[1]
4612	statn = plotsecs[3].split('.')[0]
4613	return Nvars, plotk, varn, statn
4614	else:
4615	print errmsg
4616	print ' ' + fname + ": kind of figure '" + plotk + "' not ready !!"
4617	print ' ready ones _______'
4618	print " 'pair_figures': figures of pair of variables: ", \
4619	config['pairfigures'].split(':')
4620	print " 'N-sources_figures': figures of pair of variables: ", \
4621	config['Nsourcesfigures'].split(':')
4622	quit(-1)
4623
4624	# Files with information about the configuration of the script
4625	inffiles = ['varcompute.inf', 'all_computevars.inf', 'all_statsvars.inf']
4626
4627	####### #######
4628	## MAIN
4629	#######
4630	if not os.path.isfile('model_graphics.dat'):
4631	print errmsg
4632	print main + ": configuration file 'model_graphics.dat' does not exist!!"
4633	quit(-1)
4634
4635	# Getting configuration from external ASCII file 'model_graphics.dat'
4636	cnf = gen.get_configuration('model_graphics.dat', False)
4637
4638	verify_configuration(cnf, gen.Str_Bool(cnf['debug']))
4639
4640	# scratches
4641	scratch, filescratch, figscratch, diffscratch, figdiffscratch, moddiffscratch, \
4642	figmoddiffscratch, figallmodexpscratch, addfiles, addfigures, adddiffs, \
4643	adddifffigures, addmoddiffs, addmoddifffigures, addallmodexpfigures, dbg = \
4644	scratches(cnf)
4645
4646	if dbg:
4647	print 'Experiment configuration scratches _______'
4648	print ' scratch:', scratch, 'filescratch:', filescratch,'figscratch:', figscratch
4649	print ' diffscratch:', diffscratch, 'figdiffscratch:', figdiffscratch
4650	print ' moddiffscratch:', moddiffscratch, 'figmoddiffscratch:', figmoddiffscratch
4651	print 'Experiment configuration adding _______'
4652	print ' addfiles:', addfiles, 'addfigures:', addfigures
4653	print ' adddiffs:', adddiffs, 'adddifffigures:', adddifffigures
4654	print ' addmoddiffs:', addmoddiffs, 'addmoddifffigures:', addmoddifffigures
4655
4656	# Getting models
4657	mods = cnf['models'].split(':')
4658
4659	# Getting graphical characeristics
4660	modGC, expGC = graphical_characteristics(cnf, dbg)
4661
4662	# Models loop
4663	##
4664
4665	# dictionary with the experiments of each model
4666	modexps = {}
4667
4668	for mod in mods:
4669	print mod
4670	if scratch:
4671	if cnf['ofold'] != cnf['ifold']:
4672	sout = sub.call('rm -rf '+cnf['ofold']+'/'+mod+' > /dev/null', shell=True)
4673	else:
4674	print warnmsg
4675	print ' ' + main + ": input '" + cnf['ifold'] + "' and output '" + \
4676	cnf['ofold'] + "' are the same folder !!"
4677	print " Keeping output folder although it has 'scratch' start"
4678
4679	# Get experiments and headers of model
4680	exps, fheaders = exp_headers(mod,cnf)
4681
4682	modexps[mod] = exps
4683	# Characteristics of the model
4684	Modinf = ncvar.model_characteristics(mod,'None','False')
4685	dnx = Modinf.dimxn
4686	dny = Modinf.dimyn
4687	dnz = Modinf.dimzn
4688	dnt = Modinf.dimtn
4689	dns = Modinf.dimsn
4690	vdnx = Modinf.vardxn
4691	vdny = Modinf.vardyn
4692	vdnz = Modinf.vardzn
4693	vdnt = Modinf.vardtn
4694	vdns = Modinf.vardsn
4695
4696	modgraphv = modGC[mod]
4697	Modinf = ModelInf(mod, mod, dnx, dny, dnz, dnt, dns, vdnx, vdny, vdnz, vdnt, \
4698	vdns, modgraphv.tmod)
4699
4700	if dbg:
4701	print ' model characteristics _______'
4702	gen.printing_class(Modinf)
4703
4704	moddims = dnx + ',' + dny + ',' + dnz + ',' + dnt
4705	modvdims = vdnx + ',' + vdny + ',' + vdnz + ',' + vdnt
4706
4707	# Experiments loop
4708	##
4709	for exp in exps:
4710	print ' ' + exp + '...'
4711
4712	# input folder
4713	iwdir = cnf['ifold'] + '/' + mod + '/' + exp
4714
4715	# Does input folder exist?
4716	if not os.path.isdir(iwdir):
4717	if filescratch*addfiles:
4718	print errmsg
4719	print " " + main + ": folder '" + iwdir + "' does not exist !!"
4720	quit(-1)
4721	else:
4722	print warnmsg
4723	print " " + main + ": folder '" + iwdir + "' does not exist !!"
4724	print " but start of files from scratch 'filescratch':", \
4725	filescratch, "and there is not the add of any file 'addfiles':", \
4726	addfiles
4727	print ' so, keep moving'
4728
4729	owdir = cnf['ofold'] + '/' + mod + '/' + exp
4730	sout = sub.call('mkdir -p ' + owdir, shell=True)
4731
4732	# Need to pass to analyze all the data?
4733	if filescratch:
4734	for inff in inffiles:
4735	if dbg: print " removing information file '" + inff + "' ..."
4736	ins = 'rm ' + owdir + '/' + inff + ' >& /dev/null'
4737	sout = sub.call(ins, shell=True)
4738
4739	objf = open(owdir+'/all_computevars.inf','w')
4740	objf.write("## Computation of variables \n")
4741	objf.close()
4742	objf = open(owdir+'/all_statsvars.inf','w')
4743	objf.write("## Computation of statistics \n")
4744	objf.close()
4745
4746	varcompf = owdir + '/varcompute.inf'
4747	if addfiles:
4748	if dbg:
4749	print ' ' + main + ": adding variables to compute removing " + \
4750	"file '" + varcompf + "' ..."
4751	sub.call('rm ' + varcompf +' >& /dev/null', shell=True)
4752
4753	if not os.path.isfile(varcompf):
4754	# Does input folder has header files?
4755	ih=1
4756	# Dictionary with the list of files for each headers
4757	files = {}
4758	# Dictionary with a file fromor each headers
4759	testfiles = {}
4760
4761	for fh in fheaders:
4762	if filescratch:
4763	ins = 'rm '+ owdir+'/_' + fh + '.nc >& /dev/null'
4764	sout = sub.call(ins, shell=True)
4765	files1h = gen.files_folder(iwdir,fh)
4766	if len(files1h) < 1:
4767	print errmsg
4768	print ' ' + main + ": folder '" + iwdir + "' does not " + \
4769	"contain files '" + fh + "*' !!"
4770	quit(-1)
4771	files[fh] = files1h
4772	testfiles[fh] = files1h[0]
4773
4774	if dbg:
4775	print ' Dictionary of files _______'
4776	gen.printing_dictionary(files)
4777
4778	print ' Dictionary of test-files _______'
4779	gen.printing_dictionary(testfiles)
4780
4781	allcompvar, Nvar = compvars_listconstruct(cnf, Modinf, files, testfiles, \
4782	iwdir, owdir, dbg)
4783	else:
4784	if dbg:
4785	print warnmsg
4786	print ' '+main+": getting variables to compute already from file !!"
4787	files, testfiles, allcompvar, Nvar = read_varcomp_file(varcompf)
4788
4789	# End of avoiding to repeat all the experiment search
4790
4791	print " For experiment '" + exp + "' is required to compute:", Nvar, \
4792	"variables"
4793
4794	if dbg:
4795	print 'Variables to compute _______'
4796	gen.printing_dictionary(allcompvar)
4797
4798	# Computing variables
4799	print " Computing variables ..."
4800	compute_vars(cnf, Modinf, iwdir, owdir, files, allcompvar, filescratch, dbg)
4801
4802	# Figures of variables direct from files
4803	print " Plotting direct figures ..."
4804	dirfigf = owdir + '/directplotsdraw.inf'
4805	if figscratch:
4806	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4807
4808	objf = open(owdir+'/all_figures.inf','w')
4809	objf.write("## Drawing of all figures \n")
4810	objf.close()
4811
4812	if addfigures:
4813	if dbg:
4814	print ' ' + main + ": adding direct figures removing file '" + \
4815	dirfigf + "' ..."
4816	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4817
4818	if not os.path.isfile(dirfigf):
4819	listplots, Nplt = plots_listconstruct(cnf, 'DIRPLT', dirfigf, owdir, dbg)
4820	else:
4821	if dbg:
4822	print warnmsg
4823	print ' ' + main + ": getting plots to draw already from file !!"
4824	listplots, Nplt = read_plot_file(dirfigf)
4825
4826	# End of avoiding to repeat all the plots search
4827
4828	print " For experiment '" + exp + "' is required to plot:", Nplt, "plots"
4829
4830	if dbg:
4831	print 'Plots to draw _______'
4832	gen.printing_dictionary(listplots)
4833
4834	draw_plots(cnf, listplots, mod, exp, owdir, allcompvar, figscratch, dbg)
4835
4836	# end of experiments loop
4837
4838	### ## #
4839	# Experiment differences
4840	## # ## #
4841
4842	# Experiments loop
4843	##
4844	print ' ***** ** * * ** * * ** * ** *****'
4845	print " ** '" + mod + "': Inter experiments differences "
4846	print ' ***** ** * * ** * * ** * ** *****'
4847	print ' experiments:', exps
4848	# There are two kind of differences:
4849	# DIFFop: differences between operations of each given variable.
4850	# Differences are computed directly from the last stage of the operation
4851	# DIFFvar: operations of differences between variables
4852	# First are computed the differences from the initial variable file
4853	# and then operations are made
4854	# NOTE: remember that: meanvar2 - meanvar1 = mean(var2 - var1)
4855	difffiles = ['diffop.inf', 'diffvar.inf']
4856
4857	Nexps = len(exps)
4858	for iexp1 in range(0,Nexps-1):
4859	exp1 = exps[iexp1]
4860	for iexp2 in range(iexp1+1,Nexps):
4861	exp2 = exps[iexp2]
4862	Sexps = exp2 + '-' + exp1
4863	print ' ' + Sexps + '...'
4864	owdir = cnf['ofold'] + '/' + mod + '/' + exp2 + '-' + exp1
4865	sout = sub.call('mkdir -p ' + owdir, shell=True)
4866
4867	# Removing files with list of differences if should be started from scratch or add differences
4868	for fdiff in difffiles:
4869	difff = owdir + '/' + fdiff
4870	if diffscratch:
4871	sub.call('rm -rf' + owdir +' >& /dev/null', shell=True)
4872	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4873	objf = open(owdir+'/all_' + fdiff,'w')
4874	objf.write("## Computation and drawing of differences " + \
4875	"between '" + exp2 + "'-'" + exp1 + "'\n")
4876	objf.close()
4877	difff = owdir + '/all_vardiffstatistics.inf'
4878	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4879	objf = open(owdir+'/all_' + fdiff,'w')
4880	objf.write("## Computation of all differences statistics " + \
4881	"between '" + exp2 + "'-'" + exp1 + "'\n")
4882	objf.close()
4883
4884	if adddiffs:
4885	if dbg:
4886	print ' ' + main + ": adding differences removing file '" + \
4887	difff + "' ..."
4888	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4889
4890	# op and var differences
4891	diffvarcompf = owdir + '/' + difffiles[0]
4892	if not os.path.isfile(diffvarcompf):
4893	alldiffop, alldiffvar,Nopdiffs,Nvardiffs=diffvarop_listconstruct(cnf,\
4894	[mod, mod], [exp1, exp2], [False, False], owdir, dbg)
4895	else:
4896	if dbg:
4897	print warnmsg
4898	print ' ' + main + ": getting 'op' and 'var' differences to " + \
4899	"compute already from file !!"
4900	alldiffop, Nopdiffops = read_diff_file(diffvarcompf)
4901	alldiffvar, Nopdiffvars = read_diff_file(owdir+'/'+difffiles[1])
4902	print 'Nopdiffops:', Nopdiffops,'Nopdiffvars:', Nopdiffvars
4903
4904	# End of avoiding to repeat all the experiment search
4905	print " For experiments '"+exp2+"'-'"+exp1+"' is required to " + \
4906	"compute:", Nvar, "differences"
4907
4908	if dbg:
4909	print 'Differences to compute _______'
4910	gen.printing_dictionary(alldiffop)
4911	gen.printing_dictionary(alldiffvar)
4912
4913	# Computing differences
4914	##
4915	print " Computing operation differences ..."
4916	compute_op_diffs(cnf, alldiffop, owdir, diffscratch, dbg)
4917	print " Computing variable differences ..."
4918	compute_var_diffs(cnf, alldiffvar, owdir, diffscratch, dbg)
4919
4920	# Plotting operation differences
4921	##
4922	print " " + main + ": Plotting operation differences' figures ..."
4923	dirfigf = owdir + '/diffopplotsdraw.inf'
4924	if figscratch:
4925	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4926
4927	objf = open(owdir+'/all_diffopfigures.inf','w')
4928	objf.write("## Drawing of all operation difference figures \n")
4929	objf.close()
4930
4931	if adddifffigures:
4932	if dbg:
4933	print ' ' + main + ": adding differences' figures removing " + \
4934	"file '" + dirfigf + "' ..."
4935
4936	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4937
4938	if not os.path.isfile(dirfigf):
4939	listplots, Nplt = plots_listconstruct(cnf, 'PLOTDIFFOP', dirfigf, \
4940	owdir, dbg)
4941	else:
4942	if dbg:
4943	print warnmsg
4944	print ' ' + main + ": getting plots to draw already from file !!"
4945	listplots, Nplt = read_plot_file(dirfigf)
4946
4947	# End of avoiding to repeat all the plots search
4948
4949	print " For experiment 'operation' differences '" + Sexps + "' is " + \
4950	"required to plot:" , Nplt, "plots"
4951
4952	if dbg:
4953	print ' Plots to draw _______'
4954	gen.printing_dictionary(listplots)
4955
4956	draw_diff_plots(cnf, listplots, owdir, alldiffop, 'diffop', \
4957	figdiffscratch, dbg)
4958
4959	# Plotting variable differences
4960	##
4961	print " " + main + ": Plotting variable differences' figures ..."
4962	dirfigf = owdir + '/diffvarplotsdraw.inf'
4963	if figscratch:
4964	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4965
4966	objf = open(owdir+'/all_diffvarfigures.inf','w')
4967	objf.write("## Drawing of all variables difference figures \n")
4968	objf.close()
4969
4970	if adddifffigures:
4971	if dbg:
4972	print ' '+main+": adding differences' figures removing file '" +\
4973	dirfigf + "' ..."
4974	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4975
4976	if not os.path.isfile(dirfigf):
4977	listplots, Nplt = plots_listconstruct(cnf, 'PLOTDIFFVAR', dirfigf, \
4978	owdir, dbg)
4979	else:
4980	if dbg:
4981	print warnmsg
4982	print ' ' + main + ": getting plots to draw already from file !!"
4983	listplots, Nplt = read_plot_file(dirfigf)
4984
4985	# End of avoiding to repeat all the plots search
4986
4987	print " For experiment 'variables' differences '" + Sexps + "' is " + \
4988	"required to plot:" , Nplt, "plots"
4989
4990	if dbg:
4991	print ' Plots to draw _______'
4992	gen.printing_dictionary(listplots)
4993
4994	draw_diff_plots(cnf, listplots, owdir, alldiffvar, 'diffvar', \
4995	figdiffscratch, dbg)
4996
4997	# end of mods loop
4998
4999	### ## #
5000	# Model differences
5001	### ## #
5002
5003	# Models already at the reference projection
5004	ModComProj = cnf['RefProj'].split(':')
5005	if dbg:
5006	print 'Models to use as reference for the projection:', ModComProj
5007
5008	# Operators to use for the reprojection of each variable
5009	REPRJvar0 = gen.get_specdictionary_HMT(cnf, H='reprojectvar_')
5010	REPRJvar = {}
5011	for Skey in REPRJvar0.keys():
5012	REPRJvar[Skey] = gen.str_list(REPRJvar0[Skey],':')
5013
5014	if dbg:
5015	print 'Remaping operators to use _______'
5016	gen.printing_dictionary(REPRJvar)
5017
5018	# Getting common base weights for grid point remapping with CDO
5019	frefgridn = 'RefProj.nc'
5020	for mod in mods:
5021	frefgriddes = cnf['ofold'] + '/' + mod + '/' + frefgridn
5022	if moddiffscratch:
5023	sout = sub.call('rm ' + frefgriddes + ' >& /dev/null', shell=True)
5024	if gen.searchInlist(ModComProj,mod) and not os.path.isfile(frefgriddes):
5025	# Looking for a file to use (all files are CF compilant, but we need to
5026	# be sure that at least they have 'lon', 'lat' variables)
5027	exps = modexps[mod]
5028	exp = exps[0]
5029	reprjfdir = cnf['ofold'] + '/' + mod + '/' + exp
5030	reprjfile = None
5031	for reprjop in REPRJvar.keys():
5032	reprjvars = REPRJvar[reprjop]
5033	for reprjvar in reprjvars:
5034	reprjfs= gen.files_folder_HMT(folder=reprjfdir,head=reprjvar+'_',tail='.nc')
5035	for reprjf in reprjfs:
5036	# Getting direct files as [var]_[fhead].nc
5037	if len(reprjf.split('_')) == 2:
5038	reprjfile = reprjfdir + '/' + reprjf
5039	reprjvn = reprjvar
5040	break
5041	if reprjfile is None:
5042	print errmsg
5043	print ' ' + main + ": no proper file to get projection information " + \
5044	"from '" + reprjfdir + "' has been found !!"
5045	quit(-1)
5046
5047	print " Creation of reference projection information file from '" + mod+ "'"
5048	if dbg:
5049	print " using file: '" + reprjfile
5050
5051	# Getting only 1 time-step for disk space issues
5052	try:
5053	with gen.Capturing() as output:
5054	ncvar.DataSetSection('time,0,1,1', reprjfile)
5055	except:
5056	print errmsg
5057	print 'ncvar.DataSetSection(time,0,1,1, ' + reprjfile + ')'
5058	for s1out in output: print s1out
5059	quit(-1)
5060
5061	of = reprjfile.split('.')[0] + '_time_B0-E1-I1.nc'
5062
5063	# Cleaning variables
5064	try:
5065	with gen.Capturing() as output:
5066	ncvar.selvar('lon@lon,lat@lat,time@time', of, reprjvn)
5067	except:
5068	print errmsg
5069	print 'ncvar.selvar(lon@lon,lat@lat, ' + of + ', ' + reprjvn + ')'
5070	for s1out in output: print s1out
5071	quit(-1)
5072	sout = sub.call('mv selvar_new.nc ' + frefgriddes, shell=True)
5073	sout = sub.call('rm ' + of + ' >& /dev/null', shell=True)
5074
5075	cnf['RefProjfile'] = frefgriddes
5076
5077	if dbg:
5078	for s1out in output: print s1out
5079	break
5080
5081	elif gen.searchInlist(ModComProj,mod) and os.path.isfile(frefgriddes):
5082	# Including reprojecting reference file in configuration dictionary
5083	cnf['RefProjfile'] = frefgriddes
5084
5085	Nmods = len(mods)
5086	for imod1 in range(0,Nmods-1):
5087	mod1 = mods[imod1]
5088	exps1 = modexps[mod1]
5089
5090	for imod2 in range(imod1+1,Nmods):
5091	mod2 = mods[imod2]
5092	exps2 = modexps[mod2]
5093
5094	Smods = mod2 + '-' + mod1
5095	print ' ***** ** * * ** * * ** * ** *****'
5096	print " ** '" + Smods + "': Inter models differences "
5097	print ' ***** ** * * ** * * ** * ** *****'
5098	print ' experiments mod1:', exps1
5099	print ' experiments mod2:', exps2
5100
5101	# Experiments loop
5102	##
5103	difffiles = ['diffop.inf', 'diffvar.inf']
5104
5105	Nexps1 = len(exps1)
5106	Nexps2 = len(exps2)
5107	for iexp1 in range(0,Nexps1):
5108	exp1 = exps1[iexp1]
5109	for iexp2 in range(0,Nexps2):
5110	exp2 = exps2[iexp2]
5111	Sexps = exp2 + '-' + exp1
5112	print ' ' + Sexps + '...'
5113	owdir = cnf['ofold'] + '/' + Smods + '/' + Sexps
5114	sout = sub.call('mkdir -p ' + owdir, shell=True)
5115
5116	# Getting the right projection in order to perform differences
5117	difmods = [mod1, mod2]
5118	difexps = [exp1, exp2]
5119	difreproj = [False, False]
5120
5121	for imod in range(2):
5122	mod = difmods[imod]
5123	exp = difexps[imod]
5124	if not gen.searchInlist(ModComProj,mod):
5125	print " Projecting files of model '" + mod + "' exp: '" + \
5126	exp + "' with file '" + cnf['RefProjfile'] + "'"
5127	reproject_modexp(mod, exp, cnf, REPRJvar, moddiffscratch, \
5128	addmoddiffs, dbg)
5129	difreproj[imod] = True
5130
5131	# Removing files with list of differences if should be started from
5132	# scratch or add differences
5133	for fdiff in difffiles:
5134	difff = owdir + '/' + fdiff
5135	if moddiffscratch:
5136	sub.call('rm -rf' + owdir +' >& /dev/null', shell=True)
5137	sub.call('rm ' + difff +' >& /dev/null', shell=True)
5138	objf = open(owdir+'/all_' + fdiff,'w')
5139	objf.write("## Computation and drawing of differences " + \
5140	"between '" + mod2+'@'+exp2 + "'-'" + mod1+'@'+exp1 + "'\n")
5141	objf.close()
5142	difff = owdir + '/all_vardiffstatistics.inf'
5143	sub.call('rm ' + difff +' >& /dev/null', shell=True)
5144	objf = open(owdir+'/all_' + fdiff,'w')
5145	objf.write("## Computation of all differences statistics " + \
5146	"between '" + mod2+'@'+exp2 + "'-'" + mod1+'@'+exp1 + "'\n")
5147	objf.close()
5148
5149	if addmoddiffs:
5150	if dbg:
5151	print ' ' + main + ": adding model differences " + \
5152	"removing file '" + difff + "' ..."
5153	sub.call('rm ' + difff +' >& /dev/null', shell=True)
5154
5155	# op and var differences
5156	diffvarcompf = owdir + '/' + difffiles[0]
5157	if not os.path.isfile(owdir+'/'+difffiles[0]) or \
5158	not os.path.isfile(owdir+'/'+difffiles[1]):
5159	alldiffop, alldiffvar, Nopdiffs, Nvardiffs = \
5160	diffvarop_listconstruct(cnf, difmods, difexps, difreproj,owdir,\
5161	dbg)
5162	else:
5163	if dbg:
5164	print warnmsg
5165	print ' '+main+": getting 'op' and 'var' differences to " + \
5166	"compute already from file !!"
5167	alldiffop, Nopdiffops = read_diff_file(diffvarcompf)
5168	alldiffvar, Nopdiffvars = read_diff_file(owdir+'/'+difffiles[1])
5169
5170	# End of avoiding to repeat all the experiment search
5171
5172	print " For experiments '" + mod2+'@'+exp2 + "'-'" + mod+'@'+exp1+\
5173	"' is required to compute:", Nvar, "differences"
5174
5175	if dbg:
5176	print ' op differences to compute _______'
5177	gen.printing_dictionary(alldiffop)
5178	print ' var differences to compute _______'
5179	gen.printing_dictionary(alldiffvar)
5180
5181	# Computing differences
5182	##
5183	print " Computing operation differences ..."
5184	compute_op_diffs(cnf, alldiffop, owdir, moddiffscratch, dbg)
5185	print " Computing variable differences ..."
5186	compute_var_diffs(cnf, alldiffvar, owdir, moddiffscratch, dbg)
5187
5188	# Plotting operation differences
5189	##
5190	print " " + main + ": Plotting operation differences' figures ..."
5191	dirfigf = owdir + '/diffopplotsdraw.inf'
5192	if figmoddiffscratch:
5193	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5194
5195	objf = open(owdir+'/all_diffopfigures.inf','w')
5196	objf.write("## Drawing of all operation difference figures \n")
5197	objf.close()
5198
5199	if addmoddifffigures:
5200	if dbg:
5201	print ' ' + main + ": adding model differences' figures " + \
5202	"removing file '" + dirfigf + "' ..."
5203	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5204
5205	if not os.path.isfile(dirfigf):
5206	listplots, Nplt = plots_listconstruct(cnf, 'PLOTDIFFOP', dirfigf,\
5207	owdir, dbg)
5208	else:
5209	if dbg:
5210	print warnmsg
5211	print ' '+main+": getting plots to draw already from file !!"
5212	listplots, Nplt = read_plot_file(dirfigf)
5213
5214	# End of avoiding to repeat all the plots search
5215
5216	print " For experiment 'operation' differences '" + Sexps + "' is "+\
5217	"required to plot:" , Nplt, "plots"
5218
5219	if dbg:
5220	print 'Plots to draw _______'
5221	gen.printing_dictionary(listplots)
5222
5223	draw_diff_plots(cnf, listplots, owdir, alldiffop, 'diffop', \
5224	figmoddiffscratch, dbg)
5225
5226	# Plotting variable differences
5227	##
5228	print " " + main + ": Plotting variable differences' figures ..."
5229	dirfigf = owdir + '/diffvarplotsdraw.inf'
5230	if figscratch:
5231	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5232
5233	objf = open(owdir+'/all_diffvarfigures.inf','w')
5234	objf.write("## Drawing of all variables difference figures \n")
5235	objf.close()
5236
5237	if adddifffigures:
5238	if dbg:
5239	print ' ' + main + ": adding differences's figures " + \
5240	"removing file '" + dirfigf + "' ..."
5241	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5242
5243	if not os.path.isfile(dirfigf):
5244	listplots, Nplt = plots_listconstruct(cnf,'PLOTDIFFVAR',dirfigf, \
5245	owdir, dbg)
5246	else:
5247	if dbg:
5248	print warnmsg
5249	print ' '+main+": getting plots to draw already from file !!"
5250	listplots, Nplt = read_plot_file(dirfigf)
5251
5252	# End of avoiding to repeat all the plots search
5253
5254	print " For experiment 'variables' differences '" + Sexps+ "' is "+ \
5255	"required to plot:" , Nplt, "plots"
5256
5257	if dbg:
5258	print ' Plots to draw _______'
5259	gen.printing_dictionary(listplots)
5260
5261	draw_diff_plots(cnf, listplots, owdir, alldiffvar, 'diffvar', \
5262	figdiffscratch, dbg)
5263
5264	# end of exp2 loop
5265	# end of exp1 loop
5266	# end of mod2 loop
5267	# end of mod1 loop
5268
5269	### ## #
5270	# All model/exps linear plots
5271	### ## #
5272
5273	allmodexps = []
5274	for mod in mods:
5275	for exp in modexps[mod]:
5276	allmodexps.append(mod + '/' + exp)
5277
5278	print ' ***** ** * * ** * * ** * ** *****'
5279	print ' ** Multi models, multi experiments plots '
5280	print ' ***** ** * * ** * * ** * ** *****'
5281	print ' Model/experiments:', allmodexps
5282
5283	owdir = cnf['ofold'] + '/allmodexps'
5284	sout = sub.call('mkdir -p ' + owdir, shell=True)
5285
5286	# Plotting all model-experiments lines
5287	##
5288	print " " + main + ": Plotting all model-experiments variable figures ..."
5289	allf = owdir + '/allmodexp.inf'
5290	dirfigf = owdir + '/allmodexpfigures.inf'
5291	allfigf = owdir+'/all_figures.inf'
5292	if figallmodexpscratch:
5293	sout = sub.call('rm ' + allf + ' >& /dev/null', shell=True)
5294	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5295	sout = sub.call('rm ' + allfigf + ' >& /dev/null', shell=True)
5296
5297	objf = open(allfigf,'w')
5298	objf.write("## Drawing of all variables figures for all model-experiments\n")
5299	objf.close()
5300
5301	if addallmodexpfigures:
5302	if dbg:
5303	print ' ' + main + ": adding model-experiment differences removing " + \
5304	" file '" + allfigf + "' ..."
5305	print ' ' + main + ": adding model-experiment differences' figures " + \
5306	"removing file '" + dirfigf + "' ..."
5307	sout = sub.call('rm ' + allfigf + ' >& /dev/null', shell=True)
5308	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5309
5310	if not os.path.isfile(dirfigf):
5311	listplots, Nplt = plots_listconstruct(cnf, 'PLOTALLMODEXP', dirfigf, owdir, dbg)
5312	else:
5313	if dbg:
5314	print warnmsg
5315	print ' ' + main + ": getting plots to draw already from file !!"
5316	listplots, Nplt = read_plot_file(dirfigf)
5317
5318	# End of avoiding to repeat all the plots search
5319
5320	print " Is required to plot:", Nplt, " all model-experiment plots"
5321
5322	if dbg:
5323	print ' Plots to draw _______'
5324	gen.printing_dictionary(listplots)
5325
5326	if not os.path.isfile(allf):
5327	# Constructing the information of modexp / variable
5328	allmodexpvar, Nallmodexpvar = allmodexps_listconstruct(cnf, allmodexps, owdir, dbg)
5329	else:
5330	allmodexpvar, Nallmodexpvar = allmodexps_read(allf)
5331
5332	draw_allmodexp_plots(cnf, allmodexpvar, listplots, modexps, owdir, figallmodexpscratch, dbg)
5333
5334	print main + ': all files and figures have been properly done !!'
5335
5336	# Creation of a pdf with all figures
5337	##
5338	###
5339	owdir = cnf['ofold']
5340	os.chdir(owdir)
5341
5342	texout ='model_graphics-images'
5343	print ' ***** ** * * ** * * ** * ** *****'
5344	print ' ** Creation of a pdf with all figures '
5345	print ' ***** ** * * ** * * ** * ** *****'
5346	print ' ' + owdir + '/' + texout + '.pdf'
5347
5348	if dbg:
5349	print main + ": generation of a pdf file '" + owdir + '/' + texout + \
5350	"' with all the figures..."
5351
5352	otexf = open(owdir + '/' + texout + '.tex', 'w')
5353	otexf.write('\\documentclass{article}\n')
5354	otexf.write('\\usepackage{graphicx}\n')
5355	otexf.write('\\usepackage[colorlinks=true,urlcolor=blue]{hyperref}\n')
5356	otexf.write('\\textheight=23cm\n')
5357	otexf.write('\\textwidth=18cm\n')
5358	otexf.write('\\oddsidemargin=-1cm\n')
5359	otexf.write('\\evensidemargin=-1cm\n')
5360	otexf.write('\\topmargin=-1cm\n')
5361
5362	otexf.write('\n')
5363	otexf.write('\\begin{document}\n')
5364	otexf.write('\n')
5365	otexf.write('\\def\\fontBloc{\\fontfamily{\\sfdefault}\\large\\bfseries}\n')
5366	otexf.write('\\def\\linia{\\setlength{\\unitlength}{1mm}\n')
5367	otexf.write('\\line(1,0){80}}\n')
5368	otexf.write('\\newcommand{\\modexp}[1]{\\clearpage\n')
5369	otexf.write('\\noindent\\linia\n')
5370	otexf.write('\\section{#1}\n')
5371	otexf.write('\\linia}\n')
5372	otexf.write('\n')
5373
5374	otexf.write('\\title{model\_graphics: ' + gen.latex_text(owdir) + '}\n')
5375	otexf.write('\\maketitle\n')
5376	otexf.write('\n')
5377	otexf.write('\\tableofcontents\n')
5378	otexf.write('\\listoffigures\n')
5379	otexf.write('\\clearpage\n')
5380
5381	Nmods = len(mods)
5382	lmodexps = []
5383	ldiffmodexps = []
5384	# Figures from mod/exp pairs
5385	for mod in mods:
5386	exps = modexps[mod]
5387	for exp in exps:
5388	print ' direct plots:', mod, exp
5389	lmodexps.append(mod+'/'+exp)
5390	owdirme = owdir + '/' + mod + '/' + exp
5391	otexf.write('% ' + mod + ' ' + exp + '\n')
5392	otexf.write('\\modexp{' + gen.latex_text(mod+' '+exp) + '}\n')
5393	listplots, Nplt = read_plot_file(owdirme + '/directplotsdraw.inf')
5394	for plot in listplots.keys():
5395	plots = listplots[plot]
5396	ops = []
5397	for plt in plots:
5398	opn = plt.split('#')[0].split('\|')[1]
5399	if not gen.searchInlist(ops,opn): ops.append(opn)
5400	for op in ops:
5401	opS = op.replace('+','_')
5402	modexpimg = gen.files_folder_HMT(owdirme, head=plot, middle=opS, \
5403	tail=cnf['kindfig'])
5404	Nimages = len(modexpimg)
5405	if Nimages > 0:
5406	for img in range(Nimages): modexpimg[img] = owdirme + '/' + \
5407	modexpimg[img]
5408	caption = mod + ' ' + exp + ' ' + plot + ' ' + op
5409	flab = 'fig:' + caption.replace(' ', '_')
5410	try:
5411	with gen.Capturing() as output:
5412	gen.latex_fig_array(modexpimg, otexf, \
5413	gen.latex_text(caption), flab, refsize=0.5, dist='sqr',\
5414	dorest='center')
5415	except:
5416	print errmsg
5417	print 'gen.latex_fig_array(',modexpimg,', '+otexf+', '+ \
5418	'gen.latex_text(' + caption + '), ' + flab + \
5419	", refsize=0.5, dist='sqr', dorest='center')"
5420	for s1out in output: print s1out
5421	quit(-1)
5422	otexf.write('%\\clearpage\n')
5423
5424	# differences between experiments
5425	Nexps = len(exps)
5426	for iexp1 in range(0,Nexps-1):
5427	exp1 = exps[iexp1]
5428	for iexp2 in range(iexp1+1,Nexps):
5429	exp2 = exps[iexp2]
5430	Sexps = exp2 + '-' + exp1
5431	owdirme = owdir + '/' + mod + '/' + Sexps
5432	print ' diffs:', mod + '/' + Sexps + ' ...'
5433	ldiffmodexps.append(mod + '/' + Sexps)
5434	for diffn in ['op', 'var']:
5435	fileplotinf = owdirme + '/diff' + diffn + 'plotsdraw.inf'
5436	if os.path.isfile(fileplotinf):
5437	otexf.write('% ' + mod + ' ' + Sexps + ' diff' + diffn + '\n')
5438	otexf.write('\\modexp{' + gen.latex_text(mod+' '+Sexps+' diff'+ \
5439	diffn)+ '}\n')
5440	listplots, Nplt = read_plot_file(fileplotinf)
5441	for plot in listplots.keys():
5442	plots = listplots[plot]
5443	ops = []
5444	for plt in plots:
5445	opn = plt.split('#')[0].split('\|')[1]
5446	if not gen.searchInlist(ops,opn): ops.append(opn)
5447	for op in ops:
5448	opS = op.replace('+','-')
5449	modexpimg = gen.files_folder_HMT(owdirme, head=plot, \
5450	middle=opS, tail=cnf['kindfig'])
5451	Nimages = len(modexpimg)
5452	if Nimages > 0:
5453	for img in range(Nimages): modexpimg[img] = owdirme +\
5454	'/' + modexpimg[img]
5455	caption = mod + ' ' + Sexps + ' ' + plot + ' ' + op +\
5456	' diff' + diffn
5457	flab = 'fig:' + caption.replace(' ', '_')
5458	try:
5459	with gen.Capturing() as output:
5460	gen.latex_fig_array(modexpimg, otexf, \
5461	gen.latex_text(caption), flab, refsize=0.5,\
5462	dist='sqr', dorest='center')
5463	except:
5464	print errmsg
5465	print 'gen.latex_fig_array(',modexpimg,', '+ \
5466	otexf+', gen.latex_text(' + caption + '), ' + \
5467	flab + ", refsize=0.5, dist='sqr', " + \
5468	"dorest='center')"
5469	for s1out in output: print s1out
5470	quit(-1)
5471
5472	otexf.write('%\\clearpage\n')
5473	print ' mods-exps diffs ...'
5474	for imod1 in range(0,Nmods-1):
5475	mod1 = mods[imod1]
5476	exps1 = modexps[mod1]
5477	for imod2 in range(imod1+1,Nmods):
5478	mod2 = mods[imod2]
5479	exps2 = modexps[mod2]
5480
5481	Smods = mod2 + '-' + mod1
5482	Nexps1 = len(exps1)
5483	Nexps2 = len(exps2)
5484	for iexp1 in range(0,Nexps1):
5485	exp1 = exps1[iexp1]
5486	for iexp2 in range(0,Nexps2):
5487	exp2 = exps2[iexp2]
5488	Sexps = exp2 + '-' + exp1
5489	print ' ' + Smods + ' ' + Sexps + ' ...'
5490	ldiffmodexps.append(Smods + '/' + Sexps)
5491	owdirme = owdir + '/' + Smods + '/' + Sexps
5492	for diffn in ['op', 'var']:
5493	fileplotinf = owdirme + '/diff' + diffn + 'plotsdraw.inf'
5494	if os.path.isfile(fileplotinf):
5495	otexf.write('% '+Smods + ' ' + Sexps + ' diff'+diffn+'\n')
5496	otexf.write('\\modexp{' + gen.latex_text(Smods+' '+Sexps+\
5497	' diff'+diffn) + '}\n')
5498	listplots, Nplt = read_plot_file(fileplotinf)
5499	for plot in listplots.keys():
5500	plots = listplots[plot]
5501	ops = []
5502	for plt in plots:
5503	opn = plt.split('#')[0].split('\|')[1]
5504	if not gen.searchInlist(ops,opn): ops.append(opn)
5505	for op in ops:
5506	opS = op.replace('+','-')
5507	modexpimg = gen.files_folder_HMT(owdirme, \
5508	head=plot, middle=opS, tail=cnf['kindfig'])
5509	Nimages = len(modexpimg)
5510	if Nimages > 0:
5511	for img in range(Nimages): modexpimg[img] = \
5512	owdirme + '/' + modexpimg[img]
5513	caption = Smods + ' ' + Sexps + ' ' + plot + \
5514	' ' + op + ' diff' + diffn
5515	flab = 'fig:' + caption.replace(' ', '_')
5516	try:
5517	with gen.Capturing() as output:
5518	gen.latex_fig_array(modexpimg, otexf,\
5519	gen.latex_text(caption), flab, \
5520	refsize=0.5, dist='sqr', \
5521	dorest='center')
5522	except:
5523	print errmsg
5524	print 'gen.latex_fig_array(',modexpimg, \
5525	', '+otexf+', gen.latex_text(' + \
5526	caption + '), '+flab+ ", refsize=0.5,"+\
5527	" dist='sqr', dorest='center')"
5528	for s1out in output: print s1out
5529	quit(-1)
5530	otexf.write('%\\clearpage\n')
5531
5532	# allmodexp
5533	owdirme = owdir + '/allmodexps'
5534	print ' allmodexps...'
5535	fileplotinf = owdirme + '/allmodexpfigures.inf'
5536	if os.path.isfile(fileplotinf):
5537	otexf.write('% allmodexps\n')
5538	otexf.write('\\modexp{allmodexps}\n')
5539	listplots, Nplt = read_plot_file(fileplotinf)
5540	for plot in listplots.keys():
5541	plots = listplots[plot]
5542	ops = []
5543	for plt in plots:
5544	opn = plt.split('\|')[1]
5545	if not gen.searchInlist(ops,opn): ops.append(opn)
5546	for op in ops:
5547	opS = op.replace('+','-')
5548	modexpimg = gen.files_folder_HMT(owdirme, head=plot, middle=opS, \
5549	tail=cnf['kindfig'])
5550	Nimages = len(modexpimg)
5551	if Nimages > 0:
5552	for img in range(Nimages): modexpimg[img] = owdirme + '/' + \
5553	modexpimg[img]
5554	caption = 'allmodexps ' + plot + ' ' + op + ' diff' + diffn
5555	flab = 'fig:' + caption.replace(' ', '_')
5556	try:
5557	with gen.Capturing() as output:
5558	gen.latex_fig_array(modexpimg,otexf, gen.latex_text(caption),\
5559	flab, refsize=0.9, dist='sqr', dorest='center')
5560	except:
5561	print errmsg
5562	print 'gen.latex_fig_array(',modexpimg,', '+otexf+ \
5563	', gen.latex_text(' + caption + '), ' +flab+ ", refsize=0.9," +\
5564	" dist='sqr', dorest='center')"
5565	for s1out in output: print s1out
5566	quit(-1)
5567
5568	otexf.write('%\\clearpage\n')
5569	#
5570	# Grouping by type of figure
5571	#
5572	Nmods = len(mods)
5573	# Figures from mod/exp pairs
5574	owsearch = owdir + '/' + mods[0] + '/' + modexps[mods[0]][0]
5575	print " Looking in '" + owsearch + "' for figures by kind and variable..."
5576	otexf.write('% figures by kind and variable from:' + owsearch + '\n')
5577	modexpimg = gen.files_folder_HMT(owsearch, tail=cnf['kindfig'])
5578	# dictionary with kind of plots
5579	dplots = {}
5580	# list with kind of plots
5581	lplots = []
5582	# dictionary with variable-statistics of each kind of plot
5583	dvarstats = {}
5584	# list with variables of each kind of plot
5585	lvars = []
5586	for plot in modexpimg:
5587	Nvars, plotk, varn, statn = figinf_figname(plot,cnf)
5588	if not gen.searchInlist(lplots,plotk):
5589	lplots.append(plotk)
5590	if not dplots.has_key(plotk+'_'+varn):
5591	if not gen.searchInlist(lvars,varn): lvars.append(varn)
5592	dplots[plotk+'_'+varn] = [statn]
5593	else:
5594	vals = dplots[plotk+'_'+varn]
5595	dplots[plotk+'_'+varn] = vals + [statn]
5596	else:
5597	if not dplots.has_key(plotk+'_'+varn):
5598	if not gen.searchInlist(lvars,varn): lvars.append(varn)
5599	dplots[plotk+'_'+varn] = [statn]
5600	else:
5601	vals = dplots[plotk+'_'+varn]
5602	dplots[plotk+'_'+varn] = vals + [statn]
5603
5604	# Figures found
5605	print ' direct figures to group:', modexpimg
5606	print ' plots:', lplots
5607	print ' variables:', lvars
5608
5609	for dplot in dplots.keys():
5610	plot = dplot.split('_')[0]
5611	var = dplot.split('_')[1]
5612	otexf.write('% ' + var + '\n')
5613	otexf.write('\\modexp{' + gen.latex_text(var) + '}\n')
5614	varn1 = var.split('-')[0]
5615	varn2 = var.split('-')[1]
5616	varn = varn1+'-'+varn2
5617	for stn12 in dplots[dplot]:
5618	stn1 = stn12.split('@')[0]
5619	stn2 = stn12.split('@')[1]
5620	stn = gen.lstring_values(stn12, '@',' & ')
5621	plotvarfigs = []
5622	caption = varn.replace('-',' & ') + ' ' + plot + ' ' + stn
5623	flab = 'fig:' + varn + '_' + plot + '_' + stn12 + '_allmodexp'
5624	for me in lmodexps:
5625	modexpn = expGC[me].label
5626	modn = me.split('/')[0]
5627	imgns = gen.files_folder_HMT(owdir+'/'+me, head=plot+'_'+varn1, \
5628	middle=stn1+'-'+varn2, tail=stn2+'.'+cnf['kindfig'])
5629	caption = caption + ', ' + modexpn
5630	imgn = owdir+'/'+me+'/'+imgns[0]
5631	if os.path.isfile(imgn):
5632	plotvarfigs.append(imgn)
5633	if dbg:
5634	print ' Grouping figures:', plotvarfigs
5635	try:
5636	with gen.Capturing() as output:
5637	gen.latex_fig_array(plotvarfigs, otexf, gen.latex_text(caption), \
5638	flab, refsize=0.9, dist='sqr', dorest='center')
5639	except:
5640	print errmsg
5641	print 'gen.latex_fig_array(',plotvarfigs,', '+otexf+', gen.latex_text(' +\
5642	caption + '), ' + flab + ", refsize=0.9, dist='sqr', dorest='center')"
5643	for s1out in output: print s1out
5644	quit(-1)
5645
5646	Nmods = len(mods)
5647	# Figures from mod/exp differences
5648	owsearch = owdir + '/' + ldiffmodexps[0]
5649	print " Looking in '" + owsearch + "' for difference figures by kind and variable..."
5650	otexf.write('% difference figures by kind and variable from:' + owsearch + '\n')
5651	modexpimg = gen.files_folder_HMT(owsearch, tail=cnf['kindfig'])
5652	# dictionary with kind of plots
5653	dplots = {}
5654	# list with kind of plots
5655	lplots = []
5656	# dictionary with variable-statistics of each kind of plot
5657	dvarstats = {}
5658	# list with variables of each kind of plot
5659	lvars = []
5660	for plot in modexpimg:
5661	Nvars, plotk, varn, statn = figinf_figname(plot,cnf)
5662	if not gen.searchInlist(lplots,plotk):
5663	lplots.append(plotk)
5664	if not dplots.has_key(plotk+'_'+varn):
5665	if not gen.searchInlist(lvars,varn): lvars.append(varn)
5666	dplots[plotk+'_'+varn] = [statn]
5667	else:
5668	vals = dplots[plotk+'_'+varn]
5669	dplots[plotk+'_'+varn] = vals + [statn]
5670	else:
5671	if not dplots.has_key(plotk+'_'+varn):
5672	if not gen.searchInlist(lvars,varn): lvars.append(varn)
5673	dplots[plotk+'_'+varn] = [statn]
5674	else:
5675	vals = dplots[plotk+'_'+varn]
5676	dplots[plotk+'_'+varn] = vals + [statn]
5677
5678	# Figures found
5679	print ' difference figures to group:', modexpimg
5680	print ' plots:', lplots
5681	print ' variables:', lvars
5682
5683	for dplot in dplots.keys():
5684	plot = dplot.split('_')[0]
5685	var = dplot.split('_')[1]
5686	otexf.write('% ' + var + '\n')
5687	otexf.write('\\modexp{diff ' + gen.latex_text(var) + '}\n')
5688	varn1 = var.split('-')[0]
5689	varn2 = var.split('-')[1]
5690	varn = varn1+'-'+varn2
5691	for stn12 in dplots[dplot]:
5692	stn1 = stn12.split('@')[0]
5693	stn2 = stn12.split('@')[1]
5694	stn = gen.lstring_values(stn12, '@',' & ')
5695	plotvarfigs = []
5696	caption = varn.replace('-',' & ') + ' ' + plot + ' ' + stn
5697	flab = 'fig:' + varn + '_' + plot + '_' + stn12 + '_alldiffmodexp'
5698	for me in ldiffmodexps:
5699	mods = me.split('/')[0]
5700	exps = me.split('/')[1]
5701	if exps.find('-') != -1:
5702	expn1 = exps.split('-')[0]
5703	expn2 = exps.split('-')[1]
5704	else:
5705	expn1 = exps
5706	expn2 = exps
5707
5708	if mods.find('-') != -1:
5709	modn1 = mods.split('-')[0]
5710	modn2 = mods.split('-')[1]
5711	else:
5712	modn1 = mods
5713	modn2 = mods
5714	modexpn1 = expGC[modn1 + '/' + expn1].label
5715	modexpn2 = expGC[modn2 + '/' + expn2].label
5716	modexpn = modexpn1 + '-' + modexpn2
5717
5718	modn = me.split('/')[0]
5719	imgns = gen.files_folder_HMT(owdir+'/'+me, head=plot+'_'+varn1, \
5720	middle=stn1+'-'+varn2, tail=stn2+'.'+cnf['kindfig'])
5721	caption = caption + ', ' + modexpn
5722	imgn = owdir+'/'+me+'/'+imgns[0]
5723	if os.path.isfile(imgn):
5724	plotvarfigs.append(imgn)
5725	if dbg:
5726	print ' Grouping figures:', plotvarfigs
5727	try:
5728	with gen.Capturing() as output:
5729	gen.latex_fig_array(plotvarfigs, otexf, gen.latex_text(caption), \
5730	flab, refsize=0.9, dist='sqr', dorest='center')
5731	except:
5732	print errmsg
5733	print 'gen.latex_fig_array(',plotvarfigs,', '+otexf+', gen.latex_text('+ \
5734	caption + '), ' + flab + ", refsize=0.9, dist='sqr', dorest='center')"
5735	for s1out in output: print s1out
5736	quit(-1)
5737
5738	otexf.write('\\end{document}\n')
5739	otexf.close()
5740
5741	try:
5742	with gen.Capturing() as output:
5743	ins = 'pdflatex ' + owdir + '/' + texout
5744	sout = sub.call(ins, shell=True)
5745	except:
5746	print errormsg
5747	print ' ' + ins
5748	print sout
5749	for s1out in output: print s1out
5750	quit(-1)
5751
5752	with gen.Capturing() as output:
5753	sout = sub.call(ins + '>& /dev/null', shell=True)
5754	with gen.Capturing() as output:
5755	sout = sub.call(ins + '>& /dev/null', shell=True)
5756	with gen.Capturing() as output:
5757	sout = sub.call(ins + '>& /dev/null', shell=True)
5758	with gen.Capturing() as output:
5759	sout = sub.call(ins, shell=True)
5760	if dbg:
5761	print sout
5762	for s1out in output: print s1out
5763
5764	sub.call('evince ' + owdir + '/' + texout + '.pdf &', shell=True)
5765

Note: See TracBrowser for help on using the repository browser.

Download in other formats: