Context Navigation

source: lmdz_wrf/trunk/tools/model_graphics.py @ 1636

Last change on this file since 1636 was 1632, checked in by lfita, 8 years ago
Fixing typo
File size: 217.4 KB

Line
1	# Python script to generate the files and the plots for a sensitivity study with multiple modle configurations and models
2	# L. Fita
3	# LMD, Jussieu/Palaisseau, France
4	# Script configuration get from ASCII file 'model_graphics.dat'
5
6	# `Wheeler-Kiladis' diagram: https://github.com/UV-CDAT/wk
7	# $ cd /home/lluis/Downloads
8	# $ git clone https://github.com/UV-CDAT/wk
9	# $ cd wk
10	# $ su
11	# # python setup.py build >& run_pysetup.log
12	# # python setup.py install >& run_pyinstall.log
13
14	import numpy as np
15	import nc_var_tools as ncvar
16	import generic_tools as gen
17	import drawing_tools as drw
18	import time as tim
19	# To avoid errors like: '/bin/sh: 1: Syntax error: Bad fd number'
20	# Make sure that /bin/sh directs to bash and not to dash:
21	# http://stackoverflow.com/questions/15809060/sh-syntax-error-bad-fd-number
22	import subprocess as sub
23	import os
24
25	main = 'model_graphics.py'
26
27	errmsg = ncvar.errormsg
28	warnmsg = ncvar.warnmsg
29
30	def verify_configuration(ExpConf, debug):
31	""" Function to verify mandatory keys from experiment configuration
32	ExpConf= dictionary with the configuration
33	"""
34	fname = 'verify_configuration'
35
36	# List with the mandatory keys
37	mandatorykeys = ['pyHOME', 'cdoHOME', 'scratch', 'filescratch', 'figscratch', \
38	'diffscratch', 'figdiffscratch', 'figallmodexpscratch', 'addfiles', \
39	'addfigures', 'adddiffs', 'adddifffigures', 'addallmodexpfigures', 'debug', \
40	'models', 'modgraphchar', 'expgraphchar', 'ifold', 'ofold', 'warnmsg', \
41	'errmsg', 'titleoperations', 'kindfig', 'CFreftime', 'CFunitstime', 'mapval', \
42	'timekind', 'timefmt', 'timelabel' ]
43
44	# List with the optional keys
45	optionalkeys = ['specificvarplot', 'specificdiffopplot', 'specificdiffvarplot', \
46	'RefProj']
47
48	# Dictionary with the mandatory keys which are required as function of a given
49	# key from the dictionary
50	# if [key] must exist [keyB] = [values]
51	ifkeys = {'WRFexps': ['models','WRF'], 'LMDZexps': ['models','LMDZ'], \
52	'WRF_LMDZexps': ['models','WRF_LMDZ'], 'DYNAMICOexps': ['models','DYNAMICO'], \
53	'mthDYNAMICOexps': ['models','mthDYNAMICO'], \
54	'WRFheaders': ['models','WRF'], 'WRF_LMDZheaders': ['models','WRF_LMDZ'], \
55	'LMDZheaders': ['models','LMDZ'], 'DYNAMICOheaders': ['models','DYNAMICO'], \
56	'mthDYNAMICOheaders': ['models','mthDYNAMICO']}
57
58	# Dictionary with the optional keys (at least one) which are required as function
59	# of a given key from the dictionary
60	# if [key] must exist at least one of the [keyA, keyB, keyC, ...] with values
61	ifopkeys = {'RefProj': ['reprojectvar_remapbil', 'reprojectvar_remapbic', \
62	'reprojectvar_remapdis', 'reprojectvar_remapnn', 'reprojectvar_remapcon', \
63	'reprojectvar_remapcon2', 'reprojectvar_remaplaf', 'reprojectvar_dis', \
64	'reprojecvar_dis']}
65
66	for kdict in mandatorykeys:
67	if not ExpConf.has_key(kdict):
68	print errmsg
69	print ' ' + fname + "' configuration without required '" + kdict + "' !!"
70	quit(-1)
71	if debug:
72	print ExpConf[kdict]
73
74	for kdict in ifkeys.keys():
75	vals = ifkeys[kdict]
76	keyn = vals[0]
77	keyv = vals[1]
78	if type(ExpConf[keyn]) == type('A'):
79	if not ExpConf.has_key(kdict) and ExpConf[keyn] == keyv:
80	print errmsg
81	print ' ' + fname + "' configuration without '" + kdict + \
82	"' when it is required with configuration '" + keyn + '=' + keyv + \
83	"' !!"
84	quit(-1)
85	elif type(ExpConf[keyn]) == type(['A', 'B']):
86	keyvals = ExpConf[keyn]
87	if not ExpConf.has_key(kdict) and gen.searchInlist(keyvals, keyv):
88	print errmsg
89	print ' ' + fname + "' configuration without '" + kdict + \
90	"' when it is required with configuration '" + keyn + '= [..., ' + \
91	keyv + ", ...]' !!"
92	quit(-1)
93	else:
94	print errmsg
95	print ' ' +fname+ 'dictionary type ', type(ExpConf[keyn]), ' not ready!!'
96	quit(-1)
97	if debug:
98	print ' Experiments configuration _______'
99	gen.printing_dictionary(ExpConf)
100
101	for kdict in ifopkeys.keys():
102	if ExpConf.has_key(kdict):
103	opkeys = ifopkeys[kdict]
104	hasopkey = False
105	opdictvals = {}
106	for opkv in opkeys:
107	if ExpConf.has_key(opkv):
108	hasopkey = True
109	opdictvals[opkv] = ExpConf[opkv]
110
111	if not hasopkey:
112	print errmsg
113	print ' ' + fname + "' configuration wit '" + kdict + "' but " + \
114	"without any key value:", opkeys, "' when it is required !!"
115	quit(-1)
116
117	if debug:
118	print " Optional key '" + kdict + "' with values ________"
119	gen.printing_dictionary(opdictvals)
120
121	print fname + ': configuration seems to be fine'
122
123	return
124
125	def scratches(config):
126	""" Function to set-up if it is needed to start from the scratch
127	config: dictionary with the configuration
128	"""
129	fname = 'scratches'
130
131	if gen.Str_Bool(config['scratch']):
132	scr = True
133	print warnmsg
134	print " " + fname + ": starting from the SCRATCH !!"
135	print " 10 seconds left!!"
136	filescr = True
137	figscr = True
138	difscr = True
139	figdifscr = True
140	moddifscr = True
141	figmoddifscr = True
142	figallmodexpscr = True
143	tim.sleep(10)
144	else:
145	scr = False
146	if gen.Str_Bool(config['filescratch']):
147	filescr = True
148	print warnmsg
149	print " " + main + ": files starting from the SCRATCH !!"
150	print " 5 seconds left!!"
151	tim.sleep(5)
152	else:
153	filescr = False
154
155	if gen.Str_Bool(config['figscratch']):
156	figscr = True
157	print warnmsg
158	print " " + main + ": figures starting from the SCRATCH !!"
159	print " 5 seconds left!!"
160	tim.sleep(5)
161	else:
162	figscr = False
163
164	if gen.Str_Bool(config['diffscratch']):
165	difscr = True
166	print warnmsg
167	print " " + main + ": experiment differences starting from the SCRATCH !!"
168	print " 5 seconds left!!"
169	tim.sleep(5)
170	else:
171	difscr = False
172
173	if gen.Str_Bool(config['figdiffscratch']):
174	figdifscr = True
175	print warnmsg
176	print " " + main + ": figures experiment differences starting from the SCRATCH !!"
177	print " 5 seconds left!!"
178	tim.sleep(5)
179	else:
180	figdifscr = False
181
182	if gen.Str_Bool(config['moddiffscratch']):
183	moddifscr = True
184	print warnmsg
185	print " " + main + ": model differences starting from the SCRATCH !!"
186	print " 5 seconds left!!"
187	tim.sleep(5)
188	else:
189	moddifscr = False
190
191	if gen.Str_Bool(config['figmoddiffscratch']):
192	figmoddifscr = True
193	print warnmsg
194	print " " + main + ": figures model differences starting from the SCRATCH !!"
195	print " 5 seconds left!!"
196	tim.sleep(5)
197	else:
198	figmoddifscr = False
199
200	if gen.Str_Bool(config['figallmodexpscratch']):
201	figallmodexpscr = True
202	print warnmsg
203	print " " + main + ": figures all model-experiment starting from the SCRATCH !!"
204	print " 5 seconds left!!"
205	tim.sleep(5)
206	else:
207	figallmodexpscr = False
208
209	if gen.Str_Bool(config['addall']):
210	print warnmsg
211	print " " + fname + ": adding new values everywhere !!"
212	addfils = True
213	addfigs = True
214	adddiffs = True
215	adddifffigs = True
216	addmoddiffs = True
217	addmoddifffigs = True
218	addallmodexpfigs = True
219	else:
220	addfils = gen.Str_Bool(config['addfiles'])
221	addfigs = gen.Str_Bool(config['addfigures'])
222	adddiffs = gen.Str_Bool(config['adddiffs'])
223	adddifffigs = gen.Str_Bool(config['adddifffigures'])
224	addmoddiffs = gen.Str_Bool(config['addmoddiffs'])
225	addmoddifffigs = gen.Str_Bool(config['addmoddifffigures'])
226	addallmodexpfigs = gen.Str_Bool(config['addallmodexpfigures'])
227
228	debug = gen.Str_Bool(config['debug'])
229
230	return scr, filescr, figscr, difscr, figdifscr, moddifscr, figmoddifscr, \
231	figallmodexpscr, addfils, addfigs, adddiffs, adddifffigs, addmoddiffs, \
232	addmoddifffigs, addallmodexpfigs, debug
233
234
235	class gc(object):
236	""" Class which holds all the graphical characteristics
237	[label]: label as it should appear in the graphics
238	[color]: specific color of the model
239	[linetype]: specific type of line of the model
240	[marker]: specific marker of the model
241	[sizes]: line width and point size for the model
242	[tmod]: time modification to apply to the model files (None for nothing)
243	'setorigin',[YYYYMMDDHHMISS]: re-set origin of times at [YYYYMMDDHHMISS]
244	"""
245	def __init__( self, label, color, linetype, marker, sizes, Tmod):
246	self.label = None
247	self.color = None
248	self.linetype = None
249	self.marker = None
250	self.sizes = None
251	self.tmod = None
252	if label is not None:
253	self.label = label
254	self.color = color
255	self.linetype = linetype
256	self.marker = marker
257	self.sizes = sizes
258	if Tmod != 'None':
259	self.tmod = Tmod
260
261	def graphical_characteristics(config, debug):
262	""" Function to provide the graphical characteristics of models and experiments
263	config= configuration of the experiment
264	"""
265	fname = 'graphical_characteristics'
266
267	modgraphchar = {}
268	modvals = config['modgraphchar'].split(':')
269	for modval in modvals:
270	modv = modval.split('\|')
271	modgraphchar[modv[0]] = gc(modv[1], modv[2], modv[3], modv[4], modv[5], modv[6])
272
273	expgraphchar = {}
274	expvals = config['expgraphchar'].split(':')
275	for expval in expvals:
276	expvs = expval.split('\|')
277	modexp = expvs[0] + '/' + expvs[1]
278	if not modgraphchar.has_key(expvs[0]):
279	print errmsg
280	print ' ' + fname + ": a model called '" +expvs[0]+ "' does not exist !!"
281	print ' existing models with graphic characteristics:', \
282	modgraphchar.keys()
283	quit(-1)
284
285	modvs = modgraphchar[expvs[0]]
286	expv = expvs[2:7]
287	if expv[1] == 'asmodel': expv[1] = modvs.color
288	if expv[2] == 'asmodel': expv[2] = modvs.linetype
289	if expv[3] == 'asmodel': expv[3] = modvs.marker
290	if expv[4] == 'asmodel': expv[4] = modvs.sizes
291
292	expgraphchar[modexp] = gc(expv[0], expv[1], expv[2], expv[3], expv[4], 'None')
293
294	# Running some tests
295	mods = config['models'].split(':')
296	for mod in mods:
297	if not modgraphchar.has_key(mod):
298	print errmsg
299	print ' ' + fname + ": model called '" + mod + "' does not have " + \
300	"graphical characteristics !!"
301	print ' existing models with graphic characteristics:', \
302	modgraphchar.keys()
303	quit(-1)
304
305	exps = config[mod + 'exps'].split(':')
306	for exp in exps:
307	modexp = mod + '/' + exp
308	if not expgraphchar.has_key(modexp):
309	print errmsg
310	print ' ' + fname + ": model/experiment called '" + modexp + \
311	"' does not have graphical characteristics !!"
312	print ' existing model/experiments with graphic ' + \
313	'characteristics:', expgraphchar.keys()
314	quit(-1)
315
316	if debug:
317	print ' ' + fname + ': model graphical characteristics _______'
318	for modn in modgraphchar.keys():
319	with gen.Capturing() as output:
320	gen.printing_class(modgraphchar[modn])
321	print "'" + modn + "'; " + ', '.join(output)
322	print ' ' + fname + ': experiment graphical characteristics _______'
323	for modexpn in expgraphchar.keys():
324	with gen.Capturing() as output:
325	gen.printing_class(expgraphchar[modexpn])
326	print "'" + modexpn + "'; " + ', '.join(output)
327
328	if debug: print " End of '" + fname + "' "
329
330	return modgraphchar, expgraphchar
331
332	def exp_headers(mod,config):
333	""" Function to provide the headers and the experiments of a given model
334	model= model
335	config= configuration of the experiment
336	"""
337	fname = 'exp_headers'
338
339	# No CASE in python!
340	# case ${mod} in ...
341	if mod == 'WRF':
342	expers = config['WRFexps'].split(':')
343	fhs = config['WRFheaders'].split(':')
344	elif mod == 'LMDZ':
345	expers = config['LMDZexps'].split(':')
346	fhs = config['LMDZheaders'].split(':')
347	elif mod == 'WRF_LMDZ':
348	expers = config['WRF_LMDZexps'].split(':')
349	fhs = config['WRF_LMDZheaders'].split(':')
350	elif mod == 'DYNAMICO':
351	expers = config['DYNAMICOexps'].split(':')
352	fhs = config['DYNAMICOheaders'].split(':')
353	elif mod == 'mthDYNAMICO':
354	expers = config['mthDYNAMICOexps'].split(':')
355	fhs = config['mthDYNAMICOheaders'].split(':')
356	else:
357	print errmsg
358	print " " + fname + ": model '" + mod + "' not ready!!"
359	quit(-1)
360
361	return expers, fhs
362
363	class VariableInf(object):
364	""" Class which holds all information of a given variable
365	name= CF name of the variabgle
366	header= header of the file from which it can be computed
367	model= file-header variable from which it can be directly computed
368	diag= file-header diagnostics from which it can be computed
369	"""
370	def __init__( self, name, fheader, model, diag):
371	self.name= None
372	self.fheader= None
373	self.model= None
374	self.diag= None
375	if name is not None:
376	self.name = name
377	self.fheader= fheader
378	self.model= model
379	self.diag= diag
380
381	class ModelInf(object):
382	""" Class which holds all information from a given model
383	name= Acronym of the model
384	model= Long description of the model
385	dim[x/y/z/t/s]n= name of the dimensions of the model
386	vdim[x/y/z/t/s]n= name of the variable-dimensions of the model
387	dimensions= list of the dimensions
388	vardimensions= list of the variable-dimensions
389	timemodif= time modification to apply to the model files ('None' for nothing)
390	'setorigin',[YYYYMMDDHHMISS]: re-set origin of times at [YYYYMMDDHHMISS]
391	"""
392	def __init__( self, name, model, dx, dy, dz, dt, ds, vdx, vdy, vdz, vdt, vds, \
393	tmod):
394	self.name= None
395	self.model= None
396	self.dimxn= None
397	self.dimyn= None
398	self.dimzn= None
399	self.dimtn= None
400	self.dimsn= None
401	self.vardxn= None
402	self.vardyn= None
403	self.vardzn= None
404	self.vardtn= None
405	self.vardsn= None
406	self.timemodif = None
407	if name is not None:
408	self.name = name
409	self.model= model
410	self.dimxn= dx
411	self.dimyn= dy
412	self.dimzn= dz
413	self.dimtn= dt
414	self.dimsn= ds
415	self.vardxn= vdx
416	self.vardyn= vdy
417	self.vardzn= vdz
418	self.vardtn= vdt
419	self.vardsn= vds
420	if tmod is not None:
421	self.timemodif = tmod
422	self.dimensions = [dx, dy, dz, dt, ds]
423	self.vardimensions = [vdx, vdy, vdz, vdt, vds]
424
425	def variable_compute(idir,var,ftests,db):
426	""" Function to retrieve the computation way of a given variable using a series of test files
427	iwdir= directory with the test files
428	var= name of the variable to test
429	filtests= dictionary with the test files for each header
430	"""
431	fname='variable_compute'
432
433	cancompute = True
434	for headerf in ftests.keys():
435	filen=ftests[headerf]
436	try:
437	with gen.Capturing() as output:
438	vmod, vdiag = ncvar.computevar_model(var, idir + '/' + filen)
439	except:
440	print errmsg
441	print 'ncvar.computevar_model(' + var + ', ' + idir + '/' + filen + ')'
442	for s1out in output: print s1out
443	quit(-1)
444
445	if vmod is None and vdiag is None:
446	cancompute = False
447	else:
448	cancompute = True
449	# Should be considered that variable can also be computed by both ways?
450	break
451
452	if not cancompute:
453	print warnmsg
454	print ' ' + fname + ": there is no way to compute '" + var + \
455	"' for model '" + mod
456	# Too extrict!
457	# quit(-1)
458
459	# Getting only the first possibility of model and diagnostic
460	if vmod is not None:
461	modV = vmod[0]
462	else:
463	modV = None
464	if vdiag is not None:
465	diagV = vdiag[0]
466	else:
467	diagV = None
468
469	varcomp = VariableInf(var, headerf, modV, diagV)
470
471	# a ';' list 'varcompute' it is created for each variable giving:
472	# [var]\|[vark]\|[headerf][varmod]\|[vardiag]
473	# This list will be used to compute a new file for each variable
474
475	if db:
476	print 'Variable information _______'
477	gen.printing_class(varcomp)
478
479	if db: print " End of '" + fname + "' "
480
481	return varcomp
482
483	def get_operations_var(vc,db):
484	""" Function to provide the operations to make for each variable from 'VAR_' dictionary
485	vc= 'VAR_' dictionary, dictionary with all the parameters started with 'VAR_'
486	"""
487	fname = 'get_operations_var'
488
489	LH = len('VAR_')
490	iop = 1
491	# list of operations
492	doopers = {}
493	# operations by variable
494	operationsvar = {}
495	# individual operations by variable (not repeated)
496	indivoperationsvar = {}
497
498	# Variables with a calculation whic requires vertical interpolation to all the
499	# file (operations stating by 'VAR_pinterp')
500	LVp = len('VAR_pinterp')
501	varglobalp = []
502
503	for oper in vc.keys():
504	Loper = len(oper)
505	opn = oper[LH:Loper+1]
506	vns = vc[oper].split(':')
507	ops1 = opn.split('+')
508	doopers[opn] = ops1
509	for vn in vns:
510	if not operationsvar.has_key(vn):
511	operationsvar[vn] = [opn]
512	indivoperationsvar[vn] = ops1
513	else:
514	opers = operationsvar[vn]
515	opers.append(opn)
516	operationsvar[vn] = opers
517	opers1 = indivoperationsvar[vn]
518	for op1 in ops1:
519	if not gen.searchInlist(opers1, op1):
520	opers1.append(op1)
521	indivoperationsvar[vn] = opers1
522
523	if oper[0:LVp] == 'VAR_pinterp':
524	for vn in vns:
525	if not gen.searchInlist(varglobalp,vn): varglobalp.append(vn)
526
527	iop = iop + 1
528
529	if db:
530	print ' operations to make _______'
531	gen.printing_dictionary(doopers)
532	print ' operations by variable _______'
533	gen.printing_dictionary(operationsvar)
534	print ' individual operations by variable _______'
535	gen.printing_dictionary(indivoperationsvar)
536	print ' variables with the vertical interpolation for all the file _______'
537	print ' #', varglobalp
538
539	if db: print " End of '" + fname + "' "
540
541	return doopers, operationsvar, indivoperationsvar, varglobalp
542
543	def pinterp_var(oper):
544	""" Function to retrieve characteristics of the vertical interpolation for the operation
545	oper= operation
546	Wether vertical interpolation is:
547	# 'global': for all file
548	# 'local': at a given step of the process
549	# 'none': no vertical interpolation
550	>>> pinterp_var('last+pinterp+xmean')
551	local
552	>>> pinterp_var('pinterp+tmean+xmean')
553	global
554	>>> pinterp_var('xmean')
555	none
556	"""
557	fname = 'pinterp_var'
558
559	pinn = 'pinterp'
560
561	Lpinterp = len(pinn)
562	if oper[0:Lpinterp] == pinn:
563	pkind = 'global'
564	return pkind
565
566	if oper.find(pinn) != -1:
567	pkind = 'local'
568	else:
569	pkind = 'none'
570
571	return pkind
572
573	def compvars_listconstruct(config, minf, Files, TestFiles, idir, odir, debug):
574	""" Function to construct the list of variables to compute
575	config= dictionary with the configuration of the execution
576	variables to compute are get with all values started by `VAR_' in 'module_graphics.dat', and then
577	providing a consecutive number of calculations separated by '+'
578	VAR_[calc1]+[calc2] = tas:wss
579	will compute first [calc1] and then [calc2] for 'tas' and 'wss'
580	minf= class with information about the model
581	Files= dictionary of files for heach header
582	TestFiles= dictionary of files to test how to compute the variable
583	odir= output directory
584	"""
585	fname='compvars_listconstruct'
586
587	varcomp = gen.get_specdictionary_HMT(config,H='VAR_')
588
589	if debug:
590	print ' variables to compute ________'
591	gen.printing_dictionary(varcomp)
592
593	# Getting operations by variable
594	opers, varoper, indivaroper, varglobp = get_operations_var(varcomp,debug)
595
596	strfiles = gen.dictKeysVals_stringList(Files)
597	strtestfiles = gen.dictKeysVals_stringList(TestFiles)
598
599	Svarcompute = ''
600
601	# Main dictionary with all the variables and their calculation
602	allvarcomp = {}
603
604	ivop = 0
605	for vn in varoper:
606	vcomp = variable_compute(idir,vn,TestFiles,False)
607	for op in varoper[vn]:
608	# Creation of a String as: [CFname]\|[operation]\|[fheader]\|[vmodel]\|[vdiag]\|[globalP]
609	# [CFname]: CF-name of the variable (must appear in 'variables_values.dat')
610	# [operation]: operation to compute
611	# [fheader]: header of file with the required variables
612	# [vmodel]: Variable from model output
613	# [vdiag]: Varible as diagnostic from different variables of the model output
614	# [globalP]: Wether vertical interpolation is:
615	# 'global': for all file
616	# 'local': at a given step of the process
617	# 'none': no vertical interpolation
618	globalP = pinterp_var(op)
619	if vcomp.model is not None:
620	if type(vcomp.model) == type(list([1, 2])):
621	Smodel = ':'.join(vcomp.model)
622	elif type(vcomp.model) == type('A'):
623	Smodel = vcomp.model
624	else:
625	Smodel = 'None'
626	if vcomp.diag is not None:
627	if type(vcomp.diag) == type(list([1, 2])):
628	Sdiag = ':'.join(vcomp.diag)
629	elif type(vcomp.diag) == type('A'):
630	Sdiag = vcomp.diag
631	else:
632	Sdiag = 'None'
633
634	# To get a combination of operations, all the precedent steps are kept
635	# thus, they are also available !
636	seqops = op.split('+')
637	for seqop in seqops:
638	if seqop == seqops[0]: seqopS=seqop
639	else: seqopS = seqopS + '+' + seqop
640	allvarcomp[vn+'_'+seqopS] = [vcomp.fheader, vcomp.model, vcomp.diag, \
641	globalP]
642
643	Svc = vcomp.name + '\|' + seqopS + '\|' + vcomp.fheader + '\|' + \
644	Smodel + '\|' + Sdiag + '\|' + globalP
645	if ivop == 0:
646	Svarcompute = Svc
647	else:
648	Svarcompute = Svarcompute + ',' + Svc
649
650	ivop = ivop + 1
651
652	if debug:
653	print ' Variables to compute _______'
654	gen.printing_dictionary(allvarcomp)
655
656	# Outwritting the varcompute to avoid next time (if it is not filescratch!)
657	objf = open(odir + '/varcompute.inf', 'w')
658	objf.write('files: ' + strfiles + '\n')
659	objf.write('testfiles: ' + strtestfiles + '\n')
660	objf.write('varcompute: ' + Svarcompute + '\n')
661	objf.write('itotv: ' + str(ivop) + '\n')
662	objf.close()
663
664	if debug: print " End of '" + fname + "' "
665
666	return allvarcomp, ivop
667
668	def read_varcomp_file(filen):
669	""" Function to read 'varcompute.inf' and reconstruct the dictionaries
670	"""
671	fname = 'read_varcomp_file'
672
673	allvarcomp = {}
674
675	objf = open(filen, 'r')
676
677	for line in objf:
678	vals = line.replace('\n','').split(' ')
679	if vals[0] == 'files:':
680	Files = gen.stringList_dictKeysVals(vals[1])
681	elif vals[0] == 'testfiles:':
682	TestFiles = gen.stringList_dictKeysVals(vals[1])
683	elif vals[0] == 'varcompute:':
684	for VvalsS in vals[1].split(','):
685
686	Vvals = VvalsS.split('\|')
687	if Vvals[3] == 'None':
688	mod = None
689	else:
690	mod = Vvals[3].split(':')
691	if Vvals[4] == 'None':
692	diag = None
693	else:
694	diag = Vvals[4].split(':')
695
696	allvarcomp[Vvals[0]+'_'+Vvals[1]] = [Vvals[2], mod, diag, Vvals[5]]
697
698	# # To get a combination of operations, all the precedent steps are kept
699	# # thus, they are also available !
700	# seqops = Vvals[1].split('+')
701	# for seqop in seqops:
702	# if seqop == seqops[0]: seqopS=seqop
703	# else: seqopS = seqopS + '+' + seqop
704	# allvarcomp[Vvals[0]+'_'+seqopS] = [Vvals[2], mod, diag, Vvals[5]]
705	elif vals[0] == 'itotv:':
706	ivop = int(vals[1])
707
708	objf.close()
709
710	return Files, TestFiles, allvarcomp, ivop
711
712	def pinterpS(gP):
713	""" For that variables which require vertical interpolation 'p' suffix to the file header is added
714	gP= kind of pinterp: 'global', 'local', 'none'
715	"""
716	fname = 'pinterpS'
717
718	if gP != 'none':
719	gPS = 'p'
720	else:
721	gPS = ''
722
723	return gPS
724
725	def compute_variable(minf, idir, usefiles, odir, cvar, gP, scr, pyH, Tref, Tunits, db):
726	""" Function to compute a variable
727	minf= class with the information of the model
728	idir= directory with the input files
729	usefiles= dictionary of files as dict([headerf]) = [file1], ..., [fileN]
730	odir= directory to write the output files
731	cvar= class with the information of the variable: 'name', 'fheader', 'varmod', 'vardiag'
732	gP= kind of vertical interpolation ('global', 'local', 'none')
733	scr= should it be done from the scratch?
734	pyH= location of the python HOME
735	Tref= CF time reference
736	Tunits= CF time units
737	"""
738	fname='compute_variable'
739
740	CFvarn=cvar.name
741	headerf = cvar.fheader
742	modvar = cvar.model
743	diagvar = cvar.diag
744
745	cfiles = usefiles[headerf]
746
747	# dimensions
748	dnx = minf.dimxn
749	dny = minf.dimyn
750	# var-dimensions
751	vdnx = minf.vardxn
752	vdny = minf.vardyn
753
754	# Computing separately and then joinging for all files
755	Ntotfiles = len(cfiles)
756	Nzeros = len(str(Ntotfiles))
757	NStot = str(Ntotfiles).zfill(Nzeros)
758	# For that variables which require vertical interpolation 'p' suffix to the
759	# file header is added
760	SgP = pinterpS(gP)
761
762	# Getting in working dir
763	os.chdir(odir)
764
765	# File to keep track of all operations
766	otrackf = open( odir + '/all_computevars.inf', 'a')
767
768	# Computing variable
769	ifile=1
770	for cf in cfiles:
771	ifS = str(ifile).zfill(Nzeros)
772	ifilen = odir + '/' + CFvarn + '_' + headerf + SgP + '_' + ifS + '-' + \
773	NStot + '.nc'
774	fileon = odir + '/' + CFvarn + '_' + headerf + SgP + '.nc'
775
776	if scr:
777	sout = sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
778	sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
779
780	if not os.path.isfile(ifilen) and not os.path.isfile(fileon):
781	# Since model direct values are retrieved from `variables_valules.dat' which was
782	# initially coincived as a way to only give variable attributes, range and color
783	# bars, if a variable has a diagnostic way to be computed, the later one will be
784	# preferred
785	if db:
786	print ' ' + fname + ": creation of variable file '" + CFvarn + \
787	"' in file '" + ifilen + "' ..."
788	print ' model variable:', modvar
789	print ' diagostic variable:', diagvar
790
791	if diagvar is None:
792	# model variable
793	if db: print ' '+fname+ ": variable direct from model '" +modvar+ "'"
794	values = modvar + ',0,-1,-1'
795	vs = modvar + ',' + vdnx + ',' + vdny + ',' + vdnz + ',' + vdnt
796
797	try:
798	with gen.Capturing() as output:
799	ncvar.DataSetSection_multivars(values, idir + '/' + cf, vs)
800	except:
801	print errmsg
802	print 'ncvar.DataSetSection_multivars('+values+', '+idir+'/'+cf+ \
803	', '+vs+')'
804	for s1out in output: print s1out
805	quit(-1)
806
807	newfile, loc = gen.search_sec_list(output,'succesfull')
808	ofile = newfile[0].split(' ')[7]
809	sub.call('mv ' + ofile + ' ' + ifilen, shell=True)
810
811	# Keeping track of the operations
812	pyins = pyH + "/nc_var.py -f " + idir + '/' + cf + \
813	" -o DataSetSection_multivars -v " + vs + " -S '" + values + "'"
814	otrackf.write('\n')
815	otrackf.write('# ' + CFvarn + " " + modvar + '\n')
816	otrackf.write('python ' + pyins + '\n')
817
818	# CF renaming of variable
819	ncvar.chvarname(CFvarn,ifilen,modvar)
820	else:
821	# diagnostic variable
822	if db: print ' '+fname+ ": variable as diagnostic '" +diagvar[0]+ "'"
823	if minf.name == 'WRF':
824	dims = dnt+'@WRFtime,'+dnz+'@'+vdnz+','+dny+'@'+vdny+','+dnx+ \
825	'@'+vdnx
826	else:
827	dims = dnt+'@'+vdnt+','+dnz+'@'+vdnz+','+dny+'@'+vdny+','+dnx+ \
828	'@'+vdnx
829	diagn = diagvar[0]
830	Ndiagvars = len(diagvar)
831	diagc = '@'.join(diagvar[1:])
832
833	values = '-f ' + idir + '/' + cf + " -d '" + dims + "' -v '" + \
834	diagn + '\|' + diagc + "'"
835	try:
836	with gen.Capturing() as output:
837	sout = sub.call('python ' +pyH+ '/diagnostics.py ' + values, \
838	shell=True)
839	except:
840	print errmsg
841	print 'python ' + pyH + '/diagnostics.py ' + values
842	for s1out in output: print s1out
843	quit(-1)
844	if db:
845	for s1out in output: print s1out
846
847	sout = sub.call(' mv diagnostics.nc ' + ifilen, shell=True)
848
849	# Keeping track of the operations
850	pyins = 'python ' + pyH + '/diagnostics.py ' + values
851	otrackf.write('\n')
852	otrackf.write('# ' + CFvarn + " " + diagn + '\n')
853	otrackf.write(pyins + '\n')
854
855	# Attaching necessary variables for the pressure interpolation
856	if gP != 'none':
857	if minf.name == 'WRF' or minf.name == 'WRF_LMDZ':
858	requiredinterpvars = ['P', 'PB', 'PSFC', 'PH', 'PHB', 'HGT', 'T',\
859	'QVAPOR', 'XLONG', 'XLAT', 'Times']
860	elif minf.name == 'LMDZ':
861	requiredinterpvars = ['pres', 'psol', 'geop', 'phis', 'temp', \
862	'ovap', 'lon', 'lat', 'time_counter']
863	elif minf.name == 'DYNAMICO':
864	requiredinterpvars = ['pres', 'psol', 'geop', 'phis', 'temp', \
865	'ovap', 'lon', 'lat', 'time_counter']
866	elif minf.name == 'mthDYNAMICO':
867	requiredinterpvars = ['pres', 'psol', 'geop', 'phis', 'temp', \
868	'ovap', 'lon', 'lat', 'time_centered']
869	else:
870	print errmsg
871	print fname + ": for model '" + minf.name + "' required " + \
872	" variables for vertical interpolation are not known !!"
873	quit(-1)
874
875	print " " + fname + ": adding variables:", requiredinterpvars, \
876	' to allow pressure interpolation'
877	for rqv in requiredinterpvars:
878	try:
879	with gen.Capturing() as output:
880	ncvar.fvaradd(idir+'/'+cf+','+rqv,ifilen)
881	except:
882	print errmsg
883	print 'fvaradd('+idir+'/'+cf+','+rqv+', '+ifilen+')'
884	for s1out in output: print s1out
885	quit(-1)
886	if db:
887	for s1out in output: print s1out
888
889	# CFification of files
890	if minf.name == 'WRF' or minf.name == 'WRF_LMDZ':
891	values = vdnx + ':' + vdny + ':' + vdnt + ':'+ Tref + ':' + Tunits
892	try:
893	with gen.Capturing() as output:
894	ncvar.WRF_to_newCF(values, ifilen, 'all')
895	sub.call('mv CF_WRF.nc '+ifilen + ' >& /dev/null', shell=True)
896	except:
897	print errmsg
898	print 'WRF_to_newCF('+ values +', ' + ifilen + ',all)'
899	for s1out in output: print s1out
900	# Removing file in order to make sure that it will be redone
901	print ' ' + fname + " removing file '" + ifilen + "' ..."
902	sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
903	quit(-1)
904	if db:
905	for s1out in output: print s1out
906	elif minf.name == 'LMDZ':
907	try:
908	with gen.Capturing() as output:
909	ncvar.LMDZ_toCF(Tunits + '!since!' + Tref, ifilen)
910	except:
911	print errmsg
912	print 'LMDZ_toCF('+ Tunits +'!since!'+ Tref + ', ' + ifilen + ')'
913	for s1out in output: print s1out
914	# Removing file in order to make sure that it will be redone
915	print ' ' + fname + " removing file '" + ifilen + "' ..."
916	sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
917	quit(-1)
918	if db:
919	for s1out in output: print s1out
920	# At this stage, let's be simple. It might be necessary to complicate it! Since
921	# DYNAMICO might have 'time_counter' or 'time_instant'
922	elif minf.name == 'DYNAMICO':
923	try:
924	with gen.Capturing() as output:
925	ncvar.DYNAMICO_toCF(Tunits + '!since!' + Tref, ifilen)
926	except:
927	print errmsg
928	print 'DYNAMICO_toCF('+Tunits+'!since!'+Tref+ ', ' + ifilen + ')'
929	for s1out in output: print s1out
930	# Removing file in order to make sure that it will be redone
931	print ' ' + fname + " removing file '" + ifilen + "' ..."
932	sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
933	quit(-1)
934	if db:
935	for s1out in output: print s1out
936	elif minf.name == 'mthDYNAMICO':
937	try:
938	with gen.Capturing() as output:
939	ncvar.mthDYNAMICO_toCF(Tunits + '!since!' + Tref, ifilen)
940	except:
941	print errmsg
942	print 'mthDYNAMICO_toCF('+Tunits+'!since!'+Tref+ ', ' + ifilen + ')'
943	for s1out in output: print s1out
944	# Removing file in order to make sure that it will be redone
945	print ' ' + fname + " removing file '" + ifilen + "' ..."
946	sub.call('rm ' + ifilen + ' >& /dev/null', shell=True)
947	quit(-1)
948	if db:
949	for s1out in output: print s1out
950	else:
951	print errmsg
952	print ' ' + fname + ": no CFification for model '" +minf.name+ "' !!"
953	print " available ones: 'WRF', 'WRF_LMDZ', 'LMDZ', 'DYNAMICO'" + \
954	", 'mthDYNAMICO'"
955	quit(-1)
956
957	ifile = ifile + 1
958
959	otrackf.close()
960
961	# Joining variable files
962	if not os.path.isfile(fileon):
963	if Ntotfiles > 1:
964	if db:
965	print ' ' + fname + ": concatenating all variable files to " + \
966	"create '" + fileon + "..."
967	try:
968	with gen.Capturing() as output:
969	ncvar.netcdf_fold_concatenation_HMT('./,time', CFvarn + '_' + \
970	headerf + SgP +'_,-,.nc', 'all')
971	except:
972	print errmsg
973	print 'netcdf_fold_concatenation_HMT(./,time, ' + CFvarn + '_' + \
974	headerf +SgP + '_,-,.nc, all)'
975	for s1out in output: print s1out
976	quit(-1)
977	if db:
978	for s1out in output: print s1out
979
980	sout = sub.call('mv netcdf_fold_concatenated_HMT.nc '+fileon, shell=True)
981	if os.path.isfile(fileon):
982	sout = sub.call('rm '+CFvarn + '_' + headerf + SgP + '_-.nc >& ' + \
983	'/dev/null', shell=True)
984	else:
985	sout = sub.call('mv ' + CFvarn + '_' + headerf + SgP + '_1-1.nc ' + \
986	fileon, shell=True)
987
988	# Re-setting time of final concatenated file
989	if minf.timemodif is not None:
990	if db:
991	print ' ' + fname + ": Modifying times of the file by '" + \
992	minf.timemodif + "' !!"
993	try:
994	with gen.Capturing() as output:
995	ncvar.time_reset(minf.timemodif, fileon,'time')
996	except:
997	print errmsg
998	print 'time_reset(' + minf.timemodif + ', ' + fileon + ', time)'
999	for s1out in output: print s1out
1000	# Removing file in order to make sure that it will be redone
1001	print ' ' + fname + " removing file '" + fileon + "' ..."
1002	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1003	quit(-1)
1004	if db:
1005	for s1out in output: print s1out
1006
1007	if db: print " End of '" + fname + "' "
1008
1009	return
1010
1011	def compute_statistics(minf, config, idir, usefiles, odir, cvar, gP, Opers, scr, db):
1012	""" Function to compute different statistics it will take previous steps if they
1013	are availale
1014	minf= class with the information of the model
1015	config= dictionary with the configuration of the experiment
1016	idir= directory with the input files
1017	usefiles= ',' list of files to use [file1],...,[fileN]
1018	odir= directory to write the output files
1019	cvar= class with the information of the variable: 'name', 'fheader', 'varmod', 'vardiag'
1020	gP= kind of vertical interpolation ('global', 'local', 'none')
1021	Opers= kind of operation: (as possible multiple consecutive combination of operations separated by '+'
1022	[calc1]+[calc2] will compute first [calc1] and then [calc2]
1023	acc: temporal accumulated values
1024	diff: differences between models
1025	direct: no statistics
1026	last: last temporal value
1027	Lmean: latitudinal mean values
1028	Lsec: latitudinal section (latitudinal value must be given, [var]@[lat])
1029	lmean: longitudinal mean values
1030	lsec: longitudinal section (longitudinal value must be given, [var]@[lat])
1031	pinterp: pressure interpolation (to the given $plevels)
1032	smean: spatial-weighted mean values
1033	tmean: temporal mean values
1034	tstd: temporal standard deviation values
1035	tturb: Taylor's turbulence decomposition value (x - <x>) for time
1036	tvar: temporal variance values
1037	xmean: x-axis mean values
1038	xvar: x-axis variance values
1039	ymean: y-axis mean values
1040	zsum: vertical aggregated values
1041	scr= should it be done from the scratch?
1042	"""
1043	fname='compute_statistics'
1044
1045	# Getting a previous file name to continue with(for combinations)
1046	CFvarn=cvar.name
1047	headerf = cvar.fheader
1048	modvar = cvar.model
1049	diagvar = cvar.diag
1050
1051	cfiles = usefiles[headerf]
1052
1053	# dimensions
1054	dnx = minf.dimxn
1055	dny = minf.dimyn
1056	# var-dimensions
1057	vdnx = minf.vardxn
1058	vdny = minf.vardyn
1059	# Model dimension-variable dictionary
1060	Modeldimvardict = {dnx: vdnx, dny: vdny, dnz: vdnz, dnt: vdnt}
1061
1062	# Some experiment configuration values
1063	# plevels= ':' separated list of pressures (in Pa) to use for the vertical interpolation
1064	plevels = config['plevels']
1065	# pyH= location of the python HOME
1066	pyH = config['pyHOME']
1067	# Tref= CF time reference
1068	Tref = config['CFreftime']
1069	# Tunits= CF time units
1070	Tunits = config['CFunitstime']
1071	# opsur = operation surnames
1072	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
1073	Opsurs = {}
1074	for opk in opsur:
1075	opn = opk.split('_')[1]
1076	vals = opsur[opk]
1077	Opsurs[opn] = vals.split(':')
1078	# if db:
1079	# print ' ' + fname + ' operation surnames _______'
1080	# gen.printing_dictionary(Opsurs)
1081
1082	# Getting in working dir
1083	os.chdir(odir)
1084
1085	# File to keep track of all operations
1086	otrackf = open( odir + '/all_statsvars.inf', 'a')
1087
1088	# For that variables which require vertical interpolation 'p' suffix to the file
1089	# header is added.
1090	# After some operations some CF dimension-variables might not be kept
1091
1092	if gP != 'none':
1093	SgP = 'p'
1094	varnCFs = ['lon', 'lat', 'pres', 'time']
1095	else:
1096	SgP = ''
1097	varnCFs = ['lon', 'lat', 'time']
1098
1099	# CF dimension-variable dictionary
1100	CFdimvardict = {'lon': 'lon', 'lat': 'lat', 'pres': 'pres', 'time': 'time'}
1101
1102	# Input file
1103	ifilen = odir + '/' + CFvarn + '_' + headerf + SgP + '.nc'
1104
1105	# Computing variable statisitcs
1106	istats=0
1107
1108	# List of not computed statistics
1109	wrongstats = []
1110
1111	opers = Opers.split('+')
1112	Fopers = ''
1113	if db: print " computing statistics of variable:'", CFvarn, "' operation '" + \
1114	Opers + "' using file '" + ifilen + "...'"
1115	for op in opers:
1116
1117	# File name from previous operations, and name of the variable within the
1118	# file (some operations change the name of the variable)
1119	if op == opers[0]:
1120	Fopers = op
1121	prevfile = ifilen
1122	vninF = CFvarn
1123	else:
1124	Fopers = Fopers + '_' + op
1125	prevfile = fileon
1126	fileon = odir + '/' + CFvarn + '_' + headerf + SgP + '_' + Fopers + '.nc'
1127
1128	# Adding required variables for the vertical interpolation in all operations
1129	SvarnCFs = ',' + ','.join(varnCFs)
1130
1131	if gP != 'none':
1132	if gP == 'local':
1133	CFvarnp = vninF + ',P,PB,PSFC,PH,PHB,HGT,T,QVAPOR,XLONG,XLAT,Times'+ \
1134	SvarnCFs
1135	else:
1136	CFvarnp = vninF + SvarnCFs
1137	else:
1138	CFvarnp = vninF + SvarnCFs
1139
1140	if scr:
1141	sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1142
1143	hprevfile = prevfile[0:len(prevfile)-3]
1144
1145	if db: print ' ', op, "using '" + prevfile + "' "
1146	# Just produce file in case output file does not exist
1147	if not os.path.isfile(fileon):
1148	if db:
1149	print ' ' + fname + ": creation of file '" + fileon + "' ..."
1150	dofile = True
1151	else:
1152	dofile = False
1153
1154	# Variables to be kept in the final file
1155	varkeep = []
1156
1157	if op == 'acc':
1158	# temporal accumulated values
1159	print " " + fname + ": kind '" + op + "' not ready !!"
1160	wrongstats.append(CFvarn + '_' + opers)
1161	break
1162
1163	elif op[0:11] == 'dimvarvalue':
1164	# Slicing along the index at the nearest given value of a dimension-variable
1165	dimvarn = op.split('~')[1]
1166	dimvalue = op.split('~')[2]
1167	vals = dimvarn + ',' + dimvalue + ',' + dimvalue + ',0'
1168	if dofile:
1169	try:
1170	with gen.Capturing() as output:
1171	ncvar.DataSetSection_multivars(vals,prevfile,CFvarnp)
1172	except:
1173	print errmsg
1174	print 'DataSetSection_multivars('+vals+',', prevfile, ',' + \
1175	CFvarnp + ')'
1176	for s1out in output: print s1out
1177	quit(-1)
1178
1179	if db:
1180	for s1out in output: print s1out
1181
1182	sout = sub.call('mv DataSetSection_multivars.nc '+fileon, shell=True)
1183
1184	# Keeping the operations
1185	pyins=pyH + "/nc_var.py -o DataSetSection_multivars -S '" + vals + \
1186	"' -f " + prevfile + " -v " + CFvarnp
1187	otrackf.write("\n")
1188	otrackf.write("# " + CFvarnp + " " + Fopers + "\n")
1189	otrackf.write(pyins + "\n")
1190
1191	# removing the given variable-dimension
1192	varnCFs.remove(dimvarn)
1193
1194	elif op == 'direct':
1195	# no statistics
1196	if dofile:
1197	sout = sub.call('mv ' + prevfile + ' ' + fileon, shell=True)
1198	elif op == 'last':
1199	# last temporal value
1200	vals='time,-9,0,0'
1201	if dofile:
1202	try:
1203	with gen.Capturing() as output:
1204	ncvar.DataSetSection(vals,prevfile)
1205	except:
1206	print errmsg
1207	print 'DataSetSection('+vals+',', prevfile, ')'
1208	for s1out in output: print s1out
1209	quit(-1)
1210
1211	if db:
1212	for s1out in output: print s1out
1213
1214	sout = sub.call('mv ' + hprevfile + '_time_B-9-E0-I0.nc ' + fileon, \
1215	shell=True)
1216
1217	# Keeping the operations
1218	pyins=pyH + "/nc_var.py -o DataSetSection -S '" + vals + "' -f " + \
1219	prevfile
1220	otrackf.write("\n")
1221	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1222	otrackf.write(pyins + "\n")
1223
1224	elif op =='Lmean':
1225	# latitudinal mean values
1226	print " " + fname + ": kind '" + op + "' not ready !!"
1227	wrongstats.append(CFvarn + '_' + opers)
1228	break
1229	elif op =='Lsec':
1230	# latitudinal section (latitudinal value must be given, [var]@[lat])
1231	print " " + fname + ": kind '" + op + "' not ready !!"
1232	wrongstats.append(CFvarn + '_' + opers)
1233	break
1234	elif op =='lmean':
1235	# longitudinal mean values
1236	print " " + fname + ": kind '" + op + "' not ready !!"
1237	wrongstats.append(CFvarn + '_' + opers)
1238	break
1239	elif op =='lsec':
1240	# longitudinal section (longitudinal value must be given, [var]@[lon])
1241	print " " + fname + ": kind '" + op + "' not ready !!"
1242	wrongstats.append(CFvarn + '_' + opers)
1243	break
1244	elif op == 'pinterp':
1245	# pinterp: pressure interpolation (to the given $plevels)
1246	vals=plevels + ',1,1'
1247	if dofile:
1248	try:
1249	with gen.Capturing() as output:
1250	ncvar.pinterp(vals,prevfile,vninF)
1251	except:
1252	print errmsg
1253	print 'pinterp('+vals+',', prevfile, ','+vninF+')'
1254	for s1out in output: print s1out
1255	ncvar.pinterp(vals,prevfile,vninF)
1256	quit(-1)
1257
1258	if db:
1259	for s1out in output: print s1out
1260
1261	sout = sub.call('mv pinterp.nc ' + fileon, shell=True)
1262
1263	# Keeping the operations
1264	pyins=pyH + "/nc_var.py -o pinterp -S '" + vals + "' -f " + \
1265	prevfile + "-v " + vninF
1266	otrackf.write("\n")
1267	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1268	otrackf.write(pyins + "\n")
1269
1270	# adding CF lon,lat,time in WRF files
1271	if minf.name == 'WRF' or minf.name == 'WRF_LMDZ':
1272	values = vdnx + ':' + vdny + ':'+ Tref + ':' + Tunits
1273	if db:
1274	print " CFification of '" + fileon + "' with:", values, "..."
1275	try:
1276	with gen.Capturing() as output:
1277	ncvar.WRF_toCF(values, fileon)
1278	except:
1279	print errmsg
1280	print 'ncvar.WRF_toCF(' + values+ ', ' + fileon + ')'
1281	for s1out in output: print s1out
1282	# Removing file in order to make sure that it will be redone
1283	print ' ' + fname + " removing file '" + fileon + "' ..."
1284	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1285	quit(-1)
1286
1287	# vertical interpolation variables are no more needed
1288	CFvarnp = vninF
1289	gP = 'none'
1290
1291	elif op == 'smean':
1292	# spatial-weighted mean values
1293	noLlvdims = list(varnCFs)
1294	noLlvdims.remove('lon')
1295	noLlvdims.remove('lat')
1296	Saddvars = ':'.join(noLlvdims)
1297	if len(Saddvars) > 1:
1298	vals = 'reglonlat,lon,lat,lon,lat,' + Saddvars
1299	else:
1300	vals = 'reglonlat,lon,lat,lon,lat,None'
1301
1302	if dofile:
1303	try:
1304	with gen.Capturing() as output:
1305	ncvar.SpatialWeightedMean(vals,prevfile,vninF)
1306	except:
1307	print errmsg
1308	print 'SpatialWeightedMean('+vals+',', prevfile, ','+vninF+')'
1309	for s1out in output: print s1out
1310	quit(-1)
1311
1312	if db:
1313	for s1out in output: print s1out
1314
1315	# renaming variable
1316	ncvar.chvarname(vninF+'mean', 'SpatialWeightedMean_reglonlat.nc', \
1317	vninF+'spaceweightmean')
1318	sout = sub.call('mv SpatialWeightedMean_reglonlat.nc '+fileon, \
1319	shell=True)
1320
1321	# Keeping the operations
1322	pyins=pyH + "/nc_var.py -o SpatialWeightedMean -S '" + vals + \
1323	"' -f " + prevfile + " -v " + vninF
1324	otrackf.write("\n")
1325	otrackf.write("# " + vninF + " " + Fopers + "\n")
1326	otrackf.write(pyins + "\n")
1327
1328	# removing dimension variable-dimension 'lon', 'lat'
1329	varnCFs.remove('lon')
1330	varnCFs.remove('lat')
1331
1332	varkeep.append(vninF + 'mean')
1333
1334	elif op == 'tmean':
1335	# temporal mean values
1336	vals='time\|-1,time,mean,' + ':'.join(varnCFs) + ':' + vdnz
1337	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1338	if dofile:
1339	try:
1340	with gen.Capturing() as output:
1341	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1342	except:
1343	print errmsg
1344	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1345	for s1out in output: print s1out
1346	quit(-1)
1347
1348	if db:
1349	for s1out in output: print s1out
1350
1351	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
1352
1353	# Keeping the operations
1354	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1355	"' -f " + prevfile + " -v " + CFvarnp
1356	otrackf.write("\n")
1357	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1358	otrackf.write(pyins + "\n")
1359
1360	# removing dimension variable-dimension 'time'
1361	varnCFs.remove('time')
1362
1363	varkeep.append('timestats')
1364
1365	elif op == 'tstd':
1366	# temporal standard deviation values
1367	vals='time\|-1,time,std,' + ':'.join(varnCFs) + ':' + vdnz
1368	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1369	if dofile:
1370	try:
1371	with gen.Capturing() as output:
1372	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1373	except:
1374	print errmsg
1375	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1376	for s1out in output: print s1out
1377	quit(-1)
1378
1379	if db:
1380	for s1out in output: print s1out
1381
1382	sout = sub.call('mv file_oper_alongdims_std.nc '+fileon, shell=True)
1383
1384	# Keeping the operations
1385	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1386	"' -f " + prevfile + " -v " + CFvarnp
1387	otrackf.write("\n")
1388	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1389	otrackf.write(pyins + "\n")
1390
1391	# removing dimension variable-dimension 'time'
1392	varnCFs.remove('time')
1393
1394	varkeep.append('timestats')
1395
1396	elif op == 'tturb':
1397	# temporal turbulent values
1398	vals='time\|-1,time,turb,' + ':'.join(varnCFs) + ':' + vdnz
1399	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1400	if dofile:
1401	try:
1402	with gen.Capturing() as output:
1403	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1404	except:
1405	print errmsg
1406	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1407	for s1out in output: print s1out
1408	quit(-1)
1409
1410	if db:
1411	for s1out in output: print s1out
1412
1413	sout = sub.call('mv file_oper_alongdims_turb.nc '+fileon, shell=True)
1414
1415	# Keeping the operations
1416	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1417	"' -f " + prevfile + " -v " + CFvarnp
1418	otrackf.write("\n")
1419	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1420	otrackf.write(pyins + "\n")
1421
1422	# removing dimension variable-dimension 'time'
1423	varkeep.append('timestats')
1424
1425	elif op == 'tvar':
1426	# temporal variance values (Taylor's turbulence)
1427	vals='time\|-1,time,var,' + ':'.join(varnCFs) + ':' + vdnz
1428	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1429	if dofile:
1430	try:
1431	with gen.Capturing() as output:
1432	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1433	except:
1434	print errmsg
1435	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1436	for s1out in output: print s1out
1437	quit(-1)
1438
1439	if db:
1440	for s1out in output: print s1out
1441
1442	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
1443
1444	# Keeping the operations
1445	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1446	"' -f " + prevfile + " -v " + CFvarnp
1447	otrackf.write("\n")
1448	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1449	otrackf.write(pyins + "\n")
1450
1451	# removing dimension variable-dimension 'time'
1452	varnCFs.remove('time')
1453
1454	varkeep.append('timestats')
1455
1456	elif op == 'xmean':
1457	# x-axis mean values
1458	vals='lon\|-1,lon,mean,' + ':'.join(varnCFs) + ':' + vdnz
1459	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1460	if dofile:
1461	try:
1462	with gen.Capturing() as output:
1463	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1464	except:
1465	print errmsg
1466	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1467	for s1out in output: print s1out
1468	quit(-1)
1469
1470	if db:
1471	for s1out in output: print s1out
1472
1473	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
1474
1475	# Keeping the operations
1476	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1477	"' -f " + prevfile + " -v " + CFvarnp
1478	otrackf.write("\n")
1479	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1480	otrackf.write(pyins + "\n")
1481
1482	# removing dimension variable-dimension 'lon'
1483	varnCFs.remove('lon')
1484
1485	varkeep.append('lonstats')
1486
1487	elif op == 'xvar':
1488	# x-axis variance values
1489	vals='lon\|-1,lon,var,' + ':'.join(varnCFs) + ':' + vdnz
1490	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1491	if dofile:
1492	try:
1493	with gen.Capturing() as output:
1494	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1495	except:
1496	print errmsg
1497	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1498	for s1out in output: print s1out
1499	quit(-1)
1500
1501	if db:
1502	for s1out in output: print s1out
1503
1504	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
1505
1506	# Keeping the operations
1507	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1508	"' -f " + prevfile + " -v " + CFvarnp
1509	otrackf.write("\n")
1510	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1511	otrackf.write(pyins + "\n")
1512
1513	# removing dimension variable-dimension 'lon'
1514	varnCFs.remove('lon')
1515
1516	varkeep.append('lonstats')
1517
1518	elif op == 'ymean':
1519	# y-axis mean values
1520	vals='lat\|-1,lat,mean,' + ':'.join(varnCFs) + ':' + vdnz
1521	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
1522	if dofile:
1523	try:
1524	with gen.Capturing() as output:
1525	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
1526	except:
1527	print errmsg
1528	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
1529	for s1out in output: print s1out
1530	quit(-1)
1531
1532	if db:
1533	for s1out in output: print s1out
1534
1535	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
1536
1537	# Keeping the operations
1538	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
1539	"' -f " + prevfile + " -v " + CFvarnp
1540	otrackf.write("\n")
1541	otrackf.write("# " + CFvarn + " " + Fopers + "\n")
1542	otrackf.write(pyins + "\n")
1543
1544	# removing dimension variable-dimension 'lat'
1545	varnCFs.remove('lat')
1546
1547	varkeep.append('latstats')
1548
1549	elif op == 'zsum':
1550	# vertical aggregated values
1551	print " " + fname + ": kind '" + op + "' not ready !!"
1552	wrongstats.append(CFvarn + '_' + opers)
1553	break
1554	else:
1555	print errmsg
1556	print ' ' + fname + ": operation '" + op + "' not ready !!"
1557	quit(-1)
1558
1559	# End of kind of operation
1560
1561	# Variable name in file (vninF) changed due to operation
1562	# but only if previous operation does not the same 'statistic'
1563	chvn = gen.dictionary_key_list(Opsurs, op)
1564	if chvn is not None:
1565	oldvninF = vninF
1566	vninF = vninF + chvn
1567	CFvarnp = CFvarnp.replace(oldvninF,vninF)
1568
1569	if dofile:
1570	if len(varkeep) > 0:
1571	varkeepS = ',' + ','.join(varkeep)
1572	else:
1573	varkeepS = ''
1574
1575	# Adding such CF dimension variables which might not be in the
1576	# post-operation file
1577	try:
1578	with gen.Capturing() as output:
1579	varinfile = ncvar.ivars(fileon)
1580	except:
1581	print errmsg
1582	print 'ivars('+fileon+')'
1583	for s1out in output: print s1out
1584	# Removing file in order to make sure that it will be redone
1585	print ' ' + fname + " removing file '" + fileon + "' ..."
1586	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1587	quit(-1)
1588
1589	for CFvn in varnCFs:
1590	if not gen.searchInlist(varinfile, CFvn):
1591	if db:
1592	print ' ' + fname + ": recupering CF variable '" + CFvn + "'"
1593	try:
1594	with gen.Capturing() as output:
1595	ncvar.fvaradd(prevfile+','+CFvn, fileon)
1596	except:
1597	print errmsg
1598	print 'fvaradd(' + prevfile + ',' + CFvn +', '+fileon+')'
1599	ncvar.fvaradd(prevfile+','+CFvn, fileon)
1600	for s1out in output: print s1out
1601	# Removing file in order to make sure that it will be redone
1602	print ' ' + fname + " removing file '" + fileon + "' ..."
1603	#sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1604	quit(-1)
1605
1606	if db:
1607	for s1out in output: print s1out
1608
1609	totalvarkeeps = CFvarnp + ',' + ','.join(varnCFs) + varkeepS
1610	try:
1611	with gen.Capturing() as output:
1612	oclean = ncvar.cleaning_varsfile(totalvarkeeps,fileon)
1613	except:
1614	print errmsg
1615	print 'cleaning_varsfile('+totalvarkeeps+','+fileon+')'
1616	ncvar.cleaning_varsfile(totalvarkeeps,fileon)
1617	for s1out in output: print s1out
1618	# Removing file in order to make sure that it will be redone
1619	print ' ' + fname + " removing file '" + fileon + "' ..."
1620	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
1621	quit(-1)
1622
1623	if db:
1624	for s1out in output: print s1out
1625
1626	# End of operations
1627	if db:
1628	print fname + ": successful calculation of '" + vninF + "' from '" + \
1629	CFvarn + "' '" + Opers + "' !!"
1630
1631	if len(wrongstats) > 1:
1632	print warnmsg
1633	print ' ' + fname + ": statisitcs not possible to compute:", wrongstats
1634
1635	if db: print " End of '" + fname + "' "
1636
1637	return
1638
1639	def compute_vars(config, modinf, idir, odir, Files, allvarcomp, fscr, debug):
1640	""" Function to compute the variables
1641	config= Configuration of the experiment
1642	modinf= class with information about the model
1643	idir= input experiment folder
1644	odir= output experiment folder
1645	Files= dictionary with the header and the files' correspondence
1646	allvarcomp= dictionary with all the variables to compute and their information
1647	fscr= whether files should be done from the scratch or not
1648	"""
1649	fname = 'compute_vars'
1650
1651	for vopn in allvarcomp.keys():
1652	# variable & operation
1653	vn = vopn.split('_')[0]
1654	op = vopn.split('_')[1]
1655	vopnV = allvarcomp[vopn]
1656	fheader = vopnV[0]
1657	if type(vopnV[1]) == type([1, 2]) and len(vopnV[1]) == 1:
1658	vals = vopnV[1]
1659	model = vals[0]
1660	else:
1661	model = vopnV[1]
1662	diag = vopnV[2]
1663	globalP = vopnV[3]
1664
1665	if debug:
1666	print ' ' + vn + ': ' + op
1667
1668	# Computing CF variable
1669	if model is not None or diag is not None:
1670	vinf = VariableInf(vn,fheader,model,diag)
1671	if op.find('pinterp') != -1:
1672	SP = 'p'
1673	else:
1674	SP = ''
1675
1676	# Comppute variable
1677	finalfilen = odir + '/' + vn + '_' + fheader + SP + '.nc'
1678	if fscr:
1679	sout = sub.call('rm ' + finalfilen + ' >& /dev/null', shell=True)
1680
1681	if not os.path.isfile(finalfilen):
1682	compute_variable(modinf, idir, Files, odir, vinf, globalP, fscr, \
1683	config['pyHOME'], config['CFreftime'], config['CFunitstime'], debug)
1684
1685	# Compute variable statistics
1686	finalfilen = odir + '/' + vn + '_' + fheader + SP + '_' + \
1687	op.replace('+','_') + '.nc'
1688	if fscr:
1689	sout = sub.call('rm ' + finalfilen + ' >& /dev/null', shell=True)
1690
1691	if not os.path.isfile(finalfilen):
1692	compute_statistics(modinf, cnf, iwdir, Files, owdir, vinf, globalP, \
1693	op, fscr, debug)
1694
1695	else:
1696	print errmsg
1697	print ' ' + fname + ": neither 'model' or 'diag' variables for '" + vn \
1698	+ "' !!"
1699	quit(-1)
1700
1701	return
1702
1703	def get_plots_var(vc,pltk,db):
1704	""" Function to provide the plots to make for each variable from pltk ('DIRPLT_', or 'DIFFPLT_') dictionary
1705	vc= pltk dictionary, dictionary with all the parameters started with pltk
1706	pltk= kind of plots. Values started by 'DIRPLT_', or 'DIFFPLT_' in model configuration
1707	"""
1708	fname = 'get_plots_var'
1709
1710	LH = len(pltk)
1711	# list of plots
1712	doplots = {}
1713	# plots by variable
1714	plotsvar = {}
1715	# individual plots by variable (not repeated)
1716	indivplotsvar = {}
1717
1718	# Variables with a plot which requires vertical interpolation
1719	varplotp = []
1720
1721	ipl = 0
1722	for plot in vc.keys():
1723	Lplot = len(plot)
1724	pln = plot[LH:Lplot+1]
1725	varops = vc[plot].split(':')
1726	for vn in varops:
1727	VOn = vn.split('#')
1728	# Variables and operations in plot
1729	vplt = []
1730	oplt = []
1731	voplt = []
1732	for vop in VOn:
1733	vv = vop.split('\|')[0]
1734	oo = vop.split('\|')[1]
1735	vplt.append(vv)
1736	oplt.append(oo)
1737	if not gen.searchInlist(voplt,vop): voplt.append(vop)
1738	vpltS = '#'.join(vplt)
1739	opltS = '#'.join(oplt)
1740
1741	if not doplots.has_key(pln):
1742	doplots[pln] = [vn]
1743	else:
1744	plots = doplots[pln]
1745	plots.append(vn)
1746	doplots[pln] = plots
1747
1748	if not plotsvar.has_key(vn):
1749	plotsvar[vn] = [pln]
1750	else:
1751	plots = plotsvar[vn]
1752	plots.append(pln)
1753	plotsvar[vn] = plots
1754
1755	for VOv in voplt:
1756	if not indivplotsvar.has_key(VOv):
1757	indivplotsvar[VOv] = [pln]
1758	else:
1759	plots = indivplotsvar[VOv]
1760	plots.append(pln)
1761	indivplotsvar[VOv] = plots
1762
1763	if vc[plot].find('pinterp') != -1:
1764	if not gen.searchInlist(varplotp,vn): varplotp.append(pln)
1765
1766	ipl = ipl + 1
1767
1768	if db:
1769	print ' plots to make _______'
1770	gen.printing_dictionary(doplots)
1771	print ' plots by variable\|operation _______'
1772	gen.printing_dictionary(plotsvar)
1773	print ' individual plots by variable\|operation _______'
1774	gen.printing_dictionary(indivplotsvar)
1775	print ' plots with the vertical interpolation _______'
1776	print ' #', varplotp
1777
1778	if db: print " End of '" + fname + "' "
1779
1780	return doplots, plotsvar, indivplotsvar, varplotp
1781
1782	def plots_listconstruct(config, pkind, finf, odir, debug):
1783	""" Function to create the list of plots to draw
1784	config= Configuration of the experiment
1785	pkind= kind of plots ('DIRPLT', 'DIFFPLT')
1786	finf= file with the instructions of the plots
1787	odir= output experiment folder
1788	"""
1789	fname='plots_listconstruct'
1790
1791	dirplot = gen.get_specdictionary_HMT(config, H=pkind+'_')
1792
1793	# No plots because there are not plots of this kind
1794	if dirplot is None:
1795	print warnmsg
1796	print ' ' + fname + ": no plots for type '" + pkind + "' !!"
1797	return {}, 0
1798
1799	if debug:
1800	print " '" + pkind + "' plots to draw ________"
1801	gen.printing_dictionary(dirplot)
1802
1803	# Getting plots by variable
1804	plots, varplot, indivarplot, plotp = get_plots_var(dirplot, pkind+'_', debug)
1805
1806	Svarplot = gen.dictKeysVals_stringList(plots,cV=':')
1807
1808	Nplots = 0
1809	for pl in Svarplot.split(','):
1810	Nplots = Nplots + len(pl.split(':'))
1811
1812	# Outwritting the varcompute to avoid next time (if it is not filescratch!)
1813	objf = open(finf, 'w')
1814	objf.write('plots: ' + Svarplot + '\n')
1815	objf.write('itotp: ' + str(Nplots) + '\n')
1816	objf.close()
1817
1818	if debug: print " End of '" + fname + "' "
1819
1820	return plots, Nplots
1821
1822	def read_plot_file(figfile):
1823	""" Function to read the file with the information about the plots to draw
1824	figfile= file with the information
1825	"""
1826	fname = 'read_plot_file'
1827
1828	if not os.path.isfile(figfile):
1829	print errormsg
1830	print ' ' + fname + ": figures file '" + figfile + "' does not exist !!"
1831	quit(-1)
1832
1833	objf = open(figfile, 'r')
1834	for line in objf:
1835	if line[0:1] != '#' and len(line) > 1:
1836	values = line.replace('\\n','').split(' ')
1837	if values[0] == 'plots:':
1838	if values[1] != 'none':
1839	plots = gen.stringList_dictKeysVals(values[1],cV=':')
1840	else:
1841	pots = {}
1842	elif values[0] == 'itotp:':
1843	Nplots = int(values[1])
1844
1845	objf.close()
1846
1847	return plots, Nplots
1848
1849	def varnoper(vn, oper, opsurdict):
1850	""" Function to provide name of the variable after operation as result of addition of the 'surnames'
1851	vn= variable name
1852	oper= '+' separated list of operations
1853	opsurdict= dictionary with the surname of operations
1854	>>> OpSurDict = {'mean': ['tmean', 'xmean', 'ymean']}
1855	>>> varnoper('tas', 'pinterp+tmean+xmean', OpSurDict)
1856	tasmeanmean
1857	"""
1858	fname = 'varnoper'
1859
1860	newvn = vn
1861
1862	for op in oper.split('+'):
1863	surname = gen.dictionary_key_list(opsurdict,op)
1864	if surname is not None:
1865	newvn = newvn + surname
1866
1867	return newvn
1868
1869	def draw_plot(kplot, CFvplot, fplot, vplot, dplot, pplot, finame, tfig, kfig, mapval,\
1870	tvals, expgraphc, od, pyH, fscr, db):
1871	""" Function to draw a plot
1872	kplot= kind of plot
1873	CFvplot= list of the CFnames of the variables
1874	fplot= list with the files in the plot
1875	vplot= list with the name of the variables in each file
1876	dplot= list of the dimensions in each file
1877	pplot= list of pictoric characteristics for each variable
1878	finame= name of the figure
1879	tfig= title of the figure
1880	kfig= kind of the figure
1881	mapval= value of the map
1882	tvals= list with time values
1883	tunits: units of time ('!' for spaces)
1884	timekind: kind of time ticks in the plot
1885	timefmt: format of time ticks in the plot
1886	timelabel: label of time-axis in the plot
1887	expgraphc= dictionary with expriment graphical characteristics
1888	od= output directory
1889	pyH= python HOME
1890	fscr= whether figure should be done from scratch
1891	"""
1892	fname = 'draw_plot'
1893
1894	if fscr:
1895	sout = sub.call('rm ' + finame + ' >& /dev/null', shell=True)
1896
1897	os.chdir(od)
1898
1899	# CF variable-dimensions
1900	CFvardims = {'lon':'lon', 'lat': 'lat', 'pres': 'pres', 'time': 'time'}
1901
1902	# Tracking file with all the figures
1903	trkobjf = open('all_figures.inf', 'a')
1904
1905	# Time-characteristics
1906	tunits = tvals[0]
1907	timekind = tvals[1]
1908	timefmt = tvals[2]
1909	timelabel = tvals[3]
1910
1911	# Graphical labels and configuration
1912
1913
1914	if not os.path.isfile(finame):
1915	if db:
1916	print " drawing '" + finame + "' ..."
1917	print ' plot kind:', kplot
1918	print ' CFvars:', CFvplot
1919	print ' files:', fplot
1920	print ' in file vars:', vplot
1921	print ' dims:', dplot
1922	print ' pictoric values:', pplot
1923	print ' figure name:', finame
1924	print ' title:', tfig.replace('!',' ')
1925	print ' kind:', kfig
1926	print ' map:', mapval
1927	print ' time units:', tunits
1928
1929	if kplot == 'diffmap2Dsfc':
1930	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1931	quit(-1)
1932	elif kplot == 'diffmap2Dz':
1933	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1934	quit(-1)
1935	elif kplot == 'map2Dsfc':
1936	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1937	quit(-1)
1938	elif kplot == 'map3D':
1939	print " " + fname + ": kind of plot '" + kplot + "' not ready !!"
1940	quit(-1)
1941	elif kplot == '2lines':
1942	figtit = tfig.replace('!','\|')
1943
1944	lineAstdn = CFvplot[0]
1945	lineBstdn = CFvplot[1]
1946	figfs = ','.join(fplot)
1947
1948	lineA = pplot[0]
1949	Arange = str(lineA[0]) + ',' + str(lineA[1])
1950	# Values are changed from specific values in `model_graphics.dat'
1951	if lineA[3].find('%') == -1:
1952	Aline = lineA[2]
1953	Akind = lineA[3]
1954	Amark = lineA[4]
1955	Asize = lineA[5]
1956	else:
1957	Aline = 'red'
1958	Akind = '-'
1959	Amark = ','
1960	Asize = 2.
1961
1962	lineB = pplot[1]
1963	Brange = str(lineB[0]) + ',' + str(lineB[1])
1964	# Values are changed from specific values in `model_graphics.dat'
1965	if lineB[3].find('%') == -1:
1966	Bline = lineB[2]
1967	Bkind = lineB[3]
1968	Bmark = lineB[4]
1969	Bsize = lineB[5]
1970	else:
1971	Bline = 'blue'
1972	Bkind = '-'
1973	Bmark = ','
1974	Bsize = '2.'
1975
1976	# It is assumed that if the space variable is 'lon': is desired a
1977	# (lon, vals) plot if it is 'lat': then (vals, lat) plot
1978	if gen.searchInlist(dplot[0],'lon'):
1979	spacedim ='lon'
1980	axisvals = 'y'
1981	figmid = 'longitudinal'
1982	elif gen.searchInlist(dplot[0],'lat'):
1983	spacedim = 'lat'
1984	axisvals = 'x'
1985	figmid = 'meridional'
1986	elif gen.searchInlist(dplot[0],'pres'):
1987	spacedim = 'pres'
1988	axisvals = 'x'
1989	figmid = 'vertical'
1990	else:
1991	print errmsg
1992	print ' ' + fname + ": in '2lines' only ready for: 'lon', 'lat'," + \
1993	" 'pres' as common dimension !!"
1994	print ' dimensions in plot:', dplot[0]
1995	quit(-1)
1996
1997	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
1998	'!'.join(tfig.split('!')[2:])
1999
2000	graphvals = spacedim + ':' + Arange + ':' + Brange + ':Extrs:' + \
2001	axisvals + ':' + ','.join(CFvplot) + ':' + Aline + ',' + Bline + \
2002	':2.,2.:' + Akind + ',' + Bkind + ':2.,2.:,:' + figtit + ':' + \
2003	spacedim + ':0\|auto:' + finame.replace('.'+kfig, '') + ':' + kfig + \
2004	':True'
2005
2006	fvarS = ','.join(vplot)
2007
2008	drwins = 'draw_2lines'
2009	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2010	" -S '" + graphvals + "' -v " + fvarS
2011
2012	try:
2013	with gen.Capturing() as output:
2014	sout = sub.call(plotins, shell=True)
2015	except:
2016	print errmsg
2017	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2018	print sout
2019	for s1out in output: print s1out
2020	quit(-1)
2021
2022	# keeping all figures
2023	trkobjf.write('\n')
2024	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2025	trkobjf.write(plotins + '\n')
2026
2027	elif kplot == '2linesTime':
2028	figtit = tfig.replace('!','\|')
2029
2030	lineAstdn = CFvplot[0]
2031	lineBstdn = CFvplot[1]
2032	figfs = ','.join(fplot)
2033
2034	lineA = pplot[0]
2035	Arange = str(lineA[0]) + ',' + str(lineA[1])
2036	# Values are changed from specific values in `model_graphics.dat'
2037	if lineA[3].find('%') == -1:
2038	Aline = lineA[2]
2039	Akind = lineA[3]
2040	Amark = lineA[4]
2041	Asize = lineA[5]
2042	else:
2043	Aline = 'red'
2044	Akind = '-'
2045	Amark = ','
2046	Asize = '2.'
2047
2048	lineB = pplot[1]
2049	Brange = str(lineB[0]) + ',' + str(lineB[1])
2050	# Values are changed from specific values in `model_graphics.dat'
2051	if lineB[3].find('%') == -1:
2052	Bline = lineB[2]
2053	Bkind = lineB[3]
2054	Bmark = lineB[4]
2055	Bsize = lineB[5]
2056	else:
2057	Bline = 'blue'
2058	Bkind = '-'
2059	Bmark = ','
2060	Bsize = '2.'
2061
2062	timeaxis = 'x'
2063	figmid = 'evolution'
2064
2065	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
2066	'!'.join(tfig.split('!')[2:])
2067
2068	graphvals = 'time:' + Arange + ':' + Brange + ':' + timekind + ';' + \
2069	timefmt + ':' + timeaxis + ':' + ','.join(CFvplot) + ':' + Aline + ',' \
2070	+ Bline + ':2.,2.:' + Akind + ',' + Bkind + ':2.,2.:,:' + figtit + ':' \
2071	+ timelabel + ':0\|auto:' + finame.replace('.'+kfig, '') + ':' + \
2072	kfig + ':True'
2073
2074	fvarS = ','.join(vplot)
2075
2076	drwins = 'draw_2lines_time'
2077	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2078	" -S '" + graphvals + "' -v " + fvarS
2079
2080	try:
2081	with gen.Capturing() as output:
2082	sout = sub.call(plotins, shell=True)
2083	except:
2084	print errmsg
2085	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2086	print sout
2087	for s1out in output: print s1out
2088	quit(-1)
2089
2090	# keeping all figures
2091	trkobjf.write('\n')
2092	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2093	trkobjf.write(plotins + '\n')
2094
2095	elif kplot == 'Nlines':
2096	figtit = tfig.replace('!','\|')
2097
2098	linestdn = CFvplot[0]
2099	figfs = ','.join(fplot)
2100
2101	Nlines = len(fplot)
2102
2103	lablines = []
2104	for il in range(Nlines):
2105	linev = pplot[il]
2106	# Values are changed from specific values in `model_graphics.dat'
2107	if linev[3].find('%') == -1:
2108	line = linev[2]
2109	kind = linev[3]
2110	mark = linev[4]
2111	size = linev[5]
2112	if il == 0:
2113	vrange = str(linev[0]) + ',' + str(linev[1])
2114	kinds = kind
2115	lines = line
2116	marks = mark
2117	sizes = str(size)
2118	else:
2119	kinds = kinds + ',' + kind
2120	lines = lines + ',' + line
2121	marks = marks + '@' + mark
2122	sizes = sizes + ',' + str(size)
2123	else:
2124	lines = 'None'
2125	kinds = '-'
2126	marks = ','
2127	sizes = '2.'
2128
2129	secsf = fplot[il].split('/')
2130	Nsecsf = len(secsf)
2131	expvs = expgraphc[secsf[Nsecsf-3] + '/' + secsf[Nsecsf-2]]
2132	lablines.append(expvs.label)
2133
2134	# It is assumed that if the space variable is 'lon': is desired a
2135	# (lon, vals) plot if it is 'lat': then (vals, lat) plot
2136	if gen.searchInlist(dplot[0],'lon'):
2137	spacedim ='lon'
2138	sdlab ='lon ($degrees\ East$)'
2139	axisvals = 'y'
2140	figmid = 'longitudinal'
2141	elif gen.searchInlist(dplot[0],'lat'):
2142	spacedim = 'lat'
2143	sdlab = 'lat ($degrees\ North$)'
2144	axisvals = 'x'
2145	figmid = 'meridional'
2146	elif gen.searchInlist(dplot[0],'pres'):
2147	spacedim = 'pres'
2148	sdlab = 'pres ($Pa$)'
2149	axisvals = 'x'
2150	figmid = 'vertical'
2151	else:
2152	print errmsg
2153	print ' ' + fname + ": in 'Nlines' only ready for: 'lon', 'lat'," + \
2154	" 'pres' as common dimension !!"
2155	print ' dimensions in plot:', dplot[0]
2156	quit(-1)
2157
2158	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
2159	'!'.join(tfig.split('!')[2:])
2160	figtit = figtit.replace('!',' ')
2161	leglabels = ','.join(lablines)
2162
2163	graphvals = spacedim + ':' + axisvals + ':' + sdlab + ':auto:' + \
2164	leglabels + ':' + CFvplot[0] +':'+ figtit + ':0\|auto:' + lines + ':' + \
2165	kinds + ':' + marks +':'+ sizes +':'+ sizes +':all:'+ \
2166	finame.replace('.'+kfig, '') + ':' + kfig + ':True'
2167
2168	fvarS = vplot[0]
2169
2170	drwins = 'draw_lines'
2171	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2172	" -S '" + graphvals + "' -v " + fvarS
2173
2174	try:
2175	with gen.Capturing() as output:
2176	sout = sub.call(plotins, shell=True)
2177	except:
2178	print errmsg
2179	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2180	print sout
2181	for s1out in output: print s1out
2182	quit(-1)
2183
2184	# keeping all figures
2185	trkobjf.write('\n')
2186	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2187	trkobjf.write(plotins + '\n')
2188
2189	elif kplot == 'Nlines_time':
2190	figtit = tfig.replace('!','\|')
2191
2192	linestdn = CFvplot[0]
2193	figfs = ','.join(fplot)
2194
2195	Nlines = len(fplot)
2196
2197	lablines = []
2198	for il in range(Nlines):
2199	linev = pplot[il]
2200	# Values are changed from specific values in `model_graphics.dat'
2201	if linev[3].find('%') == -1:
2202	line = linev[2]
2203	kind = linev[3]
2204	mark = linev[4]
2205	size = linev[5]
2206	if il == 0:
2207	vrange = str(linev[0]) + ',' + str(linev[1])
2208	kinds = kind
2209	lines = line
2210	marks = mark
2211	sizes = str(size)
2212	else:
2213	kinds = kinds + ',' + kind
2214	lines = lines + ',' + line
2215	marks = marks + '@' + mark
2216	sizes = sizes + ',' + str(size)
2217	else:
2218	lines = 'None'
2219	kinds = '-'
2220	marks = ','
2221	sizes = '2.'
2222
2223	secsf = fplot[il].split('/')
2224	Nsecsf = len(secsf)
2225	expvs = expgraphc[secsf[Nsecsf-3] + '/' + secsf[Nsecsf-2]]
2226	lablines.append(expvs.label)
2227
2228	valsaxis = 'y'
2229	figmid = 'evolution'
2230
2231	figtit = '!'.join(tfig.split('!')[0:2]) + '!' + figmid + '!' + \
2232	'!'.join(tfig.split('!')[2:])
2233	figtit = figtit.replace('!',' ')
2234	leglabels = ','.join(lablines)
2235
2236	graphvals = 'time;' + valsaxis + ';time;' + leglabels + ';' + \
2237	CFvplot[0] + ';' + figtit + ';None;time\|' + '\|'.join(tvals) + \
2238	';0\|auto;' + kfig + ';' + kinds + ';' + lines + ';' + marks + ';' + \
2239	sizes + ';' + sizes + ';all;-1;True'
2240
2241	fvarS = vplot[0]
2242
2243	drwins = 'draw_lines_time'
2244	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2245	" -S '" + graphvals + "' -v " + fvarS
2246
2247	try:
2248	with gen.Capturing() as output:
2249	sout = sub.call(plotins, shell=True)
2250	except:
2251	print errmsg
2252	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2253	print sout
2254	for s1out in output: print s1out
2255	quit(-1)
2256
2257	sout = sub.call('mv lines_time.' + kfig + ' ' + finame, shell=True)
2258
2259	# keeping all figures
2260	trkobjf.write('\n')
2261	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2262	trkobjf.write(plotins + '\n')
2263
2264	elif kplot == 'shadcont2Dsfc':
2265	figtit = tfig.replace('!','\|')
2266	shdstdn = CFvplot[0]
2267	cntstdn = CFvplot[1]
2268	figfs = ','.join(fplot)
2269
2270	shad = pplot[0]
2271	srange = str(shad[0]) + ',' + str(shad[1])
2272	cbar = shad[2]
2273	cnt = pplot[1]
2274	crange = str(cnt[0]) + ',' + str(cnt[1])
2275	cfmt = cnt[3]
2276	ckind = cnt[4]
2277	cline = cnt[5]
2278
2279	graphvals = ','.join(CFvplot)
2280	graphvals = graphvals + ':lon\|-1,lat\|-1:lon\|-1,lat\|-1:lon:lat:auto:' + \
2281	cbar + ',auto,auto:' + ckind + ',' + cline + ':' + cfmt + ':' + \
2282	srange + ':' + crange + ',9:' + figtit + ':' + kfig + ':None:' + \
2283	mapval + ':True'
2284
2285	fvarS = ','.join(vplot)
2286
2287	drwins = 'draw_2D_shad_cont'
2288	plotins = "python " + pyH + "/drawing.py -f " + figfs + " -o " + drwins +\
2289	" -S '" + graphvals + "' -v " + fvarS
2290	try:
2291	with gen.Capturing() as output:
2292	sout = sub.call(plotins, shell=True)
2293	except:
2294	print errmsg
2295	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2296	print sout
2297	for s1out in output: print s1out
2298	quit(-1)
2299
2300	sout = sub.call('mv 2Dfields_shadow-contour.'+kfig+' '+finame, shell=True)
2301
2302	# keeping all figures
2303	trkobjf.write('\n')
2304	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2305	trkobjf.write(plotins + '\n')
2306
2307	elif kplot == 'shadconthovmsfc':
2308	figtit = tfig.replace('!','\|')
2309	shdstdn = CFvplot[0]
2310	cntstdn = CFvplot[1]
2311	figfs = ','.join(fplot)
2312
2313	shad = pplot[0]
2314	srange = str(shad[0]) + ',' + str(shad[1])
2315	cbar = shad[2]
2316	cnt = pplot[1]
2317	crange = str(cnt[0]) + ',' + str(cnt[1])
2318	cfmt = cnt[3]
2319	ckind = cnt[4]
2320	cline = cnt[5]
2321	# It is assumed that if the space variable is 'lon': is desired a
2322	# (lon, time) plot if it is 'lat': then (time, lat) plot
2323	if gen.searchInlist(dplot,'lon'):
2324	spacedim ='lon'
2325	figmid = 'longitudinal\|evolution\|of'
2326	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2327	'\|'.join(tfig.split('!')[2:])
2328	reverse= 'None'
2329	dims= spacedim+'\|-1,time\|-1;'+spacedim+'\|-1,time\|-1;'+spacedim+';time'
2330	else:
2331	spacedim='lat'
2332	figmid = 'meridional\|evolution\|of'
2333	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2334	'\|'.join(tfig.split('!')[2:])
2335	reverse='None'
2336	dims= spacedim+'\|-1,time\|-1;'+spacedim+'\|-1,time\|-1;time;'+spacedim
2337
2338	graphvals = ','.join(CFvplot)
2339	graphvals = graphvals + ';' + dims +';auto;' + cbar + ',auto,auto' + ';' \
2340	+ ckind + ',' + cline + ';' + cfmt + ';' + srange + ';' + crange + \
2341	',9;' + figtit + ';' + kfig + ';' + reverse + ';' + 'time\|' + tunits + \
2342	'\|'+ timekind + '\|' + timefmt + '\|' + timelabel + ';True'
2343
2344	fvarS = ','.join(vplot)
2345
2346	drwins = 'draw_2D_shad_cont_time'
2347	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs +' -o ' + drwins + \
2348	" -S '" + graphvals + "' -v " + fvarS
2349
2350	try:
2351	with gen.Capturing() as output:
2352	sout = sub.call(plotins, shell=True)
2353	except:
2354	print errmsg
2355	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2356	print sout
2357	for s1out in output: print s1out
2358	quit(-1)
2359
2360	sout = sub.call('mv 2Dfields_shadow-contour.'+kfig+' '+finame, shell=True)
2361
2362	# keeping all figures
2363	trkobjf.write('\n')
2364	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2365	trkobjf.write(plotins + '\n')
2366
2367	elif kplot == 'shadcont2Dzsec':
2368	figtit = tfig.replace('!','\|')
2369	shdstdn = CFvplot[0]
2370	cntstdn = CFvplot[1]
2371	figfs = ','.join(fplot)
2372
2373	shad = pplot[0]
2374	srange = str(shad[0]) + ',' + str(shad[1])
2375	cbar = shad[2]
2376	cnt = pplot[1]
2377	crange = str(cnt[0]) + ',' + str(cnt[1])
2378	cfmt = cnt[3]
2379	ckind = cnt[4]
2380	cline = cnt[5]
2381
2382	# It is assumed that if the space variable is 'lon': is desired a
2383	# (lon, pres) plot it it is 'lat': then (pres, lat) plot
2384	if gen.searchInlist(dplot, 'lon'):
2385	spacedim='lon'
2386	figmid = 'long.\|vert.\|cross\|sec.\|of'
2387	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2388	'\|'.join(tfig.split('!')[2:])
2389	dims= spacedim+'\|-1,pres\|-1:'+spacedim+'\|-1,pres\|-1:'+spacedim+':pres'
2390	else:
2391	spacedim='lat'
2392	figmid = 'mer.\|vert.\|cross\|sec.\|of'
2393	figtit = '\|'.join(tfig.split('!')[0:2]) + '\|' + figmid + '\|' + \
2394	'\|'.join(tfig.split('!')[2:])
2395	# NOT WORKING? dims= spacedim+'\|-1,pres\|-1:'+spacedim+'\|-1,pres\|-1:'+'pres:'+spacedim
2396	dims= spacedim+'\|-1,pres\|-1:'+spacedim+'\|-1,pres\|-1:'+spacedim+':pres'
2397
2398	reverse = 'None'
2399
2400	graphvals = ','.join(CFvplot)
2401	graphvals = graphvals + ':' + dims + ':auto:' + cbar + ',auto,auto:' + \
2402	ckind + ',' + cline + ':' + cfmt + ':' + srange + ':' + crange + ',9:' \
2403	+ figtit + ':' + kfig + ':' + reverse + ':None:True'
2404
2405	fvarS = ','.join(vplot)
2406
2407	drwins = 'draw_2D_shad_cont'
2408	plotins = 'python ' + pyH + '/drawing.py -f ' + figfs + ' -o ' + drwins + \
2409	" -S '" + graphvals + "' -v " + fvarS
2410	try:
2411	with gen.Capturing() as output:
2412	sout0 = sub.call(plotins, shell=True)
2413	except:
2414	print errmsg
2415	print drwins + '(' + graphvals + ',' + figfs + ',' + fvarS + ')'
2416	for s1out in output: print s1out
2417	quit(-1)
2418
2419	sout = sub.call('mv 2Dfields_shadow-contour.'+kfig+' '+finame, shell=True)
2420
2421	# keeping all figures
2422	trkobjf.write('\n')
2423	trkobjf.write("# '" + kplot + "'; " + tfig.replace('!',' ') + '\n')
2424	trkobjf.write(plotins + '\n')
2425
2426	else:
2427	print errmsg
2428	print " " + fname + ": plot kind '" + kplot + "' not ready !!"
2429	quit(-1)
2430
2431	trkobjf.close()
2432
2433	if db: print " End of '" + fname + "' "
2434
2435	return
2436
2437	def opexplained(Opn,Exp):
2438	""" Function to provide the final explanation for the title in the graph. Some operations have
2439	values separated by `~' which need to be processed
2440	Opn= full operation string
2441	Exp= explanation from dictionary
2442	>>> opexplained('dimvarvalue~pres~85000','@')
2443	pres@85000
2444	>>> opexplained('xmean','xmean')
2445	xmean
2446	"""
2447	fname = 'opexplained'
2448
2449	if Opn.find('~') == -1:
2450	explanation = Exp
2451	else:
2452	Opnv = Opn.split('~')
2453	if Opnv[0] == 'dimvarvalue':
2454	explanation = Opnv[1] + Exp + Opnv[2]
2455	else:
2456	print errormsg
2457	print ' ' + fname + ": unknow '~' separated operation '" + Opnv[0] + \
2458	"' !!"
2459	quit(-1)
2460	return explanation
2461
2462	def optitle(op,opexp):
2463	""" Function to creat the operation section in the title of a figure ('!' for spaces)
2464	op= '+' separated list of operations
2465	opexp= dictionary with the `explanation'(text in title) to appear for each operation
2466	>>> optitle('pinterp+xmean+tmean',{'pinterp':'pinterp', 'xmean':'xmean', 'tmean':'tmean'})
2467	$_{[pinterp\!xmean\!tmean]}$
2468	>>> optitle('pinterp+dimvarvalue~pres~85000+tmean',{'dimvarvalue':'@', 'pinterp':'pinterp', 'xmean':'xmean', 'tmean':'tmean'})
2469	$_{[pinterp\!pres@85000\!tmean]}$
2470	"""
2471	fname = 'optitle'
2472
2473	opvals = op.split('+')
2474	for op1 in opvals:
2475	op1n = op1.split('~')[0]
2476	if not opexp.has_key(op1n):
2477	print errmsg
2478	print ' ' + fname + ": no explanation for operation '" + op1n + "' !!"
2479	print ' provided:', opexp.keys()
2480	quit(-1)
2481	# Final explanation
2482	op1en = opexplained(op1,opexp[op1n]).replace(' ','!')
2483
2484	if op1n == opvals[0].split('~')[0]:
2485	titop = '$_{[' + op1en
2486	if len(opvals) == 1: titop = titop + ']}$'
2487	elif op1n == opvals[len(opvals)-1].split('~')[0]:
2488	titop = titop + '\!' + op1en + ']}$'
2489	else:
2490	titop = titop + '\!' + op1en
2491
2492	return titop
2493
2494	def create_figure_title(mod,exp,varops,explop):
2495	""" Function to create the title of a figure ('!' for spaces)
2496	mod: model name
2497	exp: experiment name
2498	varops: list with [var]\|[op] values from wich create the title
2499	explop: dictionary with the `explanation'(text in title) to appear for each operation
2500	>>> expops = {'pinterp':'pinterp', 'xmean':'xmean', 'tmean':'tmean', 'last':'last'}
2501	>>> create_figure_title('WRF','current',['ua\|pinterp+xmean+tmean', 'va\|pinterp+xmean+tmean'], expops)
2502	WRF!current!ua!&!va$_{[pinterp\!xmean\!tmean]}$
2503	>>> create_figure_title('WRF','current',['ua\|pinterp+xmean+tmean', 'ua\|pinterp+xmean+last'], expops)
2504	WRF!current!ua$_{[pinterp\!xmean\!tmean]}$!&!!$_{[pinterp\!xmean\!last]}$
2505	>>> create_figure_title('WRF','current',['ua\|pinterp+xmean+tmean', 'va\|pinterp+xmean+last'], expops)
2506	WRF!current!ua$_{[pinterp\!xmean\!tmean]}$!&!va$_{[pinterp\!xmean\!last]}$
2507	>>> create_figure_title('WRF','current',['uas\|dimvarvalue~lon~23.23', 'vas\|dimvarvalue~lon~23.23'], expops)
2508	WRF!current!uas!&!vas$_{[lon@23.23]}$
2509	>>> create_figure_title('WRF','current',['hus\|pinterp+tmean+dimvarvalue~pres~85000.', 'ta\|pinterp+tmean+dimvarvalue~pres~85000.'], expops)
2510	WRF!current!hus!&!ta$_{[pinterp\!tmean\!pres@85000.]}$
2511	"""
2512	fname = 'create_figure_title'
2513
2514	titfig = mod + '!' + exp + '!'
2515
2516	varns = []
2517	opns = []
2518	for varop in varops:
2519	vn = varop.split('\|')[0]
2520	op = varop.split('\|')[1]
2521	varns.append(vn)
2522	opns.append(op)
2523	if varop == varops[0]:
2524	allvarequal = True
2525	allopequal = True
2526	varinfig = vn
2527	opinfig = op
2528	else:
2529	if vn != varinfig: allvarequal = False
2530	if op != opinfig: allopequal = False
2531
2532	Nvars = len(varns)
2533	Nops = len(opns)
2534
2535	if allvarequal:
2536	titfig = titfig + '!' + varns[0]
2537	if allopequal:
2538	opS = optitle(op,explop)
2539	titfig = titfig + opS
2540	else:
2541	for opn in opns:
2542	opS = optitle(opn,explop)
2543	if opn == opns[0]:
2544	titfig = titfig + opS
2545	elif opn == opns[Nops-1]:
2546	titfig = titfig + '!&!' + opS
2547	else:
2548	titfig = titfig + ',!' + opS
2549	else:
2550	if allopequal:
2551	opS = optitle(op,explop)
2552	for vn in varns:
2553	if vn == varns[0]:
2554	titfig = titfig + '!' + vn
2555	elif vn == varns[Nvars-1]:
2556	titfig = titfig + '!&!' + vn
2557	else:
2558	titfig = titfig + ',!' + vn
2559	titfig = titfig + opS
2560	else:
2561	for ivop in range(Nvars):
2562	vn = varns[ivop]
2563	op = opns[ivop]
2564	vnop = vn + optitle(op,explop)
2565
2566	if vn == varns[0]:
2567	titfig = titfig + '!' + vnop
2568	elif vn == varns[Nvars-1]:
2569	titfig = titfig + '!&!' + vnop
2570	else:
2571	titfig = titfig + ',!' + vnop
2572
2573	return titfig.replace('!!','!')
2574	#expops = {'dimvarvalue':'@', 'last':'last', 'pinterp':'pinterp', 'xmean':'xmean', 'tmean':'tmean'}
2575
2576	def draw_plots(config, plots, mod, exp, odir, allvarcomp, figscr, debug):
2577	""" Function to draw all plots
2578	config= Configuration of the experiment
2579	plots= dictionary with the plots
2580	mod= model of the plots
2581	exp= experiment of the plots
2582	odir= output experiment folder
2583	allvarcomp= dictionary with all the variables to compute and their information
2584	figscr= whether figures should be done from the scratch or not
2585
2586	* Plot as
2587	{[kplot]} = [varn1]\|[op1]#[varn2]\|[op2]#[...[varnN]\|[opN]], ...
2588	[kplot] ___
2589	diffmap2Dsfc: 2D map of surface differences values of 1 variable
2590	diffmap2Dz: 2D map of 3D differences values of 1 variable
2591	map2Dsfc: 2D map of surface values of 1 variable
2592	map3D: 2D map of 3D values of 1 variable
2593	shadconthovmsfc: Hovmoeller diagrams of 2 variable at the surface in shadow and the other in contourn
2594	shadcont2Dsfc: 2D map of shadow (1st variable) and countour (2nd variable) [stvar1]#[stvar2]
2595	shadcont2Dzsec: 2D map of vertical section of 2 variables one in shadow and the other in contourn
2596	[varn]
2597	variable
2598	[op]
2599	'+' separated list of operations
2600	in figures with more than 1 variable, use '#' to separate the [varn]\|[op]
2601	"""
2602	fname = 'draw_plots'
2603
2604	os.chdir(odir)
2605
2606	# Dictionary with the operations with surnames for the operated variable
2607	opersurnames = {}
2608	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
2609	for opsr in opsur.keys():
2610	opn = opsr.split('_')[1]
2611	vls = opsur[opsr].split(':')
2612	opersurnames[opn] = vls
2613
2614	# time values
2615	# Units time for the plots
2616	rd = config['CFreftime']
2617	tunits = config['CFunitstime'] + '!since!' + rd[0:4] + '-' + rd[4:6] + '-' + \
2618	rd[6:8] + '!' + rd[8:10] + ':' + rd[10:12] + ':' + rd[12:14]
2619	# time ticks kind
2620	tkind = config['timekind']
2621	# time ticks format
2622	tfmt = config['timefmt']
2623	# time axis label
2624	tlab = config['timelabel']
2625	timevals = [tunits, tkind, tfmt, tlab]
2626
2627	# Dictionary of plot specificities
2628	specplotkeyn = 'specificvarplot'
2629	plotspecifics = {}
2630	if config.has_key(specplotkeyn):
2631	# [minval]: minimum value
2632	# [maxval]: minimum value
2633	# [colorbar]: name of the colorbar (from matplotlib) to use
2634	# [cntformat]: format of the contour labels
2635	# [colorcnt]: color for the countor lines
2636	plotspecs = config[specplotkeyn].split(':')
2637	for pltspc in plotspecs:
2638	pltvls = pltspc.split('\|')
2639	vn = pltvls[0]
2640	op = pltvls[1]
2641	fn = pltvls[2]
2642	plotspecifics[fn + '_' + vn + '_' + op] = pltvls[3:]
2643	if debug:
2644	print 'Specific values for plots _______'
2645	gen.printing_dictionary(plotspecifics)
2646
2647	# Kind of figures
2648	kindfigure = config['kindfig']
2649
2650	# Graphical labels and configuration
2651	modgc, expgc = graphical_characteristics(config, debug)
2652	modv = modgc[mod]
2653	modlab = modv.label
2654	expv = expgc[mod+'/'+exp]
2655	explab = expv.label
2656
2657	# Map value
2658	mapvalue = config['mapval']
2659
2660	# pythone scripts HOME
2661	pyHOME = config['pyHOME']
2662
2663	# Title-text of operations
2664	opexplained = {}
2665	optits = config['titleoperations'].split(':')
2666	for optit in optits:
2667	opn = optit.split('\|')[0]
2668	opt = optit.split('\|')[1]
2669	opexplained[opn] = opt
2670	if debug:
2671	print 'Titles for operations _______'
2672	gen.printing_dictionary(opexplained)
2673
2674	for kplot in plots.keys():
2675	varsplt = plots[kplot]
2676	for varplt in varsplt:
2677	if debug:
2678	print " printing '" + kplot + "' var ':" + varplt + "'..."
2679	varops = varplt.split('#')
2680
2681	# CF variables in plot
2682	CFvarsplot = []
2683	# Files in plot
2684	filesplot = []
2685	# Variables in plot within the files
2686	varsplot = []
2687	# Dims in figure
2688	dimsplot = []
2689	# pictoric values in figure
2690	pictplot = []
2691	# Name of the figure
2692	figname = ''
2693	# Title of the figure
2694	titfigure = ''
2695
2696	ivp = 0
2697	for varop in varops:
2698	vn = varop.split('\|')[0]
2699	op = varop.split('\|')[1]
2700
2701	# CF variables in plot
2702	CFvarsplot.append(vn)
2703
2704	vnopS = vn + '_' + op
2705	if not allvarcomp.has_key(vnopS):
2706	print errmsg
2707	print ' ' + fname + ": no file for variable-operation '" + \
2708	vnopS + "' !!"
2709	vopvals = allvarcomp[vnopS]
2710	headf = vopvals[0]
2711	globalP = vopvals[3]
2712	gP = pinterpS(globalP)
2713
2714	filen = vn + '_' + headf + gP + '_' + op.replace('+','_') + '.nc'
2715	filesplot.append(filen)
2716	# Do we have processed the given variable?
2717	if not os.path.isfile(filen):
2718	print warnmsg
2719	print " " + fname + ": there is no file for variable '" + varop \
2720	+ "' skiping it !!"
2721	break
2722
2723	# Name of the variable inside the file
2724	vnsur = varnoper(vn, op, opersurnames)
2725	varsplot.append(vnsur)
2726
2727	# Dimensions in file
2728	try:
2729	with gen.Capturing() as output:
2730	dims = ncvar.idims(filen)
2731	except:
2732	print errmsg
2733	print 'ncvar.idims('+filen+')'
2734	for s1out in output: print s1out
2735	quit(-1)
2736
2737	dimsplot.append(dims)
2738
2739	# pictoric values for the figure
2740	Sfivaop = kplot + '_' + vn + '_' + op
2741	if plotspecifics.has_key(Sfivaop):
2742	pictvals = plotspecifics[Sfivaop]
2743	else:
2744	Vvals = gen.variables_values(vn)
2745	pictvals = [Vvals[2], Vvals[3], Vvals[6], '%g', 'fixc', 'black']
2746
2747	pictplot.append(pictvals)
2748
2749	# Header of the name of the figure
2750	if ivp == 0:
2751	figname = kplot +'_'+ vn + '_' + headf + '_' + op.replace('+','-')
2752	else:
2753	figname = figname + '-'+vn+'_'+headf+'_'+op.replace('+','-')
2754
2755	ivp = ivp + 1
2756	# End of variable-operation
2757	figname = figname + '.' + kindfigure
2758
2759	# Removig double '..' (from `dimvarvalue')
2760	figname = figname.replace('..', '.')
2761
2762	# Title of figure
2763	titfigure = create_figure_title(modlab, explab, varops, opexplained)
2764
2765	draw_plot(kplot, CFvarsplot, filesplot, varsplot, dimsplot, pictplot, \
2766	figname, titfigure, kindfigure, mapvalue, timevals, expgc, odir, \
2767	pyHOME, figscr, debug)
2768
2769	# End of variables-operations
2770
2771	# End of kind of plots
2772
2773	if debug: print " End of '" + fname + "' "
2774
2775	return
2776
2777	def get_differences_var(vc,kdiff,db):
2778	""" Function to provide the operations to make for each variable from kdiff ('DIFFOP_' or 'DIFFVAR_') dictionary
2779	vc= kdiff dictionary, dictionary with all the parameters started with kdiff
2780	kdiff= kind of differences 'op', 'var'
2781	"""
2782	fname = 'get_differences_var'
2783
2784	if kdiff == 'op':
2785	diffn = 'DIFFOP'
2786	elif kdiff == 'var':
2787	diffn = 'DIFFVAR'
2788	else:
2789	print errmsg
2790	print ' ' + fname + ": differences '" + kdiff + "' not ready !!"
2791	quit(-1)
2792	LD = len(diffn + '_')
2793
2794	iop = 1
2795	# list of operations by difference
2796	doopers = {}
2797	# operations by variable
2798	operationsvar = {}
2799	# individual operations by variable (not repeated)
2800	indivoperationsvar = {}
2801
2802	# Variables with a calculation whic requires vertical interpolation to all the
2803	# file (operations stating by [kdiff]'_pinterp')
2804	LVp = len(diffn + '_pinterp')
2805	varglobalp = []
2806
2807	for oper in vc.keys():
2808	Loper = len(oper)
2809	opn = oper[LD:Loper+1]
2810	vns = vc[oper].split(':')
2811	ops1 = opn.split('+')
2812	doopers[opn] = ops1
2813	for vn in vns:
2814	if not operationsvar.has_key(vn):
2815	operationsvar[vn] = [opn]
2816	indivoperationsvar[vn] = ops1
2817	else:
2818	opers = operationsvar[vn]
2819	opers.append(opn)
2820	operationsvar[vn] = opers
2821	opers1 = indivoperationsvar[vn]
2822	for op1 in ops1:
2823	if not gen.searchInlist(opers1, op1):
2824	opers1.append(op1)
2825	indivoperationsvar[vn] = opers1
2826
2827	if oper[0:LVp] == diffn + '_pinterp':
2828	for vn in vns:
2829	if not gen.searchInlist(varglobalp,vn): varglobalp.append(vn)
2830
2831	iop = iop + 1
2832
2833	if db:
2834	print ' operations to make _______'
2835	gen.printing_dictionary(doopers)
2836	print ' operations by variable _______'
2837	gen.printing_dictionary(operationsvar)
2838	print ' individual operations by variable _______'
2839	gen.printing_dictionary(indivoperationsvar)
2840	print ' variables with the vertical interpolation for all the file _______'
2841	print ' #', varglobalp
2842
2843	if db: print " End of '" + fname + "' "
2844
2845	return doopers, operationsvar, indivoperationsvar, varglobalp
2846
2847	def diffvarop_listconstruct(config, mods, exps, reprj, odir, debug):
2848	""" Function to create the list of differences to compute and draw
2849	config= Configuration of the experiment
2850	mods= list with the models to use
2851	exps= list with the experiments to use
2852	reprj= list with whether re-rpojection needs to be done
2853	odir= output differences folder
2854	"""
2855	fname='diffvarop_listconstruct'
2856
2857	diffop = gen.get_specdictionary_HMT(config, H='DIFFOP_')
2858	diffvar = gen.get_specdictionary_HMT(config, H='DIFFVAR_')
2859
2860	if debug:
2861	if diffop is not None:
2862	print " 'op' differences ________"
2863	gen.printing_dictionary(diffop)
2864	if diffvar is not None:
2865	print " 'var' differences ________"
2866	gen.printing_dictionary(diffvar)
2867
2868	# Getting variable characteristics from each model/experiment
2869	idir1 = config['ofold'] + '/' + mods[0] + '/' + exps[0]
2870	idir2 = config['ofold'] + '/' + mods[1] + '/' + exps[1]
2871
2872	fvarcomp = 'varcompute.inf'
2873	if reprj[0]: fvc1 = 'reproj_' + fvarcomp
2874	else: fvc1 = fvarcomp
2875
2876	if reprj[1]: fvc2 = 'reproj_' + fvarcomp
2877	else: fvc2 = fvarcomp
2878
2879	files1, testfiles1, allcompvar1, Nvar1= read_varcomp_file(idir1 + '/' + fvc1)
2880	files2, testfiles2, allcompvar2, Nvar2= read_varcomp_file(idir2 + '/' + fvc2)
2881
2882	diffops = {}
2883	if diffop is not None:
2884	# Getting 'op' differences characteristics
2885	doops, opvar, indivopvar, varglobalp = get_differences_var(diffop,'op',debug)
2886
2887	Ndiffop = 0
2888	for vn in opvar.keys():
2889	vnops = opvar[vn]
2890	for op in vnops:
2891	vnop = vn + '_' + op
2892	if not allcompvar1.has_key(vnop):
2893	print warnmsg
2894	print ' ' + fname + ": DIFFOP variable+operation '" + vnop + \
2895	"' in '" + mods[0] + '/' + exps[0] + "' was not computed !!"
2896	print ' skipping it!'
2897	print ' variables computed:', allcompvar1
2898	break
2899	if not allcompvar2.has_key(vnop):
2900	print warnmsg
2901	print ' ' + fname + ": DIFFOP variable+operation '" + vnop + \
2902	"' in '" + mods[1] + '/' + exps[1] + "' was not computed !!"
2903	print ' skipping it!'
2904	print ' variables computed:', allcompvar2
2905	break
2906	vals1 = allcompvar1[vnop]
2907	vals2 = allcompvar2[vnop]
2908
2909	headerf1 = vals1[0]
2910	headerf2 = vals2[0]
2911
2912	diffops[vnop] = [mods[0],mods[1],exps[0],exps[1],headerf1,headerf2]
2913
2914	Ndiffop = Ndiffop + 1
2915	else:
2916	Ndiffop = 0
2917
2918	diffvars = {}
2919	if diffvar is not None:
2920	# Getting 'var' differences characteristics
2921	doops,opvar,indivopvar,varglobalp = get_differences_var(diffvar,'var',debug)
2922
2923	Ndiffvar = 0
2924	for vn in opvar.keys():
2925	vnops = opvar[vn]
2926	for op in vnops:
2927	vnop = vn + '_' + op
2928	if not allcompvar1.has_key(vnop):
2929	print warnmsg
2930	print ' ' + fname + ": DIFFVAR variable+operation '" + vnop + \
2931	"' in '" + mods[0] + '/' + exps[0] + "' was not computed !!"
2932	print ' skipping it!'
2933	print ' variables computed:', allcompvar1
2934	break
2935	if not allcompvar2.has_key(vnop):
2936	print warnmsg
2937	print ' ' + fname + ": DIFFVAR variable+operation '" + vnop + \
2938	"' in '" + mods[1] + '/' + exps[1] + "' was not computed !!"
2939	print ' skipping it!'
2940	print ' variables computed:', allcompvar2
2941	break
2942	vals1 = allcompvar1[vnop]
2943	vals2 = allcompvar2[vnop]
2944
2945	headerf1 = vals1[0]
2946	headerf2 = vals2[0]
2947
2948	diffvars[vnop] = [mods[0],mods[1],exps[0],exps[1],headerf1,headerf2]
2949
2950	Ndiffvar = Ndiffvar + 1
2951	else:
2952	Ndiffvar = 0
2953
2954	if diffop is not None:
2955	Sopdiffs = gen.dictKeysVals_stringList(diffops)
2956	else:
2957	Sopdiffs = 'none'
2958	# Outwritting the op diffferences file to avoid next time
2959	objf = open(odir + '/diffop.inf', 'w')
2960	objf.write('differences: ' + Sopdiffs + '\n')
2961	objf.write('Ndiff: ' + str(Ndiffop) + '\n')
2962	objf.close()
2963
2964	if diffvar is not None:
2965	Svardiffs = gen.dictKeysVals_stringList(diffvars)
2966	else:
2967	Svardiffs = 'none'
2968	# Outwritting the var diffferences file to avoid next time
2969	objf = open(odir + '/diffvar.inf', 'w')
2970	objf.write('differences: ' + Svardiffs + '\n')
2971	objf.write('Ndiff: ' + str(Ndiffvar) + '\n')
2972	objf.close()
2973
2974	if debug: print " End of '" + fname + "' "
2975
2976	return diffops, diffvars, Ndiffop, Ndiffvar
2977
2978	def read_diff_file(difffile):
2979	""" Function to read the file with the information about the differences
2980	difffile= file with the information
2981	"""
2982	fname = 'read_diff_file'
2983
2984	if not os.path.isfile(difffile):
2985	print errormsg
2986	print ' ' + fname + ": differences file '" + difffile + "' does not exist !!"
2987	quit(-1)
2988
2989	objf = open(difffile, 'r')
2990	for line in objf:
2991	if line[0:1] != '#' and len(line) > 1:
2992	values = line.replace('\n','').split(' ')
2993	if values[0] == 'differences:':
2994	if values[1] != 'none':
2995	diffs = gen.stringList_dictKeysVals(values[1])
2996	else:
2997	diffs = {}
2998	elif values[0] == 'Ndiff:':
2999	Ndiffs = int(values[1])
3000
3001	objf.close()
3002
3003	return diffs, Ndiffs
3004
3005	def compute_op_diffs(config, diffallop, odir, diffscr, debug):
3006	""" Function to compute operation differences
3007	config= experiment configuration
3008	diffallop= dictonary with the differences to compute
3009	odir= output folder
3010	diffscr= wether it should be done from the scratch
3011	"""
3012	fname = 'compute_op_diffs'
3013
3014	# output folder
3015	ofold = config['ofold']
3016
3017	# Home of the python scripts
3018	pyH = config['pyHOME']
3019
3020	# opsur = operation surnames
3021	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
3022	Opsurs = {}
3023	for opk in opsur:
3024	opn = opk.split('_')[1]
3025	vals = opsur[opk]
3026	Opsurs[opn] = vals.split(':')
3027
3028	# Getting in working dir
3029	os.chdir(odir)
3030
3031	# CF dimensions
3032	CFdims = ['lon', 'lat', 'pres', 'time']
3033
3034	# Trakcing file
3035	trkobjf = open(odir + '/all_diffop.inf', 'a')
3036
3037	for vnop in diffallop.keys():
3038	vn = vnop.split('_')[0]
3039	op = vnop.split('_')[1]
3040	opS = op.replace('+','_')
3041
3042	# `p' header add related to 'pinterp'
3043	if opS.find('pinterp') != -1: SgP = 'p'
3044	else: SgP = ''
3045
3046	# Variable name within the file
3047	vninF = varnoper(vn, op, Opsurs)
3048
3049	vals = diffallop[vnop]
3050	mod1 = vals[0]
3051	mod2 = vals[1]
3052	exp1 = vals[2]
3053	exp2 = vals[3]
3054	headerf1 = vals[4]
3055	headerf2 = vals[5]
3056	modexpdiff = mod2 + '/' + exp2 + ' - ' + mod1 + '/' + exp1
3057	modexpdiffS = mod2 + '-' + exp2 + '_' + mod1 + '-' + exp1
3058
3059	ifile1 = ofold + '/' + mod1 + '/' + exp1 + '/' + vn + '_' + headerf1 + SgP + \
3060	'_' + opS + '.nc'
3061	ifile2 = ofold + '/' + mod2 + '/' + exp2 + '/' + vn + '_' + headerf2 + SgP + \
3062	'_' + opS + '.nc'
3063	difffilen = odir + '/' + vn + '_diffop_' + modexpdiffS + '_' + opS + '.nc'
3064
3065	if diffscr:
3066	sout = sub.call('rm ' + difffilen + ' >& /dev/null', shell=True)
3067
3068	if not os.path.isfile(difffilen):
3069	if debug:
3070	print " Computing operation difference '" + op + "' of '" + vn + \
3071	"' for: '" + modexpdiff
3072	print " var in file: '" + vninF + "'"
3073
3074	# Range of the dimensions
3075	for CFd in CFdims:
3076	if CFd == CFdims[0]:
3077	CFdimS = CFd + '\|' + CFd + '\|-1'
3078	else:
3079	CFdimS = CFdimS + ';' + CFd + '\|' + CFd + '\|-1'
3080
3081	ncvar.ivattrs(ifile2,vninF)
3082
3083	# Attributes of the variable
3084	try:
3085	with gen.Capturing() as output:
3086	varattrs = ncvar.ivattrs(ifile2,vninF)
3087	except:
3088	print errmsg
3089	print 'ncvar.ivarattrs(' + ifile2 + ',' + vninF + ')'
3090	for s1out in output: print s1out
3091	quit(-1)
3092
3093	if debug:
3094	for s1out in output: print s1out
3095	varvals = vninF + ',' + varattrs['long_name'][0] + ',' + \
3096	varattrs['units'][0]
3097
3098	values = CFdimS + '@' + 'add\|' + ifile2 + '\|' + vninF + '%' + CFdimS + \
3099	'@' + 'sub\|' + ifile1 + '\|' + vninF
3100	pyins = 'python ' + pyH + "/nc_var.py -o compute_opersvarsfiles -S '" + \
3101	values + "' -v '" + varvals + "'"
3102
3103	try:
3104	with gen.Capturing() as output:
3105	ncvar.compute_opersvarsfiles(values,varvals)
3106	except:
3107	print errmsg
3108	print 'ncvar.compute_opersvarsfiles(' + values + ',' + varvals + ')'
3109	for s1out in output: print s1out
3110	quit(-1)
3111
3112	sout = sub.call('mv opersvarsfiles_'+vninF+'.nc ' + difffilen, shell=True)
3113
3114	# keeping all differences
3115	trkobjf.write('\n')
3116	trkobjf.write("#" + vn + ' ' + op + '\n')
3117	trkobjf.write( pyins + '\n')
3118
3119	trkobjf.close()
3120	if debug: print " End of '" + fname + "' "
3121
3122	return
3123
3124	def compute_diff_statistics(Ecnf, filen, ofold, varn, Opers, scr, db):
3125	""" Function to compute statistics from var diff files
3126	Ecnf= dictionary with the configuration of the experiment
3127	filen= variable difference file to use
3128	ofold= directory to write the output files
3129	varn= name of the variable
3130	Opers= kind of operation: (as possible multiple consecutive combination of operations separated by '+'
3131	[calc1]+[calc2] will compute first [calc1] and then [calc2]
3132	acc: temporal accumulated values
3133	diff: differences between models
3134	dimvarvalue~[dimvarn]~[value]: Slicing along the index at the nearest given value of a dimension-variable
3135	direct: no statistics
3136	last: last temporal value
3137	Lmean: latitudinal mean values
3138	Lsec: latitudinal section (latitudinal value must be given, [var]@[lat])
3139	lmean: longitudinal mean values
3140	lsec: longitudinal section (longitudinal value must be given, [var]@[lat])
3141	pinterp: pressure interpolation (to the given $plevels)
3142	tmean: temporal mean values
3143	tstd: temporal standard deviation values
3144	tturb: Taylor's turbulence decomposition value (x - <x>) for time
3145	tvar: temporal variance values
3146	xmean: x-axis mean values
3147	xvar: x-axis variance values
3148	ymean: y-axis mean values
3149	zsum: vertical aggregated values
3150	scr= should it be done from the scratch?
3151	"""
3152	fname='compute_diff_statistics'
3153
3154	# Full header of file
3155	Lfilen = len(filen)
3156	Hfile = filen[0:Lfilen-3]
3157
3158	# CF dimension-variables name
3159	OpersS = '+'.join(Opers)
3160	if OpersS.find('pinterp') != -1 or filen.find('pinterp') != -1:
3161	varnCFs = ['lon', 'lat', 'pres', 'time']
3162	else:
3163	varnCFs = ['lon', 'lat', 'time']
3164
3165	# Variable-dimension CF dictionary
3166	CFdimvardict = {'lon': 'lon', 'lat': 'lat', 'pres': 'pres', 'time': 'time'}
3167
3168	# Python scripts HOME
3169	pyH = Ecnf['pyHOME']
3170
3171	# opsur = operation surnames
3172	opsur = gen.get_specdictionary_HMT(Ecnf, H='opsur_',M='',T='')
3173	Opsurs = {}
3174	for opk in opsur:
3175	opn = opk.split('_')[1]
3176	vals = opsur[opk]
3177	Opsurs[opn] = vals.split(':')
3178
3179	# Getting in working dir
3180	os.chdir(ofold)
3181
3182	# File to keep track of all the statistics of var differences
3183	otrackf = open(ofold + '/all_vardiffstatistics.inf', 'a')
3184
3185	if db:
3186	print " computing statistics of diff variable: '", varn ,"' operation '"+ \
3187	OpersS + "' using file '" + filen + " ...'"
3188
3189	for op in Opers:
3190	# File name from previous operations, and name of the variable within the
3191	# file (some operations change the name of the variable)
3192	if op == Opers[0]:
3193	Fopers = op
3194	prevfile = filen
3195	vninF = varn
3196	else:
3197	Fopers = Fopers + '_' + op
3198	prevfile = fileon
3199	fileon = Hfile + '_' + Fopers + '.nc'
3200
3201	SvarnCFs = ',' + ','.join(varnCFs)
3202	CFvarnp = vninF + SvarnCFs
3203
3204	if scr:
3205	sout = sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
3206
3207	# Just produce file in case output file does not exist
3208	if not os.path.isfile(fileon):
3209	if db:
3210	print ' ' + fname + ": creation of file '" + fileon + "' ..."
3211	dofile = True
3212	else:
3213	dofile = False
3214
3215	# Variables to be kept in the final file
3216	varkeep = []
3217
3218	if op == 'acc':
3219	# temporal accumulated values
3220	print " " + fname + ": kind '" + op + "' not ready !!"
3221	wrongstats.append(varn + '_' + opers)
3222	break
3223
3224	elif op[0:11] == 'dimvarvalue':
3225	# Slicing along the index at the nearest given value of a dimension-variable
3226	dimvarn = op.split('~')[1]
3227	dimvalue = op.split('~')[2]
3228	vals = dimvarn + ',' + dimvalue + ',' + dimvalue + ',0'
3229	if dofile:
3230	try:
3231	with gen.Capturing() as output:
3232	ncvar.DataSetSection_multivars(vals,prevfile,CFvarnp)
3233	except:
3234	print errmsg
3235	print 'DataSetSection_multivars('+vals+',', prevfile, ',' + \
3236	CFvarnp + ')'
3237	for s1out in output: print s1out
3238	quit(-1)
3239
3240	if db:
3241	for s1out in output: print s1out
3242
3243	sout = sub.call('mv DataSetSection_multivars.nc '+fileon, shell=True)
3244
3245	# Keeping the operations
3246	pyins=pyH + "/nc_var.py -o DataSetSection_multivars -S '" + vals + \
3247	"' -f " + prevfile + " -v " + CFvarnp
3248	otrackf.write("\n")
3249	otrackf.write("# " + CFvarnp + " " + Fopers + "\n")
3250	otrackf.write(pyins + "\n")
3251
3252	# removing the given variable-dimension
3253	varnCFs.remove(dimvarn)
3254
3255	elif op == 'direct':
3256	# no statistics
3257	if dofile:
3258	sout = sub.call('mv ' + prevfile + ' ' + fileon, shell=True)
3259	elif op == 'last':
3260	# last temporal value
3261	vals='time,-9,0,0'
3262	if dofile:
3263	try:
3264	with gen.Capturing() as output:
3265	ncvar.DataSetSection(vals,prevfile)
3266	except:
3267	print errmsg
3268	print 'DataSetSection('+vals+',', prevfile, ')'
3269	for s1out in output: print s1out
3270	quit(-1)
3271
3272	if db:
3273	for s1out in output: print s1out
3274
3275	hprevfile = prevfile[0:len(prevfile)-3]
3276
3277	sout = sub.call('mv ' + hprevfile + '_time_B-9-E0-I0.nc ' + fileon, \
3278	shell=True)
3279
3280	# Keeping the operations
3281	pyins=pyH + "/nc_var.py -o DataSetSection -S '" + vals + "' -f " + \
3282	prevfile
3283	otrackf.write("\n")
3284	otrackf.write("# " + varn + " " + Fopers + "\n")
3285	otrackf.write(pyins + "\n")
3286
3287	elif op =='Lmean':
3288	# latitudinal mean values
3289	print " " + fname + ": kind '" + op + "' not ready !!"
3290	wrongstats.append(varn + '_' + opers)
3291	break
3292	elif op =='Lsec':
3293	# latitudinal section (latitudinal value must be given, [var]@[lat])
3294	print " " + fname + ": kind '" + op + "' not ready !!"
3295	wrongstats.append(varn + '_' + opers)
3296	break
3297	elif op =='lmean':
3298	# longitudinal mean values
3299	print " " + fname + ": kind '" + op + "' not ready !!"
3300	wrongstats.append(varn + '_' + opers)
3301	break
3302	elif op =='lsec':
3303	# longitudinal section (longitudinal value must be given, [var]@[lon])
3304	print " " + fname + ": kind '" + op + "' not ready !!"
3305	wrongstats.append(varn + '_' + opers)
3306	break
3307	elif op == 'tmean':
3308	# temporal mean values
3309	vals='time\|-1,time,mean,' + ':'.join(varnCFs)
3310	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3311	if dofile:
3312	try:
3313	with gen.Capturing() as output:
3314	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3315	except:
3316	print errmsg
3317	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3318	for s1out in output: print s1out
3319	quit(-1)
3320
3321	if db:
3322	for s1out in output: print s1out
3323
3324	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
3325
3326	# Keeping the operations
3327	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3328	"' -f " + prevfile + " -v " + CFvarnp
3329	otrackf.write("\n")
3330	otrackf.write("# " + varn + " " + Fopers + "\n")
3331	otrackf.write(pyins + "\n")
3332
3333	# removing dimension variable-dimension 'time'
3334	varnCFs.remove('time')
3335
3336	varkeep.append('timestats')
3337
3338	elif op == 'tstd':
3339	# temporal standard deviation values
3340	vals='time\|-1,time,std,' + ':'.join(varnCFs) + ':' + vdnz
3341	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
3342	if dofile:
3343	try:
3344	with gen.Capturing() as output:
3345	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3346	except:
3347	print errmsg
3348	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3349	for s1out in output: print s1out
3350	quit(-1)
3351
3352	if db:
3353	for s1out in output: print s1out
3354
3355	sout = sub.call('mv file_oper_alongdims_std.nc '+fileon, shell=True)
3356
3357	# Keeping the operations
3358	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3359	"' -f " + prevfile + " -v " + CFvarnp
3360	otrackf.write("\n")
3361	otrackf.write("# " + varn + " " + Fopers + "\n")
3362	otrackf.write(pyins + "\n")
3363
3364	# removing dimension variable-dimension 'time'
3365	varnCFs.remove('time')
3366
3367	varkeep.append('timestats')
3368
3369	elif op == 'tturb':
3370	# temporal turbulent values
3371	vals='time\|-1,time,turb,' + ':'.join(varnCFs)
3372	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3373	if dofile:
3374	try:
3375	with gen.Capturing() as output:
3376	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3377	except:
3378	print errmsg
3379	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3380	for s1out in output: print s1out
3381	quit(-1)
3382
3383	if db:
3384	for s1out in output: print s1out
3385
3386	sout = sub.call('mv file_oper_alongdims_turb.nc '+fileon, shell=True)
3387
3388	# Keeping the operations
3389	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3390	"' -f " + prevfile + " -v " + CFvarnp
3391	otrackf.write("\n")
3392	otrackf.write("# " + varn + " " + Fopers + "\n")
3393	otrackf.write(pyins + "\n")
3394
3395	varkeep.append('timestats')
3396
3397	elif op == 'tvar':
3398	# temporal variance values (Taylor's turbulence)
3399	vals='time\|-1,time,var,' + ':'.join(varnCFs) + ':' + vdnz
3400	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@') + dnz+'@'+vdnz
3401	if dofile:
3402	try:
3403	with gen.Capturing() as output:
3404	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3405	except:
3406	print errmsg
3407	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3408	for s1out in output: print s1out
3409	quit(-1)
3410
3411	if db:
3412	for s1out in output: print s1out
3413
3414	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
3415
3416	# Keeping the operations
3417	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3418	"' -f " + prevfile + " -v " + CFvarnp
3419	otrackf.write("\n")
3420	otrackf.write("# " + varn + " " + Fopers + "\n")
3421	otrackf.write(pyins + "\n")
3422
3423	# removing dimension variable-dimension 'time'
3424	varnCFs.remove('time')
3425
3426	varkeep.append('timestats')
3427
3428	elif op == 'xmean':
3429	# x-axis mean values
3430	vals='lon\|-1,lon,mean,' + ':'.join(varnCFs)
3431	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3432	if dofile:
3433	try:
3434	with gen.Capturing() as output:
3435	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3436	except:
3437	print errmsg
3438	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3439	for s1out in output: print s1out
3440	quit(-1)
3441
3442	if db:
3443	for s1out in output: print s1out
3444
3445	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
3446
3447	# Keeping the operations
3448	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3449	"' -f " + prevfile + " -v " + CFvarnp
3450	otrackf.write("\n")
3451	otrackf.write("# " + varn + " " + Fopers + "\n")
3452	otrackf.write(pyins + "\n")
3453
3454	# removing dimension variable-dimension 'lon'
3455	varnCFs.remove('lon')
3456
3457	varkeep.append('lonstats')
3458
3459	elif op == 'xvar':
3460	# x-axis variance values
3461	vals='lon\|-1,lon,var,' + ':'.join(varnCFs)
3462	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3463	if dofile:
3464	try:
3465	with gen.Capturing() as output:
3466	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3467	except:
3468	print errmsg
3469	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3470	for s1out in output: print s1out
3471	quit(-1)
3472
3473	if db:
3474	for s1out in output: print s1out
3475
3476	sout = sub.call('mv file_oper_alongdims_var.nc '+fileon, shell=True)
3477
3478	# Keeping the operations
3479	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3480	"' -f " + prevfile + " -v " + CFvarnp
3481	otrackf.write("\n")
3482	otrackf.write("# " + varn + " " + Fopers + "\n")
3483	otrackf.write(pyins + "\n")
3484
3485	# removing dimension variable-dimension 'lon'
3486	varnCFs.remove('lon')
3487
3488	varkeep.append('lonstats')
3489
3490	elif op == 'ymean':
3491	# y-axis mean values
3492	vals='lat\|-1,lat,mean,' + ':'.join(varnCFs)
3493	dims = gen.dictvar_listS(varnCFs,CFdimvardict,',','@')
3494	if dofile:
3495	try:
3496	with gen.Capturing() as output:
3497	ncvar.file_oper_alongdims(vals,prevfile,CFvarnp)
3498	except:
3499	print errmsg
3500	print 'file_oper_alongdims('+vals+',', prevfile, ','+CFvarnp+')'
3501	for s1out in output: print s1out
3502	quit(-1)
3503
3504	if db:
3505	for s1out in output: print s1out
3506
3507	sout = sub.call('mv file_oper_alongdims_mean.nc '+fileon, shell=True)
3508
3509	# Keeping the operations
3510	pyins=pyH + "/nc_var.py -o file_oper_alongdims -S '" + vals + \
3511	"' -f " + prevfile + " -v " + CFvarnp
3512	otrackf.write("\n")
3513	otrackf.write("# " + varn + " " + Fopers + "\n")
3514	otrackf.write(pyins + "\n")
3515
3516	# removing dimension variable-dimension 'lat'
3517	varnCFs.remove('lat')
3518
3519	varkeep.append('latstats')
3520
3521	elif op == 'zsum':
3522	# vertical aggregated values
3523	print " " + fname + ": kind '" + op + "' not ready !!"
3524	wrongstats.append(varn + '_' + opers)
3525	break
3526	else:
3527	print errmsg
3528	print ' ' + fname + ": operation '" + op + "' not ready !!"
3529	quit(-1)
3530
3531	# End of kind of operation
3532
3533	# Variable name in file (vninF) changed due to operation
3534	# but only if previous operation does not the same 'statistic'
3535	chvn = gen.dictionary_key_list(Opsurs, op)
3536	if chvn is not None:
3537	oldvninF = vninF
3538	vninF = vninF + chvn
3539	CFvarnp = CFvarnp.replace(oldvninF,vninF)
3540
3541	if dofile:
3542	if len(varkeep) > 0:
3543	varkeepS = ',' + ','.join(varkeep)
3544	else:
3545	varkeepS = ''
3546
3547	# Adding such CF dimension variables which might not be in the
3548	# post-operation file
3549	try:
3550	with gen.Capturing() as output:
3551	varinfile = ncvar.ivars(fileon)
3552	except:
3553	print errmsg
3554	print 'ivars('+fileon+')'
3555	for s1out in output: print s1out
3556	# Removing file in order to make sure that it will be redone
3557	print ' ' + fname + " removing file '" + fileon + "' ..."
3558	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
3559	quit(-1)
3560
3561	for CFvn in varnCFs:
3562	if not gen.searchInlist(varinfile, CFvn):
3563	if db:
3564	print ' ' + fname + ": recupering CF variable '" + CFvn + "'"
3565	try:
3566	with gen.Capturing() as output:
3567	ncvar.fvaradd(prevfile+','+CFvn, fileon)
3568	except:
3569	print errmsg
3570	print 'fvaradd(' + prevfile + ',' + CFvn +', ' + fileon + ')'
3571	for s1out in output: print s1out
3572	# Removing file in order to make sure that it will be redone
3573	print ' ' + fname + " removing file '" + fileon + "' ..."
3574	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
3575	quit(-1)
3576
3577	if db:
3578	for s1out in output: print s1out
3579
3580	totalvarkeeps = CFvarnp + ',' + ','.join(varnCFs) + varkeepS
3581	try:
3582	with gen.Capturing() as output:
3583	oclean = ncvar.cleaning_varsfile(totalvarkeeps,fileon)
3584	except:
3585	print errmsg
3586	print 'cleaning_varsfile('+totalvarkeeps+','+fileon+')'
3587	for s1out in output: print s1out
3588	# Removing file in order to make sure that it will be redone
3589	print ' ' + fname + " removing file '" + fileon + "' ..."
3590	sub.call('rm ' + fileon + ' >& /dev/null', shell=True)
3591	quit(-1)
3592
3593	if db:
3594	for s1out in output: print s1out
3595
3596	# End of operations
3597	otrackf.close()
3598
3599	if db: print " End of '" + fname + "' "
3600
3601	return
3602
3603	def compute_var_diffs(config, diffallvar, odir, diffscr, debug):
3604	""" Function to compute variable differences
3605	config= experiment configuration
3606	diffallvar= dictonary with the differences to compute
3607	odir= output folder
3608	diffscr= wether it should be done from the scratch
3609	"""
3610	fname = 'compute_var_diffs'
3611
3612	# output folder
3613	ofold = config['ofold']
3614
3615	# Home of the python scripts
3616	pyH = config['pyHOME']
3617
3618	# opsur = operation surnames
3619	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
3620	Opsurs = {}
3621	for opk in opsur:
3622	opn = opk.split('_')[1]
3623	vals = opsur[opk]
3624	Opsurs[opn] = vals.split(':')
3625
3626	# Getting in working dir
3627	os.chdir(odir)
3628
3629	# CF dimensions
3630	CFdims = ['lon', 'lat', 'pres', 'time']
3631
3632	# Trakcing file
3633	trkobjf = open(odir + '/all_diffvar.inf', 'a')
3634
3635	for vnop in diffallvar.keys():
3636	vn = vnop.split('_')[0]
3637	op = vnop.split('_')[1]
3638	opSfinal = op.replace('+','_')
3639
3640	lop = op.split('+')
3641	# `p' header add related to 'pinterp'
3642	if op.find('pinterp') != -1:
3643	print warnmsg
3644	print ' ' + fname + ': there is the vertical interpolation as operation'
3645	print ' variable differences will be computed from that files where '
3646	print ' vertical interpolation was just done'
3647	ipop = lop.index('pinterp')
3648	opS = '_' + '_'.join(lop[0:ipop+1])
3649	doops = lop[ipop+1:]
3650	SgP = 'p'
3651	vninF = varnoper(vn, '+'.join(lop[0:ipop+1]), Opsurs)
3652	print ' operations to perform:', doops
3653	else:
3654	SgP = ''
3655	opS = ''
3656	doops = lop
3657	vninF = vn
3658
3659	Sdoops = '+'.join(doops)
3660
3661	# Variable name within the file
3662
3663	vals = diffallvar[vnop]
3664	mod1 = vals[0]
3665	mod2 = vals[1]
3666	exp1 = vals[2]
3667	exp2 = vals[3]
3668	headerf1 = vals[4]
3669	headerf2 = vals[5]
3670	modexpdiff = mod2 + '/' + exp2 + ' - ' + mod1 + '/' + exp1
3671	modexpdiffS = mod2 + '-' + exp2 + '_' + mod1 + '-' + exp1
3672
3673	ifile1 = ofold + '/' + mod1 + '/' + exp1 + '/' + vn + '_' + headerf1 + SgP + \
3674	opS + '.nc'
3675	ifile2 = ofold + '/' + mod2 + '/' + exp2 + '/' + vn + '_' + headerf2 + SgP + \
3676	opS + '.nc'
3677	difffilen = odir + '/' + vn + '_diffvar_' + modexpdiffS + opS + '.nc'
3678	difffinalfilen = odir + '/' + vn + '_diffvar_' + modexpdiffS + opS + '_' + \
3679	Sdoops + '.nc'
3680
3681	if diffscr:
3682	sout = sub.call('rm ' + difffilen + ' >& /dev/null', shell=True)
3683	sout = sub.call('rm ' + difffinalfilen + ' >& /dev/null', shell=True)
3684
3685	if not os.path.isfile(difffinalfilen):
3686	if debug:
3687	print " Computing variable difference of '" + vn + "' for: '" + \
3688	modexpdiff + "' and performing operation '" + Sdoops + "'"
3689	print " var in file: '" + vninF + "'"
3690
3691	# Range of the dimensions
3692	for CFd in CFdims:
3693	if CFd == CFdims[0]:
3694	CFdimS = CFd + '\|' + CFd + '\|-1'
3695	else:
3696	CFdimS = CFdimS + ';' + CFd + '\|' + CFd + '\|-1'
3697
3698	# Attributes of the variable
3699	with gen.Capturing() as output:
3700	varattrs = ncvar.ivattrs(ifile2,vninF)
3701	if debug:
3702	for s1out in output: print s1out
3703	varvals = vninF + ',' + varattrs['long_name'][0] + ',' + \
3704	varattrs['units'][0]
3705
3706	values = CFdimS + '@' + 'add\|' + ifile2 + '\|' + vninF + '%' + CFdimS + \
3707	'@' + 'sub\|' + ifile1 + '\|' + vninF
3708	pyins = 'python ' + pyH + "/nc_var.py -o compute_opersvarsfiles -S '" + \
3709	values + "' -v '" + varvals + "'"
3710
3711	try:
3712	with gen.Capturing() as output:
3713	ncvar.compute_opersvarsfiles(values,varvals)
3714	except:
3715	print errmsg
3716	print 'ncvar.compute_opersvarsfiles(' + values + ', ' + varvals + ')'
3717	for s1out in output: print s1out
3718	quit(-1)
3719
3720	sout = sub.call('mv opersvarsfiles_'+vninF+'.nc ' + difffilen, shell=True)
3721
3722	# keeping all differences
3723	trkobjf.write('\n')
3724	trkobjf.write("#" + vn + ' ' + op + '\n')
3725	trkobjf.write( pyins + '\n')
3726
3727	# Computing statisitcs
3728	compute_diff_statistics(config, difffilen, odir, vninF, doops, diffscr, \
3729	debug)
3730
3731	trkobjf.close()
3732
3733	if debug: print " End of '" + fname + "' "
3734
3735	return
3736
3737
3738	def draw_diff_plots(config, plots, odir, allvarcomp, kdiff, figscr, debug):
3739	""" Function to draw all plots
3740	config= Configuration of the experiment
3741	plots= dictionary with the plots
3742	odir= output experiment folder
3743	allvarcomp= dictionary with all the variables to compute and their information
3744	kdiff= kind of differences:
3745	'diffop': plots from operation differences
3746	'diffvar': plots from variable differences
3747	figscr= whether figures should be done from the scratch or not
3748
3749	* Plot as
3750	{[kplot]} = [varn1]\|[op1]#[varn2]\|[op2]#[...[varnN]\|[opN]], ...
3751	[kplot] ___
3752	diffmap2Dsfc: 2D map of surface differences values of 1 variable
3753	diffmap2Dz: 2D map of 3D differences values of 1 variable
3754	map2Dsfc: 2D map of surface values of 1 variable
3755	map3D: 2D map of 3D values of 1 variable
3756	shadconthovmsfc: Hovmoeller diagrams of 2 variable at the surface in shadow and the other in contourn
3757	shadcont2Dsfc: 2D map of shadow (1st variable) and countour (2nd variable) [stvar1]#[stvar2]
3758	shadcont2Dzsec: 2D map of vertical section of 2 variables one in shadow and the other in contourn
3759	[varn]
3760	variable
3761	[op]
3762	'+' separated list of operations
3763	in figures with more than 1 variable, use '#' to separate the [varn]\|[op]
3764	"""
3765	fname = 'draw_diff_plots'
3766
3767	os.chdir(odir)
3768
3769	# Dictionary with the operations with surnames for the operated variable
3770	opersurnames = {}
3771	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
3772	for opsr in opsur.keys():
3773	opn = opsr.split('_')[1]
3774	vls = opsur[opsr].split(':')
3775	opersurnames[opn] = vls
3776
3777	# time values
3778	# Units time for the plots
3779	rd = config['CFreftime']
3780	tunits = config['CFunitstime'] + '!since!' + rd[0:4] + '-' + rd[4:6] + '-' + \
3781	rd[6:8] + '!' + rd[8:10] + ':' + rd[10:12] + ':' + rd[12:14]
3782	# time ticks kind
3783	tkind = config['timekind']
3784	# time ticks format
3785	tfmt = config['timefmt']
3786	# time axis label
3787	tlab = config['timelabel']
3788	timevals = [tunits, tkind, tfmt, tlab]
3789
3790	if kdiff == 'diffop':
3791	specplotkeyn = 'specificdiffopplot'
3792	elif kdiff == 'diffvar':
3793	specplotkeyn = 'specificdiffvarplot'
3794	else:
3795	print errmsg
3796	print ' ' + fname + ": differences kind '" + kdiff + "' not ready !!"
3797	quit(-1)
3798
3799	# Dictionary of plot specificities
3800	plotspecifics = {}
3801	if config.has_key(specplotkeyn):
3802	# [minval]: minimum value
3803	# [maxval]: minimum value
3804	# [colorbar]: name of the colorbar (from matplotlib) to use
3805	# [cntformat]: format of the contour labels
3806	# [colorcnt]: color for the countor lines
3807	plotspecs = config[specplotkeyn].split(':')
3808	for pltspc in plotspecs:
3809	pltvls = pltspc.split('\|')
3810	vn = pltvls[0]
3811	op = pltvls[1]
3812	fn = pltvls[2]
3813	plotspecifics[fn + '_' + vn + '_' + op] = pltvls[3:]
3814	if debug:
3815	print 'Specific values for plots _______'
3816	gen.printing_dictionary(plotspecifics)
3817
3818	# Kind of figures
3819	kindfigure = config['kindfig']
3820
3821	# Graphical labels and configuration
3822	modgc, expgc = graphical_characteristics(config, debug)
3823
3824	# Map value
3825	mapvalue = config['mapval']
3826
3827	# pythone scripts HOME
3828	pyHOME = config['pyHOME']
3829
3830	# Title-text of operations
3831	opexplained = {}
3832	optits = config['titleoperations'].split(':')
3833	for optit in optits:
3834	opn = optit.split('\|')[0]
3835	opt = optit.split('\|')[1]
3836	opexplained[opn] = opt
3837	if debug:
3838	print 'Titles for operations _______'
3839	gen.printing_dictionary(opexplained)
3840
3841	for kplot in plots.keys():
3842	varsplt = plots[kplot]
3843	for varplt in varsplt:
3844	if debug:
3845	print " printing '" + kplot + "' var ':" + varplt + "'..."
3846	varops = varplt.split('#')
3847
3848	# CF variables in plot
3849	CFvarsplot = []
3850	# Files in plot
3851	filesplot = []
3852	# Variables in plot within the files
3853	varsplot = []
3854	# Dims in figure
3855	dimsplot = []
3856	# pictoric values in figure
3857	pictplot = []
3858	# Name of the figure
3859	figname = ''
3860	# Title of the figure
3861	titfigure = ''
3862
3863	ivp = 0
3864	for varop in varops:
3865	vn = varop.split('\|')[0]
3866	op = varop.split('\|')[1]
3867
3868	# CF variables in plot
3869	CFvarsplot.append(vn)
3870
3871	vnopS = vn + '_' + op
3872	if not allvarcomp.has_key(vnopS):
3873	print errmsg
3874	print ' ' + fname + ": no file for variable-operation '" + \
3875	vnopS + "' !!"
3876	print ' available ones:', allvarcomp.keys()
3877	quit(-1)
3878	vopvals = allvarcomp[vnopS]
3879	mod1 = vopvals[0]
3880	mod2 = vopvals[1]
3881	mod3 = vopvals[2]
3882	mod4 = vopvals[3]
3883	headf = vopvals[4]
3884
3885	modv = modgc[mod1]
3886	expv = expgc[mod1+'/'+exp1]
3887	modlab1 = modv.label
3888	explab1 = expv.label
3889	modv = modgc[mod2]
3890	expv = expgc[mod2+'/'+exp2]
3891	modlab2 = modv.label
3892	explab2 = expv.label
3893
3894	modexpdiff = explab2 + '-' + explab1
3895	modexpdiffS = mod2 + '-' + exp2 + '_' + mod1 + '-' + exp1
3896
3897	difffilen = odir + '/' + vn + '_' + kdiff + '_' + modexpdiffS + \
3898	'_' + op.replace('+','_') + '.nc'
3899
3900	filesplot.append(difffilen)
3901	# Do we have processed the given difference?
3902	if not os.path.isfile(difffilen):
3903	print warnmsg
3904	print " " + fname + ": there is no file for '" + kdiff + \
3905	"' difference '" + varop + "' skiping it !!"
3906	break
3907
3908	# Name of the variable inside the file
3909	vnsur = varnoper(vn, op, opersurnames)
3910	varsplot.append(vnsur)
3911
3912	# Dimensions in file
3913	try:
3914	with gen.Capturing() as output:
3915	dims = ncvar.idims(difffilen)
3916	except:
3917	print errmsg
3918	print 'ncvar.idims('+difffilen+')'
3919	for s1out in output: print s1out
3920	quit(-1)
3921
3922	dimsplot.append(dims)
3923
3924	# pictoric values for the figure
3925	Sfivaop = kplot + '_' + vn + '_' + op
3926	if plotspecifics.has_key(Sfivaop):
3927	pictvals = plotspecifics[Sfivaop]
3928	else:
3929	Vvals = gen.variables_values(vn)
3930	pictvals = [Vvals[2], Vvals[3], Vvals[6], '%g', 'fixsigc','black']
3931
3932	pictplot.append(pictvals)
3933
3934	# Header of the name of the figure
3935	if ivp == 0:
3936	figname = kplot + '_' + vn + '_' + kdiff + '_' + modexpdiffS + \
3937	'_' + op.replace('+','-')
3938	else:
3939	figname = figname + '-' + vn + '_' + op.replace('+','-')
3940
3941	# Title of the figure:
3942	if mod1 == mod2:
3943	modexptit = mod1 + '!' + exp2 + '-' + exp1
3944	else:
3945	modexptit = modexpdiff
3946
3947	ivp = ivp + 1
3948	# End of variable-operation
3949	figname = figname + '.' + kindfigure
3950
3951	# Title of figure
3952	titfigure = create_figure_title(modexptit, kdiff, varops, opexplained)
3953
3954	if len(titfigure) > 80:
3955	print warnmsg
3956	print ' ' + fname + ": figure title '" + titfigure.replace('!', ' ')+\
3957	"' larger than 80 characters (actually", len(titfigure), ') !!'
3958	print " simplifying it to '" + modexptit.replace('!',' ') + '!' + \
3959	kdiff + '!' + vnS + "'"
3960	titfigure = modexptit.replace(' ','!') + '!' + kdiff + '!' + vnS
3961
3962	draw_plot(kplot, CFvarsplot, filesplot, varsplot, dimsplot, pictplot, \
3963	figname, titfigure, kindfigure, mapvalue, timevals, expgc, odir, \
3964	pyHOME, figscr, debug)
3965
3966	# End of variables-operations
3967
3968	# End of kind of plots
3969
3970	if debug: print " End of '" + fname + "' "
3971
3972	return
3973
3974	def reproject_modexp(mod,exp,config,REPRJops,fscr,fadd,debug):
3975	""" Function to re-project and re-compute files from a given model/experiment
3976	mod= model
3977	exp= experiment
3978	config= configuration
3979	REPRJops= dictionary with the reprojection operators to use for reproject and for which variable
3980	# remapbil CDO: Bilinear interpolation
3981	# remapbic CDO: Bicubic interpolation
3982	# remapdis CDO: Distance-weighted average remapping
3983	# remapnn CDO: Nearest neighbor remapping
3984	# Operators only available if projections have the corners of the grid points.
3985	# remapcon CDO: First order conservative remapping
3986	# remapcon2 CDO: Second order conservative remapping
3987	# remaplaf CDO: Largest area fraction remapping
3988	####### ####### ####### or additionally ####### ####### #######
3989	# python remapping
3990	# dis python: Distance-weighted average remapping
3991	# pnn python: Nearest neighbor remapping
3992
3993	fscr= re-projected files to compute from the scratch
3994	fadd= re-projected files to be added
3995	"""
3996	fname = 'reproject_modexp'
3997
3998	odir = config['ofold'] + '/' + mod + '/' + exp
3999
4000	# NON re-projectable operations
4001	opNOreproj = config['NOreprojops'].split(':')
4002
4003	# Need to pass to analyze all the data?
4004	inffiles = ['reproj_varcompute.inf', 'all_reproj_computevars.inf']
4005	if fscr:
4006	for inff in inffiles:
4007	if debug: print " removing information file '" + inff + "' ..."
4008	ins = 'rm ' + odir + '/' + inff + ' >& /dev/null'
4009	sout = sub.call(ins, shell=True)
4010
4011	objf = open(odir+'/all_reproj_computevars.inf','w')
4012	objf.write("## Computation of reprojected variables \n")
4013	objf.close()
4014
4015	varcompf = odir + '/reproj_varcompute.inf'
4016	if fadd:
4017	sub.call('rm ' + varcompf +' >& /dev/null', shell=True)
4018
4019	# CF dims-vardims pairs:
4020	CFvardims = ['long@lon', 'lat@lat', 'pres@pres', 'time@time']
4021
4022	# Dictionary with the operations with surnames for the operated variable
4023	opersurnames = {}
4024	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
4025	for opsr in opsur.keys():
4026	opn = opsr.split('_')[1]
4027	vls = opsur[opsr].split(':')
4028	opersurnames[opn] = vls
4029
4030	if not os.path.isfile(varcompf):
4031	# Getting original computing information
4032	origvarcompf = odir + '/varcompute.inf'
4033	origfiles,origtestfiles,origallcompvar, Nvar = read_varcomp_file(origvarcompf)
4034
4035	Svarcompute = ''
4036	allcompvar = dict(origallcompvar)
4037	ivop = 0
4038	for vnop in origallcompvar.keys():
4039	vn = vnop.split('_')[0]
4040	op = vnop.split('_')[1]
4041	values = allcompvar[vnop]
4042	values[0] = 'reproj-' + values[0]
4043	allcompvar[vnop] = values
4044
4045	# Building strings from list values
4046	model = values[1]
4047	diag = values[2]
4048	if model is not None:
4049	if type(model) == type(list([1, 2])):
4050	Smodel = ':'.join(model)
4051	elif type(model) == type('A'):
4052	Smodel = model
4053	else:
4054	Smodel = 'None'
4055	if diag is not None:
4056	if type(diag) == type(list([1, 2])):
4057	Sdiag = ':'.join(diag)
4058	elif type(diag) == type('A'):
4059	Sdiag = diag
4060	else:
4061	Sdiag = 'None'
4062
4063	Svc = vn + '\|' + op + '\|' + values[0] + '\|' + Smodel + '\|' + Sdiag + \
4064	'\|' + values[3]
4065	if ivop == 0:
4066	Svarcompute = Svc
4067	else:
4068	Svarcompute = Svarcompute + ',' + Svc
4069	ivop = ivop + 1
4070
4071	origstrfiles = gen.dictKeysVals_stringList(origfiles)
4072	origstrtestfiles = gen.dictKeysVals_stringList(origtestfiles)
4073
4074	# Outwritting the varcompute to avoid next time (if it is not filescratch!)
4075	objf = open(odir + '/reproj_varcompute.inf', 'w')
4076	objf.write('files: ' + origstrfiles + '\n')
4077	objf.write('testfiles: ' + origstrtestfiles + '\n')
4078	objf.write('varcompute: ' + Svarcompute + '\n')
4079	objf.write('itotv: ' + str(ivop) + '\n')
4080	objf.close()
4081	else:
4082	if dbg:
4083	print warnmsg
4084	print ' ' + main + ": getting variables to compute already from file !!"
4085	origfiles, origtestfiles, allcompvar, Nvar = read_varcomp_file(varcompf)
4086
4087	# End of avoiding to repeat all the experiment search
4088
4089	print " For experiment '" + exp + "' is required to re-project:", Nvar, \
4090	"variables"
4091
4092	if dbg:
4093	print 'Variables to re-project _______'
4094	gen.printing_dictionary(allcompvar)
4095
4096	# Re-projection variables
4097	print " Reprojecting variables ..."
4098	os.chdir(odir)
4099
4100	# Keeping track of reprojection
4101	trckobjf = open(odir + '/all_reproj_computevars.inf','a')
4102
4103	for vnop in allcompvar.keys():
4104	vn = vnop.split('_')[0]
4105	op = vnop.split('_')[1]
4106	values = allcompvar[vnop]
4107	VnOpS = varnoper(vn, op, opersurnames)
4108
4109	# Only are reprojected that files which do contain 'lon' and 'lat',
4110	# otherways, it is only re-projected the original file
4111	inNO, iopNO, iop = gen.list_coincidences(opNOreproj,op.split('+'))
4112
4113	reprjop = gen.dictionary_key_list(REPRJops,vn).split('_')[1]
4114	if reprjop is None:
4115	print errmsg
4116	print ' ' + fname + ": no reprojecting operator found for " + \
4117	" variable '" + vn + "' !!"
4118	print ' available values _______'
4119	gen.printing_dictionary(REPRJops)
4120	quit(-1)
4121
4122	# Re-projecting also the 'original' (at least with 'pinterp') file for varsdif
4123	if op.find('pinterp') != -1:
4124	ifileorig = vn +'_'+ values[0].replace('reproj-','') + 'p_pinterp.nc'
4125	ofileorig = vn +'_'+ values[0] + 'p_pinterp.nc'
4126	ofilen = vn +'_'+ values[0] + 'p_pinterp.nc'
4127	ifileop = vn +'_'+ values[0].replace('reproj-','') + 'p_' + \
4128	op.replace('+','_') + '.nc'
4129	else:
4130	ifileorig = vn +'_'+ values[0].replace('reproj-','') + '.nc'
4131	ofileorig = vn +'_'+ values[0] + '.nc'
4132	ifileop = vn +'_'+ values[0].replace('reproj-','') + '_' + \
4133	op.replace('+','_') + '.nc'
4134
4135	ofileop = vn + '_' + values[0] + '_' + op.replace('+','_') + '.nc'
4136
4137	if fscr:
4138	sout = sub.call('rm ' + odir+'/'+ofileorig + ' >& /dev/null', shell=True)
4139	sout = sub.call('rm ' + odir+'/'+ofileop + ' >& /dev/null', shell=True)
4140
4141	# List of files to re-project (first without operation)
4142	ifiles = [odir+'/'+ifileorig, odir+'/'+ifileop]
4143	# List of output files after re-projection (first without operation)
4144	ofiles = [odir+'/'+ofileorig, odir+'/'+ofileop]
4145
4146	for iif in range(2):
4147	ifile = ifiles[iif]
4148	ofile = ofiles[iif]
4149
4150	# Only projecting original file-variable if there are not 'lon' and 'lat'
4151	doreproj = True
4152	if iif == 1 and len(inNO) > 0:
4153	doreproj = False
4154	if dbg:
4155	print warnmsg
4156	print ' ' + fname + ": operation '" + op + "' can not be " + \
4157	"reprojected due to the lack of 'lon' or 'lat' skipping it !!"
4158	print " NOTE: difference can only be computed as:"
4159	print " - 1st: compute variable's differences between models"
4160	print " - 2nd: Perform the operations"
4161
4162	# Building up resources to compute re-projected statistics
4163	if dbg:
4164	print ' '+fname+': computing from the original reprojected file'
4165	ModI = ModelInf(mod, mod, 'lon', 'lat', 'pres', 'time', 'depth', \
4166	'lon', 'lat', 'pres', 'time', 'depth', None)
4167	if type(values) == type([1]):
4168	fileh = values[0]
4169	else:
4170	fileh = values
4171
4172	if type(values) == type([1]):
4173	fileh = values[0]
4174	else:
4175	fileh = values
4176
4177	VarI = VariableInf(vn, values[0], values[2], values[3])
4178	usefs = {values[0]: ofileorig}
4179
4180	compute_statistics(ModI, config, config['ifold'], usefs, \
4181	config['ofold']+'/'+mod+'/'+exp,VarI,pinterp_var(op), op, fscr, dbg)
4182
4183	if not os.path.isfile(ofile) and doreproj:
4184	if debug and ofile == ofiles[0]:
4185	print " reprojecting '" + vn + "' '" + op + "' into '" + \
4186	ofile + "'"
4187
4188	# Reprojecting...
4189	if reprjop[0:6] == 'remap':
4190	ins = config['cdoHOME'] + '/cdo ' + reprjop + ',' + \
4191	config['RefProjfile'] + ' ' + ifile + ' ' + ofile
4192	try:
4193	with gen.Capturing() as output:
4194	sout = sub.call(ins, shell=True)
4195	except:
4196	print errmsg
4197	print ' ' + fname + ': $ ' + ins
4198	for s1out in output: print s1out
4199	print sout
4200	quit(-1)
4201	if sout != 0:
4202	print errmsg
4203	print ' ' + fname + ': $ ' + ins
4204	sub.call(ins, shell=True)
4205	quit(-1)
4206	else:
4207	# Name of the variable to reproject
4208	if iif == 0:
4209	reprjvn = vn
4210	else:
4211	reprjvn = VnOpS
4212
4213	reprojvalues = 'lon,lat,' + config['RefProjfile'] + ',lon,lat,'+ \
4214	reprjop + ',' + ':'.join(CFvardims)
4215	ins= 'ncvar.reproject('+reprojvalues + ', ' + ifile + ', '+ \
4216	reprjvn + ')'
4217	try:
4218	with gen.Capturing() as output:
4219	sout=ncvar.reproject(reprojvalues, ifile, reprjvn)
4220	except:
4221	print errmsg
4222	print ' ' + fname + ': ' + ins
4223	for s1out in output: print s1out
4224	quit(-1)
4225
4226	sub.call('mv reproject.nc ' + ofile, shell=True)
4227
4228	if debug:
4229	for s1out in output: print s1out
4230	print sout
4231
4232	# Keeping track of re-projection
4233	trckobjf.write('\n')
4234	if iif == 0:
4235	trckobjf.write('# ' + vn + '\n')
4236	else:
4237	trckobjf.write('# ' + vn + ' ' + op + '\n')
4238
4239	trckobjf.write(ins + '\n')
4240
4241	if reprjop[0:6] == 'remap':
4242	# CFing the reprojected file
4243	try:
4244	with gen.Capturing() as output:
4245	ncvar.CDO_toCF(ofile)
4246	except:
4247	print errmsg
4248	print 'ncvar.CDO_toCF(' + ofile + ')'
4249	for s1out in output: print s1out
4250	# Removing file in order to make sure that it will be redone
4251	print ' ' + fname + " removing file '" + ofile + "' ..."
4252	sub.call('rm ' + ofile + ' >& /dev/null', shell=True)
4253	quit(-1)
4254
4255	if debug:
4256	for s1out in output: print s1out
4257
4258	trckobjf.close()
4259
4260	if debug: print " End of '" + fname + "' "
4261
4262	return
4263
4264	def allmodexps_listconstruct(config, modexps, odir, debug):
4265	""" Function to create the list of all model-experiments to draw
4266	config= Configuration of the experiment
4267	modexps= list with the models-experiments pairs to use
4268	odir= output differences folder
4269	"""
4270	fname='allmodexps_listconstruct'
4271
4272	allplts = gen.get_specdictionary_HMT(config, H='PLOTALLMODEXP_')
4273
4274	if debug:
4275	if allplts is not None:
4276	print " 'all mod-exps' plots ________"
4277	gen.printing_dictionary(allplts)
4278
4279	# Getting Model-Reference Projection
4280	ModComProj = cnf['RefProj'].split(':')
4281
4282	allmodexp = {}
4283	Nallmodexp = 0
4284	for modexp in modexps:
4285	if debug:
4286	print " " + fname + ": '" + modexp + "' ..."
4287	mod = modexp.split('/')[0]
4288	exp = modexp.split('/')[1]
4289
4290	# Getting variable characteristics from each model/experiment
4291	idir = config['ofold'] + '/' + modexp
4292
4293	reproj = not gen.searchInlist(ModComProj,mod)
4294
4295	fvarcomp = 'varcompute.inf'
4296	if reproj: fvc = 'reproj_' + fvarcomp
4297	else: fvc = fvarcomp
4298
4299	files, testfiles, allcompvar, Nvar= read_varcomp_file(idir + '/' + fvc)
4300
4301	if allplts is not None:
4302	# Getting all model-experiments plots characteristics
4303
4304	for allplt in allplts.keys():
4305	pltn = allplt.split('_')[1]
4306	if debug and modexp == modexps[0]:
4307	print ' ' + fname + ": plot '" + pltn + "' "
4308	varops = allplts[allplt].split(':')
4309	for varop in varops:
4310	vn = varop.split('\|')[0]
4311	op = varop.split('\|')[1]
4312	vnop = vn + '_' + op
4313	if debug and modexp == modexps[0]:
4314	print ' ' + fname + ": varop '" + vnop + "' "
4315	if not allcompvar.has_key(vnop):
4316	print warnmsg
4317	print ' ' + fname + ": variable+operation '" + vnop + \
4318	"' in '" + modexp + "' was not computed !!"
4319	print ' skipping it!'
4320	print ' computed values:', allcompvar.keys()
4321	break
4322
4323	vals = allcompvar[vnop]
4324
4325	headerf = vals[0]
4326	if modexp == modexps[0]:
4327	allmodexp[vnop] = [modexp,headerf]
4328	Nallmodexp = Nallmodexp + 1
4329	else:
4330	prevals = allmodexp[vnop]
4331	prevals = prevals + [modexp,headerf]
4332	allmodexp[vnop] = prevals
4333
4334	else:
4335	Nallmodexp = 0
4336
4337	if allplts is not None:
4338	Sallmodexps = gen.dictKeysVals_stringList(allmodexp)
4339	else:
4340	Sallmodexps = 'none'
4341
4342	# Outwritting the all model-experiment plots file to avoid next time
4343	objf = open(odir + '/allmodexp.inf', 'w')
4344	objf.write('allmodexps: ' + Sallmodexps + '\n')
4345	objf.write('Nallmodexp: ' + str(Nallmodexp) + '\n')
4346	objf.close()
4347
4348	if debug: print " End of '" + fname + "' "
4349
4350	return allmodexp, Nallmodexp
4351
4352	def allmodexps_read(filen):
4353	""" Function to read the file with the information about the all model-experiment plots
4354	filen= file with the information
4355	"""
4356	fname = 'allmodexps_read'
4357
4358	if not os.path.isfile(filen):
4359	print errormsg
4360	print ' ' + fname + ": all model-experiment file '" + filen + \
4361	"' does not exist !!"
4362	quit(-1)
4363
4364	objf = open(filen, 'r')
4365	for line in objf:
4366	if line[0:1] != '#' and len(line) > 1:
4367	values = line.replace('\n','').split(' ')
4368	if values[0] == 'allmodexps:':
4369	allplts = gen.stringList_dictKeysVals(values[1])
4370	elif values[0] == 'Nallmodexp:':
4371	Nallmodexp = int(values[1])
4372
4373	objf.close()
4374
4375	return allplts, Nallmodexp
4376
4377	def draw_allmodexp_plots(config, allvarcomp, plots, modexp, odir, figscr, debug):
4378	""" Function to draw all model-experiment plots
4379	config= Configuration of the experiment
4380	allvarcomp = dictionary with the file headers for each variable
4381	plots= dictionary with the plots
4382	modexps= list with the names of the model-experiment pairs
4383	odir= output experiment folder
4384	figscr= whether figures should be done from the scratch or not
4385
4386	* Plot as
4387	{[kplot]} = [varn1]\|[op1]#[varn2]\|[op2]#[...[varnN]\|[opN]], ...
4388	[kplot] ___
4389	Nlines: Multiple lines with one line by model-experiment pairs
4390	[varn]
4391	variable
4392	[op]
4393	'+' separated list of operations
4394	in figures with more than 1 variable, use '#' to separate the [varn]\|[op]
4395	"""
4396	fname = 'draw_allmodexp_plots'
4397
4398	os.chdir(odir)
4399
4400	# Folder with the processed files
4401	infold = config['ofold']
4402
4403	# Dictionary with the operations with surnames for the operated variable
4404	opersurnames = {}
4405	opsur = gen.get_specdictionary_HMT(config, H='opsur_',M='',T='')
4406	for opsr in opsur.keys():
4407	opn = opsr.split('_')[1]
4408	vls = opsur[opsr].split(':')
4409	opersurnames[opn] = vls
4410
4411	# time values
4412	# Units time for the plots
4413	rd = config['CFreftime']
4414	tunits = config['CFunitstime'] + '!since!' + rd[0:4] + '-' + rd[4:6] + '-' + \
4415	rd[6:8] + '!' + rd[8:10] + ':' + rd[10:12] + ':' + rd[12:14]
4416	# time ticks kind
4417	tkind = config['timekind']
4418	# time ticks format
4419	tfmt = config['timefmt']
4420	# time axis label
4421	tlab = config['timelabel']
4422	timevals = [tunits, tkind, tfmt, tlab]
4423
4424	# Dictionary of plot specificities
4425	specplotkeyn = 'specificvarplot'
4426	plotspecifics = {}
4427	if config.has_key(specplotkeyn):
4428	# [minval]: minimum value
4429	# [maxval]: minimum value
4430	# [colorbar]: name of the colorbar (from matplotlib) to use
4431	# [cntformat]: format of the contour labels
4432	# [colorcnt]: color for the countor lines
4433	plotspecs = config[specplotkeyn].split(':')
4434	for pltspc in plotspecs:
4435	pltvls = pltspc.split('\|')
4436	vn = pltvls[0]
4437	op = pltvls[1]
4438	fn = pltvls[2]
4439	plotspecifics[fn + '_' + vn + '_' + op] = pltvls[3:]
4440	if debug:
4441	print 'Specific values for plots _______'
4442	gen.printing_dictionary(plotspecifics)
4443
4444	# Graphical labels and configuration
4445	modgc, expgc = graphical_characteristics(config, debug)
4446
4447	# Kind of figures
4448	kindfigure = config['kindfig']
4449
4450	# Map value
4451	mapvalue = config['mapval']
4452
4453	# pythone scripts HOME
4454	pyHOME = config['pyHOME']
4455
4456	# Title-text of operations
4457	opexplained = {}
4458	optits = config['titleoperations'].split(':')
4459	for optit in optits:
4460	opn = optit.split('\|')[0]
4461	opt = optit.split('\|')[1]
4462	opexplained[opn] = opt
4463	if debug:
4464	print 'Titles for operations _______'
4465	gen.printing_dictionary(opexplained)
4466
4467	for kplot in plots.keys():
4468	varsplt = plots[kplot]
4469	for varplt in varsplt:
4470	if debug:
4471	print " printing '" + kplot + "' var: '" + varplt + "'..."
4472	varops = varplt.split('#')
4473
4474	# CF variables in plot
4475	CFvarsplot = []
4476	# Files in plot
4477	filesplot = []
4478	# Variables in plot within the files
4479	varsplot = []
4480	# Dims in figure
4481	dimsplot = []
4482	# pictoric values in figure
4483	pictplot = []
4484	# Name of the figure
4485	figname = ''
4486	# Title of the figure
4487	titfigure = ''
4488
4489	ivp = 0
4490	for varop in varops:
4491	vn = varop.split('\|')[0]
4492	op = varop.split('\|')[1]
4493
4494	# CF variables in plot
4495	CFvarsplot.append(vn)
4496
4497	if op.find('pinterp') != -1: gP = 'p'
4498	else: gP = ''
4499
4500	vnopS = vn + '_' + op
4501	if not allvarcomp.has_key(vnopS):
4502	print errmsg
4503	print ' ' + fname + ": no file for variable-operation '" + \
4504	vnopS + "' !!"
4505	modexpvals = allvarcomp[vnopS]
4506	Nmodexps = len(modexpvals)
4507
4508	for imodexp in range(0,Nmodexps,2):
4509	modexp = modexpvals[imodexp]
4510	headf = modexpvals[imodexp+1]
4511	if debug:
4512	print ' ' + fname + " model/experiment: '" + modexp + "'"
4513
4514	filen = infold + '/' + modexp + '/' + vn +'_'+ headf+gP + '_' + \
4515	op.replace('+','_') + '.nc'
4516
4517	filesplot.append(filen)
4518	# Do we have processed the given variable?
4519	if not os.path.isfile(filen):
4520	print warnmsg
4521	print " " + fname + ": there is no file for variable '" + \
4522	varop + "' skiping it !!"
4523	break
4524
4525	if imodexp == 0:
4526	# Name of the variable inside the file
4527	vnsur = varnoper(vn, op, opersurnames)
4528	varsplot.append(vnsur)
4529
4530	# Dimensions in file
4531	try:
4532	with gen.Capturing() as output:
4533	dims = ncvar.idims(filen)
4534	except:
4535	print errmsg
4536	print 'ncvar.idims('+filen+')'
4537	for s1out in output: print s1out
4538	quit(-1)
4539
4540	if imodexp == 0:
4541	# Dimensions in plot
4542	dimsplot.append(dims)
4543
4544	# Header of the name of the figure
4545	if ivp == 0:
4546	figname = kplot+'_'+vn+'_'+headf+'_'+ op.replace('+','-')
4547	else:
4548	figname = figname+'-'+vn+'_'+headf+'_'+op.replace('+','-')
4549
4550	# pictoric values for the figure
4551	Sfivaop = kplot + '_' + vn + '_' + op
4552	if plotspecifics.has_key(Sfivaop):
4553	pictvals = plotspecifics[Sfivaop]
4554	else:
4555	Vvals = gen.variables_values(vn)
4556	pictvals = [Vvals[2],Vvals[3],Vvals[6], '%g', 'fixc', 'black']
4557
4558	pictplot.append(pictvals)
4559
4560	ivp = ivp + 1
4561	# End of variable-operation
4562	figname = figname + '.' + kindfigure
4563
4564	# Title of figure
4565	titfigure = 'all!model-experiments!' + vn + optitle(op,opexplained)
4566
4567	draw_plot(kplot, CFvarsplot, filesplot, varsplot, dimsplot, pictplot, \
4568	figname, titfigure, kindfigure, mapvalue, timevals, expgc, odir, \
4569	pyHOME, figscr, debug)
4570
4571	# End of variables-operations
4572
4573	# End of kind of plots
4574
4575	if debug: print " End of '" + fname + "' "
4576
4577	return
4578
4579	def figinf_figname(figname,config):
4580	""" Function to retrieve figure information from its name
4581	figname= name of the figure
4582	config= configuration of `model_graphics.py'
4583	>>> figinf_figname('2linesTime_wss_wrfout_xvar-ymean-tas_wrfout_xvar-ymean.pdf', cnf)
4584	(2, '2linesTime', 'wss-tas', 'xvar-ymean@xvar-ymean')
4585	>>> figinf_figname('Nlines_tas_wrfout_tturb-xmean-tmean.pdf', cnf)
4586	(1, 'Nlines', 'tas', 'tturb-xmean-tmean')
4587	>>> figinf_figname('2lines_wss_diffvar_WRF-micro1_WRF-current_tturb-xmean-last-tas_tturb-xmean-last.pdf',cnf)
4588	(2, '2lines', 'wss-tas', 'tturb-xmean-last@tturb-xmean-last')
4589	"""
4590	fname = 'figinf_figname'
4591	plotsecs = figname.split('_')
4592	plotk = plotsecs[0]
4593
4594	if gen.searchInlist(config['pairfigures'].split(':'), plotk):
4595	Nvars = 2
4596	varn1 = plotsecs[1]
4597	if plotsecs[2] == 'diffvar' or plotsecs[2] == 'diffop':
4598	statn1varn2 = plotsecs[5].split('-')
4599	statn2 = plotsecs[6].split('.')[0]
4600	Lstatn1varn2 = len(statn1varn2)
4601	statn1 = '-'.join(statn1varn2[0:Lstatn1varn2-1])
4602	varn2 = statn1varn2[Lstatn1varn2-1]
4603	else:
4604	statn1varn2 = plotsecs[3].split('-')
4605	statn2 = plotsecs[5].split('.')[0]
4606	Lstatn1varn2 = len(statn1varn2)
4607	statn1 = '-'.join(statn1varn2[0:Lstatn1varn2-1])
4608	varn2 = statn1varn2[Lstatn1varn2-1]
4609	return Nvars, plotk, varn1+'-'+varn2, statn1+'@'+statn2
4610	elif gen.searchInlist(config['Nsourcesfigures'].split(':'), plotk):
4611	Nvars = 1
4612	varn = plotsecs[1]
4613	statn = plotsecs[3].split('.')[0]
4614	return Nvars, plotk, varn, statn
4615	else:
4616	print errmsg
4617	print ' ' + fname + ": kind of figure '" + plotk + "' not ready !!"
4618	print ' ready ones _______'
4619	print " 'pair_figures': figures of pair of variables: ", \
4620	config['pairfigures'].split(':')
4621	print " 'N-sources_figures': figures of pair of variables: ", \
4622	config['Nsourcesfigures'].split(':')
4623	quit(-1)
4624
4625	# Files with information about the configuration of the script
4626	inffiles = ['varcompute.inf', 'all_computevars.inf', 'all_statsvars.inf']
4627
4628	####### #######
4629	## MAIN
4630	#######
4631	if not os.path.isfile('model_graphics.dat'):
4632	print errmsg
4633	print main + ": configuration file 'model_graphics.dat' does not exist!!"
4634	quit(-1)
4635
4636	# Getting configuration from external ASCII file 'model_graphics.dat'
4637	cnf = gen.get_configuration('model_graphics.dat', False)
4638
4639	verify_configuration(cnf, gen.Str_Bool(cnf['debug']))
4640
4641	# scratches
4642	scratch, filescratch, figscratch, diffscratch, figdiffscratch, moddiffscratch, \
4643	figmoddiffscratch, figallmodexpscratch, addfiles, addfigures, adddiffs, \
4644	adddifffigures, addmoddiffs, addmoddifffigures, addallmodexpfigures, dbg = \
4645	scratches(cnf)
4646
4647	if dbg:
4648	print 'Experiment configuration scratches _______'
4649	print ' scratch:', scratch, 'filescratch:', filescratch,'figscratch:', figscratch
4650	print ' diffscratch:', diffscratch, 'figdiffscratch:', figdiffscratch
4651	print ' moddiffscratch:', moddiffscratch, 'figmoddiffscratch:', figmoddiffscratch
4652	print 'Experiment configuration adding _______'
4653	print ' addfiles:', addfiles, 'addfigures:', addfigures
4654	print ' adddiffs:', adddiffs, 'adddifffigures:', adddifffigures
4655	print ' addmoddiffs:', addmoddiffs, 'addmoddifffigures:', addmoddifffigures
4656
4657	# Getting models
4658	mods = cnf['models'].split(':')
4659
4660	# Getting graphical characeristics
4661	modGC, expGC = graphical_characteristics(cnf, dbg)
4662
4663	# Models loop
4664	##
4665
4666	# dictionary with the experiments of each model
4667	modexps = {}
4668
4669	for mod in mods:
4670	print mod
4671	if scratch:
4672	if cnf['ofold'] != cnf['ifold']:
4673	sout = sub.call('rm -rf '+cnf['ofold']+'/'+mod+' > /dev/null', shell=True)
4674	else:
4675	print warnmsg
4676	print ' ' + main + ": input '" + cnf['ifold'] + "' and output '" + \
4677	cnf['ofold'] + "' are the same folder !!"
4678	print " Keeping output folder although it has 'scratch' start"
4679
4680	# Get experiments and headers of model
4681	exps, fheaders = exp_headers(mod,cnf)
4682
4683	modexps[mod] = exps
4684	# Characteristics of the model
4685	Modinf = ncvar.model_characteristics(mod,'None','False')
4686	dnx = Modinf.dimxn
4687	dny = Modinf.dimyn
4688	dnz = Modinf.dimzn
4689	dnt = Modinf.dimtn
4690	dns = Modinf.dimsn
4691	vdnx = Modinf.vardxn
4692	vdny = Modinf.vardyn
4693	vdnz = Modinf.vardzn
4694	vdnt = Modinf.vardtn
4695	vdns = Modinf.vardsn
4696
4697	modgraphv = modGC[mod]
4698	Modinf = ModelInf(mod, mod, dnx, dny, dnz, dnt, dns, vdnx, vdny, vdnz, vdnt, \
4699	vdns, modgraphv.tmod)
4700
4701	if dbg:
4702	print ' model characteristics _______'
4703	gen.printing_class(Modinf)
4704
4705	moddims = dnx + ',' + dny + ',' + dnz + ',' + dnt
4706	modvdims = vdnx + ',' + vdny + ',' + vdnz + ',' + vdnt
4707
4708	# Experiments loop
4709	##
4710	for exp in exps:
4711	print ' ' + exp + '...'
4712
4713	# input folder
4714	iwdir = cnf['ifold'] + '/' + mod + '/' + exp
4715
4716	# Does input folder exist?
4717	if not os.path.isdir(iwdir):
4718	if filescratch*addfiles:
4719	print errmsg
4720	print " " + main + ": folder '" + iwdir + "' does not exist !!"
4721	quit(-1)
4722	else:
4723	print warnmsg
4724	print " " + main + ": folder '" + iwdir + "' does not exist !!"
4725	print " but start of files from scratch 'filescratch':", \
4726	filescratch, "and there is not the add of any file 'addfiles':", \
4727	addfiles
4728	print ' so, keep moving'
4729
4730	owdir = cnf['ofold'] + '/' + mod + '/' + exp
4731	sout = sub.call('mkdir -p ' + owdir, shell=True)
4732
4733	# Need to pass to analyze all the data?
4734	if filescratch:
4735	for inff in inffiles:
4736	if dbg: print " removing information file '" + inff + "' ..."
4737	ins = 'rm ' + owdir + '/' + inff + ' >& /dev/null'
4738	sout = sub.call(ins, shell=True)
4739
4740	objf = open(owdir+'/all_computevars.inf','w')
4741	objf.write("## Computation of variables \n")
4742	objf.close()
4743	objf = open(owdir+'/all_statsvars.inf','w')
4744	objf.write("## Computation of statistics \n")
4745	objf.close()
4746
4747	varcompf = owdir + '/varcompute.inf'
4748	if addfiles:
4749	if dbg:
4750	print ' ' + main + ": adding variables to compute removing " + \
4751	"file '" + varcompf + "' ..."
4752	sub.call('rm ' + varcompf +' >& /dev/null', shell=True)
4753
4754	if not os.path.isfile(varcompf):
4755	# Does input folder has header files?
4756	ih=1
4757	# Dictionary with the list of files for each headers
4758	files = {}
4759	# Dictionary with a file fromor each headers
4760	testfiles = {}
4761
4762	for fh in fheaders:
4763	if filescratch:
4764	ins = 'rm '+ owdir+'/_' + fh + '.nc >& /dev/null'
4765	sout = sub.call(ins, shell=True)
4766	files1h = gen.files_folder(iwdir,fh)
4767	if len(files1h) < 1:
4768	print errmsg
4769	print ' ' + main + ": folder '" + iwdir + "' does not " + \
4770	"contain files '" + fh + "*' !!"
4771	quit(-1)
4772	files[fh] = files1h
4773	testfiles[fh] = files1h[0]
4774
4775	if dbg:
4776	print ' Dictionary of files _______'
4777	gen.printing_dictionary(files)
4778
4779	print ' Dictionary of test-files _______'
4780	gen.printing_dictionary(testfiles)
4781
4782	allcompvar, Nvar = compvars_listconstruct(cnf, Modinf, files, testfiles, \
4783	iwdir, owdir, dbg)
4784	else:
4785	if dbg:
4786	print warnmsg
4787	print ' '+main+": getting variables to compute already from file !!"
4788	files, testfiles, allcompvar, Nvar = read_varcomp_file(varcompf)
4789
4790	# End of avoiding to repeat all the experiment search
4791
4792	print " For experiment '" + exp + "' is required to compute:", Nvar, \
4793	"variables"
4794
4795	if dbg:
4796	print 'Variables to compute _______'
4797	gen.printing_dictionary(allcompvar)
4798
4799	# Computing variables
4800	print " Computing variables ..."
4801	compute_vars(cnf, Modinf, iwdir, owdir, files, allcompvar, filescratch, dbg)
4802
4803	# Figures of variables direct from files
4804	print " Plotting direct figures ..."
4805	dirfigf = owdir + '/directplotsdraw.inf'
4806	if figscratch:
4807	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4808
4809	objf = open(owdir+'/all_figures.inf','w')
4810	objf.write("## Drawing of all figures \n")
4811	objf.close()
4812
4813	if addfigures:
4814	if dbg:
4815	print ' ' + main + ": adding direct figures removing file '" + \
4816	dirfigf + "' ..."
4817	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4818
4819	if not os.path.isfile(dirfigf):
4820	listplots, Nplt = plots_listconstruct(cnf, 'DIRPLT', dirfigf, owdir, dbg)
4821	else:
4822	if dbg:
4823	print warnmsg
4824	print ' ' + main + ": getting plots to draw already from file !!"
4825	listplots, Nplt = read_plot_file(dirfigf)
4826
4827	# End of avoiding to repeat all the plots search
4828
4829	print " For experiment '" + exp + "' is required to plot:", Nplt, "plots"
4830
4831	if dbg:
4832	print 'Plots to draw _______'
4833	gen.printing_dictionary(listplots)
4834
4835	draw_plots(cnf, listplots, mod, exp, owdir, allcompvar, figscratch, dbg)
4836
4837	# end of experiments loop
4838
4839	### ## #
4840	# Experiment differences
4841	## # ## #
4842
4843	# Experiments loop
4844	##
4845	print ' ***** ** * * ** * * ** * ** *****'
4846	print " ** '" + mod + "': Inter experiments differences "
4847	print ' ***** ** * * ** * * ** * ** *****'
4848	print ' experiments:', exps
4849	# There are two kind of differences:
4850	# DIFFop: differences between operations of each given variable.
4851	# Differences are computed directly from the last stage of the operation
4852	# DIFFvar: operations of differences between variables
4853	# First are computed the differences from the initial variable file
4854	# and then operations are made
4855	# NOTE: remember that: meanvar2 - meanvar1 = mean(var2 - var1)
4856	difffiles = ['diffop.inf', 'diffvar.inf']
4857
4858	Nexps = len(exps)
4859	for iexp1 in range(0,Nexps-1):
4860	exp1 = exps[iexp1]
4861	for iexp2 in range(iexp1+1,Nexps):
4862	exp2 = exps[iexp2]
4863	Sexps = exp2 + '-' + exp1
4864	print ' ' + Sexps + '...'
4865	owdir = cnf['ofold'] + '/' + mod + '/' + exp2 + '-' + exp1
4866	sout = sub.call('mkdir -p ' + owdir, shell=True)
4867
4868	# Removing files with list of differences if should be started from scratch or add differences
4869	for fdiff in difffiles:
4870	difff = owdir + '/' + fdiff
4871	if diffscratch:
4872	sub.call('rm -rf' + owdir +' >& /dev/null', shell=True)
4873	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4874	objf = open(owdir+'/all_' + fdiff,'w')
4875	objf.write("## Computation and drawing of differences " + \
4876	"between '" + exp2 + "'-'" + exp1 + "'\n")
4877	objf.close()
4878	difff = owdir + '/all_vardiffstatistics.inf'
4879	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4880	objf = open(owdir+'/all_' + fdiff,'w')
4881	objf.write("## Computation of all differences statistics " + \
4882	"between '" + exp2 + "'-'" + exp1 + "'\n")
4883	objf.close()
4884
4885	if adddiffs:
4886	if dbg:
4887	print ' ' + main + ": adding differences removing file '" + \
4888	difff + "' ..."
4889	sub.call('rm ' + difff +' >& /dev/null', shell=True)
4890
4891	# op and var differences
4892	diffvarcompf = owdir + '/' + difffiles[0]
4893	if not os.path.isfile(diffvarcompf):
4894	alldiffop, alldiffvar,Nopdiffs,Nvardiffs=diffvarop_listconstruct(cnf,\
4895	[mod, mod], [exp1, exp2], [False, False], owdir, dbg)
4896	else:
4897	if dbg:
4898	print warnmsg
4899	print ' ' + main + ": getting 'op' and 'var' differences to " + \
4900	"compute already from file !!"
4901	alldiffop, Nopdiffops = read_diff_file(diffvarcompf)
4902	alldiffvar, Nopdiffvars = read_diff_file(owdir+'/'+difffiles[1])
4903	print 'Nopdiffops:', Nopdiffops,'Nopdiffvars:', Nopdiffvars
4904
4905	# End of avoiding to repeat all the experiment search
4906	print " For experiments '"+exp2+"'-'"+exp1+"' is required to " + \
4907	"compute:", Nvar, "differences"
4908
4909	if dbg:
4910	print 'Differences to compute _______'
4911	gen.printing_dictionary(alldiffop)
4912	gen.printing_dictionary(alldiffvar)
4913
4914	# Computing differences
4915	##
4916	print " Computing operation differences ..."
4917	compute_op_diffs(cnf, alldiffop, owdir, diffscratch, dbg)
4918	print " Computing variable differences ..."
4919	compute_var_diffs(cnf, alldiffvar, owdir, diffscratch, dbg)
4920
4921	# Plotting operation differences
4922	##
4923	print " " + main + ": Plotting operation differences' figures ..."
4924	dirfigf = owdir + '/diffopplotsdraw.inf'
4925	if figscratch:
4926	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4927
4928	objf = open(owdir+'/all_diffopfigures.inf','w')
4929	objf.write("## Drawing of all operation difference figures \n")
4930	objf.close()
4931
4932	if adddifffigures:
4933	if dbg:
4934	print ' ' + main + ": adding differences' figures removing " + \
4935	"file '" + dirfigf + "' ..."
4936
4937	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4938
4939	if not os.path.isfile(dirfigf):
4940	listplots, Nplt = plots_listconstruct(cnf, 'PLOTDIFFOP', dirfigf, \
4941	owdir, dbg)
4942	else:
4943	if dbg:
4944	print warnmsg
4945	print ' ' + main + ": getting plots to draw already from file !!"
4946	listplots, Nplt = read_plot_file(dirfigf)
4947
4948	# End of avoiding to repeat all the plots search
4949
4950	print " For experiment 'operation' differences '" + Sexps + "' is " + \
4951	"required to plot:" , Nplt, "plots"
4952
4953	if dbg:
4954	print ' Plots to draw _______'
4955	gen.printing_dictionary(listplots)
4956
4957	draw_diff_plots(cnf, listplots, owdir, alldiffop, 'diffop', \
4958	figdiffscratch, dbg)
4959
4960	# Plotting variable differences
4961	##
4962	print " " + main + ": Plotting variable differences' figures ..."
4963	dirfigf = owdir + '/diffvarplotsdraw.inf'
4964	if figscratch:
4965	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4966
4967	objf = open(owdir+'/all_diffvarfigures.inf','w')
4968	objf.write("## Drawing of all variables difference figures \n")
4969	objf.close()
4970
4971	if adddifffigures:
4972	if dbg:
4973	print ' '+main+": adding differences' figures removing file '" +\
4974	dirfigf + "' ..."
4975	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
4976
4977	if not os.path.isfile(dirfigf):
4978	listplots, Nplt = plots_listconstruct(cnf, 'PLOTDIFFVAR', dirfigf, \
4979	owdir, dbg)
4980	else:
4981	if dbg:
4982	print warnmsg
4983	print ' ' + main + ": getting plots to draw already from file !!"
4984	listplots, Nplt = read_plot_file(dirfigf)
4985
4986	# End of avoiding to repeat all the plots search
4987
4988	print " For experiment 'variables' differences '" + Sexps + "' is " + \
4989	"required to plot:" , Nplt, "plots"
4990
4991	if dbg:
4992	print ' Plots to draw _______'
4993	gen.printing_dictionary(listplots)
4994
4995	draw_diff_plots(cnf, listplots, owdir, alldiffvar, 'diffvar', \
4996	figdiffscratch, dbg)
4997
4998	# end of mods loop
4999
5000	### ## #
5001	# Model differences
5002	### ## #
5003
5004	# Models already at the reference projection
5005	ModComProj = cnf['RefProj'].split(':')
5006	if dbg:
5007	print 'Models to use as reference for the projection:', ModComProj
5008
5009	# Operators to use for the reprojection of each variable
5010	REPRJvar0 = gen.get_specdictionary_HMT(cnf, H='reprojectvar_')
5011	REPRJvar = {}
5012	for Skey in REPRJvar0.keys():
5013	REPRJvar[Skey] = gen.str_list(REPRJvar0[Skey],':')
5014
5015	if dbg:
5016	print 'Remaping operators to use _______'
5017	gen.printing_dictionary(REPRJvar)
5018
5019	# Getting common base weights for grid point remapping with CDO
5020	frefgridn = 'RefProj.nc'
5021	for mod in mods:
5022	frefgriddes = cnf['ofold'] + '/' + mod + '/' + frefgridn
5023	if moddiffscratch:
5024	sout = sub.call('rm ' + frefgriddes + ' >& /dev/null', shell=True)
5025	if gen.searchInlist(ModComProj,mod) and not os.path.isfile(frefgriddes):
5026	# Looking for a file to use (all files are CF compilant, but we need to
5027	# be sure that at least they have 'lon', 'lat' variables)
5028	exps = modexps[mod]
5029	exp = exps[0]
5030	reprjfdir = cnf['ofold'] + '/' + mod + '/' + exp
5031	reprjfile = None
5032	for reprjop in REPRJvar.keys():
5033	reprjvars = REPRJvar[reprjop]
5034	for reprjvar in reprjvars:
5035	reprjfs= gen.files_folder_HMT(folder=reprjfdir,head=reprjvar+'_',tail='.nc')
5036	for reprjf in reprjfs:
5037	# Getting direct files as [var]_[fhead].nc
5038	if len(reprjf.split('_')) == 2:
5039	reprjfile = reprjfdir + '/' + reprjf
5040	reprjvn = reprjvar
5041	break
5042	if reprjfile is None:
5043	print errmsg
5044	print ' ' + main + ": no proper file to get projection information " + \
5045	"from '" + reprjfdir + "' has been found !!"
5046	quit(-1)
5047
5048	print " Creation of reference projection information file from '" + mod+ "'"
5049	if dbg:
5050	print " using file: '" + reprjfile
5051
5052	# Getting only 1 time-step for disk space issues
5053	try:
5054	with gen.Capturing() as output:
5055	ncvar.DataSetSection('time,0,1,1', reprjfile)
5056	except:
5057	print errmsg
5058	print 'ncvar.DataSetSection(time,0,1,1, ' + reprjfile + ')'
5059	for s1out in output: print s1out
5060	quit(-1)
5061
5062	of = reprjfile.split('.')[0] + '_time_B0-E1-I1.nc'
5063
5064	# Cleaning variables
5065	try:
5066	with gen.Capturing() as output:
5067	ncvar.selvar('lon@lon,lat@lat,time@time', of, reprjvn)
5068	except:
5069	print errmsg
5070	print 'ncvar.selvar(lon@lon,lat@lat, ' + of + ', ' + reprjvn + ')'
5071	for s1out in output: print s1out
5072	quit(-1)
5073	sout = sub.call('mv selvar_new.nc ' + frefgriddes, shell=True)
5074	sout = sub.call('rm ' + of + ' >& /dev/null', shell=True)
5075
5076	cnf['RefProjfile'] = frefgriddes
5077
5078	if dbg:
5079	for s1out in output: print s1out
5080	break
5081
5082	elif gen.searchInlist(ModComProj,mod) and os.path.isfile(frefgriddes):
5083	# Including reprojecting reference file in configuration dictionary
5084	cnf['RefProjfile'] = frefgriddes
5085
5086	Nmods = len(mods)
5087	for imod1 in range(0,Nmods-1):
5088	mod1 = mods[imod1]
5089	exps1 = modexps[mod1]
5090
5091	for imod2 in range(imod1+1,Nmods):
5092	mod2 = mods[imod2]
5093	exps2 = modexps[mod2]
5094
5095	Smods = mod2 + '-' + mod1
5096	print ' ***** ** * * ** * * ** * ** *****'
5097	print " ** '" + Smods + "': Inter models differences "
5098	print ' ***** ** * * ** * * ** * ** *****'
5099	print ' experiments mod1:', exps1
5100	print ' experiments mod2:', exps2
5101
5102	# Experiments loop
5103	##
5104	difffiles = ['diffop.inf', 'diffvar.inf']
5105
5106	Nexps1 = len(exps1)
5107	Nexps2 = len(exps2)
5108	for iexp1 in range(0,Nexps1):
5109	exp1 = exps1[iexp1]
5110	for iexp2 in range(0,Nexps2):
5111	exp2 = exps2[iexp2]
5112	Sexps = exp2 + '-' + exp1
5113	print ' ' + Sexps + '...'
5114	owdir = cnf['ofold'] + '/' + Smods + '/' + Sexps
5115	sout = sub.call('mkdir -p ' + owdir, shell=True)
5116
5117	# Getting the right projection in order to perform differences
5118	difmods = [mod1, mod2]
5119	difexps = [exp1, exp2]
5120	difreproj = [False, False]
5121
5122	for imod in range(2):
5123	mod = difmods[imod]
5124	exp = difexps[imod]
5125	if not gen.searchInlist(ModComProj,mod):
5126	print " Projecting files of model '" + mod + "' exp: '" + \
5127	exp + "' with file '" + cnf['RefProjfile'] + "'"
5128	reproject_modexp(mod, exp, cnf, REPRJvar, moddiffscratch, \
5129	addmoddiffs, dbg)
5130	difreproj[imod] = True
5131
5132	# Removing files with list of differences if should be started from
5133	# scratch or add differences
5134	for fdiff in difffiles:
5135	difff = owdir + '/' + fdiff
5136	if moddiffscratch:
5137	sub.call('rm -rf' + owdir +' >& /dev/null', shell=True)
5138	sub.call('rm ' + difff +' >& /dev/null', shell=True)
5139	objf = open(owdir+'/all_' + fdiff,'w')
5140	objf.write("## Computation and drawing of differences " + \
5141	"between '" + mod2+'@'+exp2 + "'-'" + mod1+'@'+exp1 + "'\n")
5142	objf.close()
5143	difff = owdir + '/all_vardiffstatistics.inf'
5144	sub.call('rm ' + difff +' >& /dev/null', shell=True)
5145	objf = open(owdir+'/all_' + fdiff,'w')
5146	objf.write("## Computation of all differences statistics " + \
5147	"between '" + mod2+'@'+exp2 + "'-'" + mod1+'@'+exp1 + "'\n")
5148	objf.close()
5149
5150	if addmoddiffs:
5151	if dbg:
5152	print ' ' + main + ": adding model differences " + \
5153	"removing file '" + difff + "' ..."
5154	sub.call('rm ' + difff +' >& /dev/null', shell=True)
5155
5156	# op and var differences
5157	diffvarcompf = owdir + '/' + difffiles[0]
5158	if not os.path.isfile(owdir+'/'+difffiles[0]) or \
5159	not os.path.isfile(owdir+'/'+difffiles[1]):
5160	alldiffop, alldiffvar, Nopdiffs, Nvardiffs = \
5161	diffvarop_listconstruct(cnf, difmods, difexps, difreproj,owdir,\
5162	dbg)
5163	else:
5164	if dbg:
5165	print warnmsg
5166	print ' '+main+": getting 'op' and 'var' differences to " + \
5167	"compute already from file !!"
5168	alldiffop, Nopdiffops = read_diff_file(diffvarcompf)
5169	alldiffvar, Nopdiffvars = read_diff_file(owdir+'/'+difffiles[1])
5170
5171	# End of avoiding to repeat all the experiment search
5172
5173	print " For experiments '" + mod2+'@'+exp2 + "'-'" + mod+'@'+exp1+\
5174	"' is required to compute:", Nvar, "differences"
5175
5176	if dbg:
5177	print ' op differences to compute _______'
5178	gen.printing_dictionary(alldiffop)
5179	print ' var differences to compute _______'
5180	gen.printing_dictionary(alldiffvar)
5181
5182	# Computing differences
5183	##
5184	print " Computing operation differences ..."
5185	compute_op_diffs(cnf, alldiffop, owdir, moddiffscratch, dbg)
5186	print " Computing variable differences ..."
5187	compute_var_diffs(cnf, alldiffvar, owdir, moddiffscratch, dbg)
5188
5189	# Plotting operation differences
5190	##
5191	print " " + main + ": Plotting operation differences' figures ..."
5192	dirfigf = owdir + '/diffopplotsdraw.inf'
5193	if figmoddiffscratch:
5194	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5195
5196	objf = open(owdir+'/all_diffopfigures.inf','w')
5197	objf.write("## Drawing of all operation difference figures \n")
5198	objf.close()
5199
5200	if addmoddifffigures:
5201	if dbg:
5202	print ' ' + main + ": adding model differences' figures " + \
5203	"removing file '" + dirfigf + "' ..."
5204	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5205
5206	if not os.path.isfile(dirfigf):
5207	listplots, Nplt = plots_listconstruct(cnf, 'PLOTDIFFOP', dirfigf,\
5208	owdir, dbg)
5209	else:
5210	if dbg:
5211	print warnmsg
5212	print ' '+main+": getting plots to draw already from file !!"
5213	listplots, Nplt = read_plot_file(dirfigf)
5214
5215	# End of avoiding to repeat all the plots search
5216
5217	print " For experiment 'operation' differences '" + Sexps + "' is "+\
5218	"required to plot:" , Nplt, "plots"
5219
5220	if dbg:
5221	print 'Plots to draw _______'
5222	gen.printing_dictionary(listplots)
5223
5224	draw_diff_plots(cnf, listplots, owdir, alldiffop, 'diffop', \
5225	figmoddiffscratch, dbg)
5226
5227	# Plotting variable differences
5228	##
5229	print " " + main + ": Plotting variable differences' figures ..."
5230	dirfigf = owdir + '/diffvarplotsdraw.inf'
5231	if figscratch:
5232	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5233
5234	objf = open(owdir+'/all_diffvarfigures.inf','w')
5235	objf.write("## Drawing of all variables difference figures \n")
5236	objf.close()
5237
5238	if adddifffigures:
5239	if dbg:
5240	print ' ' + main + ": adding differences's figures " + \
5241	"removing file '" + dirfigf + "' ..."
5242	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5243
5244	if not os.path.isfile(dirfigf):
5245	listplots, Nplt = plots_listconstruct(cnf,'PLOTDIFFVAR',dirfigf, \
5246	owdir, dbg)
5247	else:
5248	if dbg:
5249	print warnmsg
5250	print ' '+main+": getting plots to draw already from file !!"
5251	listplots, Nplt = read_plot_file(dirfigf)
5252
5253	# End of avoiding to repeat all the plots search
5254
5255	print " For experiment 'variables' differences '" + Sexps+ "' is "+ \
5256	"required to plot:" , Nplt, "plots"
5257
5258	if dbg:
5259	print ' Plots to draw _______'
5260	gen.printing_dictionary(listplots)
5261
5262	draw_diff_plots(cnf, listplots, owdir, alldiffvar, 'diffvar', \
5263	figdiffscratch, dbg)
5264
5265	# end of exp2 loop
5266	# end of exp1 loop
5267	# end of mod2 loop
5268	# end of mod1 loop
5269
5270	### ## #
5271	# All model/exps linear plots
5272	### ## #
5273
5274	allmodexps = []
5275	for mod in mods:
5276	for exp in modexps[mod]:
5277	allmodexps.append(mod + '/' + exp)
5278
5279	print ' ***** ** * * ** * * ** * ** *****'
5280	print ' ** Multi models, multi experiments plots '
5281	print ' ***** ** * * ** * * ** * ** *****'
5282	print ' Model/experiments:', allmodexps
5283
5284	owdir = cnf['ofold'] + '/allmodexps'
5285	sout = sub.call('mkdir -p ' + owdir, shell=True)
5286
5287	# Plotting all model-experiments lines
5288	##
5289	print " " + main + ": Plotting all model-experiments variable figures ..."
5290	allf = owdir + '/allmodexp.inf'
5291	dirfigf = owdir + '/allmodexpfigures.inf'
5292	allfigf = owdir+'/all_figures.inf'
5293	if figallmodexpscratch:
5294	sout = sub.call('rm ' + allf + ' >& /dev/null', shell=True)
5295	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5296	sout = sub.call('rm ' + allfigf + ' >& /dev/null', shell=True)
5297
5298	objf = open(allfigf,'w')
5299	objf.write("## Drawing of all variables figures for all model-experiments\n")
5300	objf.close()
5301
5302	if addallmodexpfigures:
5303	if dbg:
5304	print ' ' + main + ": adding model-experiment differences removing " + \
5305	" file '" + allfigf + "' ..."
5306	print ' ' + main + ": adding model-experiment differences' figures " + \
5307	"removing file '" + dirfigf + "' ..."
5308	sout = sub.call('rm ' + allfigf + ' >& /dev/null', shell=True)
5309	sout = sub.call('rm ' + dirfigf + ' >& /dev/null', shell=True)
5310
5311	if not os.path.isfile(dirfigf):
5312	listplots, Nplt = plots_listconstruct(cnf, 'PLOTALLMODEXP', dirfigf, owdir, dbg)
5313	else:
5314	if dbg:
5315	print warnmsg
5316	print ' ' + main + ": getting plots to draw already from file !!"
5317	listplots, Nplt = read_plot_file(dirfigf)
5318
5319	# End of avoiding to repeat all the plots search
5320
5321	print " Is required to plot:", Nplt, " all model-experiment plots"
5322
5323	if dbg:
5324	print ' Plots to draw _______'
5325	gen.printing_dictionary(listplots)
5326
5327	if not os.path.isfile(allf):
5328	# Constructing the information of modexp / variable
5329	allmodexpvar, Nallmodexpvar = allmodexps_listconstruct(cnf, allmodexps, owdir, dbg)
5330	else:
5331	allmodexpvar, Nallmodexpvar = allmodexps_read(allf)
5332
5333	draw_allmodexp_plots(cnf, allmodexpvar, listplots, modexps, owdir, figallmodexpscratch, dbg)
5334
5335	print main + ': all files and figures have been properly done !!'
5336
5337	# Creation of a pdf with all figures
5338	##
5339	###
5340	owdir = cnf['ofold']
5341	os.chdir(owdir)
5342
5343	texout ='model_graphics-images'
5344	print ' ***** ** * * ** * * ** * ** *****'
5345	print ' ** Creation of a pdf with all figures '
5346	print ' ***** ** * * ** * * ** * ** *****'
5347	print ' ' + owdir + '/' + texout + '.pdf'
5348
5349	if dbg:
5350	print main + ": generation of a pdf file '" + owdir + '/' + texout + \
5351	"' with all the figures..."
5352
5353	otexf = open(owdir + '/' + texout + '.tex', 'w')
5354	otexf.write('\\documentclass{article}\n')
5355	otexf.write('\\usepackage{graphicx}\n')
5356	otexf.write('\\usepackage[colorlinks=true,urlcolor=blue]{hyperref}\n')
5357	otexf.write('\\textheight=23cm\n')
5358	otexf.write('\\textwidth=18cm\n')
5359	otexf.write('\\oddsidemargin=-1cm\n')
5360	otexf.write('\\evensidemargin=-1cm\n')
5361	otexf.write('\\topmargin=-1cm\n')
5362
5363	otexf.write('\n')
5364	otexf.write('\\begin{document}\n')
5365	otexf.write('\n')
5366	otexf.write('\\def\\fontBloc{\\fontfamily{\\sfdefault}\\large\\bfseries}\n')
5367	otexf.write('\\def\\linia{\\setlength{\\unitlength}{1mm}\n')
5368	otexf.write('\\line(1,0){80}}\n')
5369	otexf.write('\\newcommand{\\modexp}[1]{\\clearpage\n')
5370	otexf.write('\\noindent\\linia\n')
5371	otexf.write('\\section{#1}\n')
5372	otexf.write('\\linia}\n')
5373	otexf.write('\n')
5374
5375	otexf.write('\\title{model\_graphics: ' + gen.latex_text(owdir) + '}\n')
5376	otexf.write('\\maketitle\n')
5377	otexf.write('\n')
5378	otexf.write('\\tableofcontents\n')
5379	otexf.write('\\listoffigures\n')
5380	otexf.write('\\clearpage\n')
5381
5382	Nmods = len(mods)
5383	lmodexps = []
5384	ldiffmodexps = []
5385	# Figures from mod/exp pairs
5386	for mod in mods:
5387	exps = modexps[mod]
5388	for exp in exps:
5389	print ' direct plots:', mod, exp
5390	lmodexps.append(mod+'/'+exp)
5391	owdirme = owdir + '/' + mod + '/' + exp
5392	otexf.write('% ' + mod + ' ' + exp + '\n')
5393	otexf.write('\\modexp{' + gen.latex_text(mod+' '+exp) + '}\n')
5394	listplots, Nplt = read_plot_file(owdirme + '/directplotsdraw.inf')
5395	for plot in listplots.keys():
5396	plots = listplots[plot]
5397	ops = []
5398	for plt in plots:
5399	opn = plt.split('#')[0].split('\|')[1]
5400	if not gen.searchInlist(ops,opn): ops.append(opn)
5401	for op in ops:
5402	opS = op.replace('+','_')
5403	modexpimg = gen.files_folder_HMT(owdirme, head=plot, middle=opS, \
5404	tail=cnf['kindfig'])
5405	Nimages = len(modexpimg)
5406	if Nimages > 0:
5407	for img in range(Nimages): modexpimg[img] = owdirme + '/' + \
5408	modexpimg[img]
5409	caption = mod + ' ' + exp + ' ' + plot + ' ' + op
5410	flab = 'fig:' + caption.replace(' ', '_')
5411	try:
5412	with gen.Capturing() as output:
5413	gen.latex_fig_array(modexpimg, otexf, \
5414	gen.latex_text(caption), flab, refsize=0.5, dist='sqr',\
5415	dorest='center')
5416	except:
5417	print errmsg
5418	print 'gen.latex_fig_array(',modexpimg,', '+otexf+', '+ \
5419	'gen.latex_text(' + caption + '), ' + flab + \
5420	", refsize=0.5, dist='sqr', dorest='center')"
5421	for s1out in output: print s1out
5422	quit(-1)
5423	otexf.write('%\\clearpage\n')
5424
5425	# differences between experiments
5426	Nexps = len(exps)
5427	for iexp1 in range(0,Nexps-1):
5428	exp1 = exps[iexp1]
5429	for iexp2 in range(iexp1+1,Nexps):
5430	exp2 = exps[iexp2]
5431	Sexps = exp2 + '-' + exp1
5432	owdirme = owdir + '/' + mod + '/' + Sexps
5433	print ' diffs:', mod + '/' + Sexps + ' ...'
5434	ldiffmodexps.append(mod + '/' + Sexps)
5435	for diffn in ['op', 'var']:
5436	fileplotinf = owdirme + '/diff' + diffn + 'plotsdraw.inf'
5437	if os.path.isfile(fileplotinf):
5438	otexf.write('% ' + mod + ' ' + Sexps + ' diff' + diffn + '\n')
5439	otexf.write('\\modexp{' + gen.latex_text(mod+' '+Sexps+' diff'+ \
5440	diffn)+ '}\n')
5441	listplots, Nplt = read_plot_file(fileplotinf)
5442	for plot in listplots.keys():
5443	plots = listplots[plot]
5444	ops = []
5445	for plt in plots:
5446	opn = plt.split('#')[0].split('\|')[1]
5447	if not gen.searchInlist(ops,opn): ops.append(opn)
5448	for op in ops:
5449	opS = op.replace('+','-')
5450	modexpimg = gen.files_folder_HMT(owdirme, head=plot, \
5451	middle=opS, tail=cnf['kindfig'])
5452	Nimages = len(modexpimg)
5453	if Nimages > 0:
5454	for img in range(Nimages): modexpimg[img] = owdirme +\
5455	'/' + modexpimg[img]
5456	caption = mod + ' ' + Sexps + ' ' + plot + ' ' + op +\
5457	' diff' + diffn
5458	flab = 'fig:' + caption.replace(' ', '_')
5459	try:
5460	with gen.Capturing() as output:
5461	gen.latex_fig_array(modexpimg, otexf, \
5462	gen.latex_text(caption), flab, refsize=0.5,\
5463	dist='sqr', dorest='center')
5464	except:
5465	print errmsg
5466	print 'gen.latex_fig_array(',modexpimg,', '+ \
5467	otexf+', gen.latex_text(' + caption + '), ' + \
5468	flab + ", refsize=0.5, dist='sqr', " + \
5469	"dorest='center')"
5470	for s1out in output: print s1out
5471	quit(-1)
5472
5473	otexf.write('%\\clearpage\n')
5474	print ' mods-exps diffs ...'
5475	for imod1 in range(0,Nmods-1):
5476	mod1 = mods[imod1]
5477	exps1 = modexps[mod1]
5478	for imod2 in range(imod1+1,Nmods):
5479	mod2 = mods[imod2]
5480	exps2 = modexps[mod2]
5481
5482	Smods = mod2 + '-' + mod1
5483	Nexps1 = len(exps1)
5484	Nexps2 = len(exps2)
5485	for iexp1 in range(0,Nexps1):
5486	exp1 = exps1[iexp1]
5487	for iexp2 in range(0,Nexps2):
5488	exp2 = exps2[iexp2]
5489	Sexps = exp2 + '-' + exp1
5490	print ' ' + Smods + ' ' + Sexps + ' ...'
5491	ldiffmodexps.append(Smods + '/' + Sexps)
5492	owdirme = owdir + '/' + Smods + '/' + Sexps
5493	for diffn in ['op', 'var']:
5494	fileplotinf = owdirme + '/diff' + diffn + 'plotsdraw.inf'
5495	if os.path.isfile(fileplotinf):
5496	otexf.write('% '+Smods + ' ' + Sexps + ' diff'+diffn+'\n')
5497	otexf.write('\\modexp{' + gen.latex_text(Smods+' '+Sexps+\
5498	' diff'+diffn) + '}\n')
5499	listplots, Nplt = read_plot_file(fileplotinf)
5500	for plot in listplots.keys():
5501	plots = listplots[plot]
5502	ops = []
5503	for plt in plots:
5504	opn = plt.split('#')[0].split('\|')[1]
5505	if not gen.searchInlist(ops,opn): ops.append(opn)
5506	for op in ops:
5507	opS = op.replace('+','-')
5508	modexpimg = gen.files_folder_HMT(owdirme, \
5509	head=plot, middle=opS, tail=cnf['kindfig'])
5510	Nimages = len(modexpimg)
5511	if Nimages > 0:
5512	for img in range(Nimages): modexpimg[img] = \
5513	owdirme + '/' + modexpimg[img]
5514	caption = Smods + ' ' + Sexps + ' ' + plot + \
5515	' ' + op + ' diff' + diffn
5516	flab = 'fig:' + caption.replace(' ', '_')
5517	try:
5518	with gen.Capturing() as output:
5519	gen.latex_fig_array(modexpimg, otexf,\
5520	gen.latex_text(caption), flab, \
5521	refsize=0.5, dist='sqr', \
5522	dorest='center')
5523	except:
5524	print errmsg
5525	print 'gen.latex_fig_array(',modexpimg, \
5526	', '+otexf+', gen.latex_text(' + \
5527	caption + '), '+flab+ ", refsize=0.5,"+\
5528	" dist='sqr', dorest='center')"
5529	for s1out in output: print s1out
5530	quit(-1)
5531	otexf.write('%\\clearpage\n')
5532
5533	# allmodexp
5534	owdirme = owdir + '/allmodexps'
5535	print ' allmodexps...'
5536	fileplotinf = owdirme + '/allmodexpfigures.inf'
5537	if os.path.isfile(fileplotinf):
5538	otexf.write('% allmodexps\n')
5539	otexf.write('\\modexp{allmodexps}\n')
5540	listplots, Nplt = read_plot_file(fileplotinf)
5541	for plot in listplots.keys():
5542	plots = listplots[plot]
5543	ops = []
5544	for plt in plots:
5545	opn = plt.split('\|')[1]
5546	if not gen.searchInlist(ops,opn): ops.append(opn)
5547	for op in ops:
5548	opS = op.replace('+','-')
5549	modexpimg = gen.files_folder_HMT(owdirme, head=plot, middle=opS, \
5550	tail=cnf['kindfig'])
5551	Nimages = len(modexpimg)
5552	if Nimages > 0:
5553	for img in range(Nimages): modexpimg[img] = owdirme + '/' + \
5554	modexpimg[img]
5555	caption = 'allmodexps ' + plot + ' ' + op + ' diff' + diffn
5556	flab = 'fig:' + caption.replace(' ', '_')
5557	try:
5558	with gen.Capturing() as output:
5559	gen.latex_fig_array(modexpimg,otexf, gen.latex_text(caption),\
5560	flab, refsize=0.9, dist='sqr', dorest='center')
5561	except:
5562	print errmsg
5563	print 'gen.latex_fig_array(',modexpimg,', '+otexf+ \
5564	', gen.latex_text(' + caption + '), ' +flab+ ", refsize=0.9," +\
5565	" dist='sqr', dorest='center')"
5566	for s1out in output: print s1out
5567	quit(-1)
5568
5569	otexf.write('%\\clearpage\n')
5570	#
5571	# Grouping by type of figure
5572	#
5573	Nmods = len(mods)
5574	# Figures from mod/exp pairs
5575	owsearch = owdir + '/' + mods[0] + '/' + modexps[mods[0]][0]
5576	print " Looking in '" + owsearch + "' for figures by kind and variable..."
5577	otexf.write('% figures by kind and variable from:' + owsearch + '\n')
5578	modexpimg = gen.files_folder_HMT(owsearch, tail=cnf['kindfig'])
5579	# dictionary with kind of plots
5580	dplots = {}
5581	# list with kind of plots
5582	lplots = []
5583	# dictionary with variable-statistics of each kind of plot
5584	dvarstats = {}
5585	# list with variables of each kind of plot
5586	lvars = []
5587	for plot in modexpimg:
5588	Nvars, plotk, varn, statn = figinf_figname(plot,cnf)
5589	if not gen.searchInlist(lplots,plotk):
5590	lplots.append(plotk)
5591	if not dplots.has_key(plotk+'_'+varn):
5592	if not gen.searchInlist(lvars,varn): lvars.append(varn)
5593	dplots[plotk+'_'+varn] = [statn]
5594	else:
5595	vals = dplots[plotk+'_'+varn]
5596	dplots[plotk+'_'+varn] = vals + [statn]
5597	else:
5598	if not dplots.has_key(plotk+'_'+varn):
5599	if not gen.searchInlist(lvars,varn): lvars.append(varn)
5600	dplots[plotk+'_'+varn] = [statn]
5601	else:
5602	vals = dplots[plotk+'_'+varn]
5603	dplots[plotk+'_'+varn] = vals + [statn]
5604
5605	# Figures found
5606	print ' direct figures to group:', modexpimg
5607	print ' plots:', lplots
5608	print ' variables:', lvars
5609
5610	for dplot in dplots.keys():
5611	plot = dplot.split('_')[0]
5612	var = dplot.split('_')[1]
5613	otexf.write('% ' + var + '\n')
5614	otexf.write('\\modexp{' + gen.latex_text(var) + '}\n')
5615	varn1 = var.split('-')[0]
5616	varn2 = var.split('-')[1]
5617	varn = varn1+'-'+varn2
5618	for stn12 in dplots[dplot]:
5619	stn1 = stn12.split('@')[0]
5620	stn2 = stn12.split('@')[1]
5621	stn = gen.lstring_values(stn12, '@',' & ')
5622	plotvarfigs = []
5623	caption = varn.replace('-',' & ') + ' ' + plot + ' ' + stn
5624	flab = 'fig:' + varn + '_' + plot + '_' + stn12 + '_allmodexp'
5625	for me in lmodexps:
5626	modexpn = expGC[me].label
5627	modn = me.split('/')[0]
5628	imgns = gen.files_folder_HMT(owdir+'/'+me, head=plot+'_'+varn1, \
5629	middle=stn1+'-'+varn2, tail=stn2+'.'+cnf['kindfig'])
5630	caption = caption + ', ' + modexpn
5631	imgn = owdir+'/'+me+'/'+imgns[0]
5632	if os.path.isfile(imgn):
5633	plotvarfigs.append(imgn)
5634	if dbg:
5635	print ' Grouping figures:', plotvarfigs
5636	try:
5637	with gen.Capturing() as output:
5638	gen.latex_fig_array(plotvarfigs, otexf, gen.latex_text(caption), \
5639	flab, refsize=0.9, dist='sqr', dorest='center')
5640	except:
5641	print errmsg
5642	print 'gen.latex_fig_array(',plotvarfigs,', '+otexf+', gen.latex_text(' +\
5643	caption + '), ' + flab + ", refsize=0.9, dist='sqr', dorest='center')"
5644	for s1out in output: print s1out
5645	quit(-1)
5646
5647	Nmods = len(mods)
5648	# Figures from mod/exp differences
5649	owsearch = owdir + '/' + ldiffmodexps[0]
5650	print " Looking in '" + owsearch + "' for difference figures by kind and variable..."
5651	otexf.write('% difference figures by kind and variable from:' + owsearch + '\n')
5652	modexpimg = gen.files_folder_HMT(owsearch, tail=cnf['kindfig'])
5653	# dictionary with kind of plots
5654	dplots = {}
5655	# list with kind of plots
5656	lplots = []
5657	# dictionary with variable-statistics of each kind of plot
5658	dvarstats = {}
5659	# list with variables of each kind of plot
5660	lvars = []
5661	for plot in modexpimg:
5662	Nvars, plotk, varn, statn = figinf_figname(plot,cnf)
5663	if not gen.searchInlist(lplots,plotk):
5664	lplots.append(plotk)
5665	if not dplots.has_key(plotk+'_'+varn):
5666	if not gen.searchInlist(lvars,varn): lvars.append(varn)
5667	dplots[plotk+'_'+varn] = [statn]
5668	else:
5669	vals = dplots[plotk+'_'+varn]
5670	dplots[plotk+'_'+varn] = vals + [statn]
5671	else:
5672	if not dplots.has_key(plotk+'_'+varn):
5673	if not gen.searchInlist(lvars,varn): lvars.append(varn)
5674	dplots[plotk+'_'+varn] = [statn]
5675	else:
5676	vals = dplots[plotk+'_'+varn]
5677	dplots[plotk+'_'+varn] = vals + [statn]
5678
5679	# Figures found
5680	print ' difference figures to group:', modexpimg
5681	print ' plots:', lplots
5682	print ' variables:', lvars
5683
5684	for dplot in dplots.keys():
5685	plot = dplot.split('_')[0]
5686	var = dplot.split('_')[1]
5687	otexf.write('% ' + var + '\n')
5688	otexf.write('\\modexp{diff ' + gen.latex_text(var) + '}\n')
5689	varn1 = var.split('-')[0]
5690	varn2 = var.split('-')[1]
5691	varn = varn1+'-'+varn2
5692	for stn12 in dplots[dplot]:
5693	stn1 = stn12.split('@')[0]
5694	stn2 = stn12.split('@')[1]
5695	stn = gen.lstring_values(stn12, '@',' & ')
5696	plotvarfigs = []
5697	caption = varn.replace('-',' & ') + ' ' + plot + ' ' + stn
5698	flab = 'fig:' + varn + '_' + plot + '_' + stn12 + '_alldiffmodexp'
5699	for me in ldiffmodexps:
5700	mods = me.split('/')[0]
5701	exps = me.split('/')[1]
5702	if exps.find('-') != -1:
5703	expn1 = exps.split('-')[0]
5704	expn2 = exps.split('-')[1]
5705	else:
5706	expn1 = exps
5707	expn2 = exps
5708
5709	if mods.find('-') != -1:
5710	modn1 = mods.split('-')[0]
5711	modn2 = mods.split('-')[1]
5712	else:
5713	modn1 = mods
5714	modn2 = mods
5715	modexpn1 = expGC[modn1 + '/' + expn1].label
5716	modexpn2 = expGC[modn2 + '/' + expn2].label
5717	modexpn = modexpn1 + '-' + modexpn2
5718
5719	modn = me.split('/')[0]
5720	imgns = gen.files_folder_HMT(owdir+'/'+me, head=plot+'_'+varn1, \
5721	middle=stn1+'-'+varn2, tail=stn2+'.'+cnf['kindfig'])
5722	caption = caption + ', ' + modexpn
5723	imgn = owdir+'/'+me+'/'+imgns[0]
5724	if os.path.isfile(imgn):
5725	plotvarfigs.append(imgn)
5726	if dbg:
5727	print ' Grouping figures:', plotvarfigs
5728	try:
5729	with gen.Capturing() as output:
5730	gen.latex_fig_array(plotvarfigs, otexf, gen.latex_text(caption), \
5731	flab, refsize=0.9, dist='sqr', dorest='center')
5732	except:
5733	print errmsg
5734	print 'gen.latex_fig_array(',plotvarfigs,', '+otexf+', gen.latex_text('+ \
5735	caption + '), ' + flab + ", refsize=0.9, dist='sqr', dorest='center')"
5736	for s1out in output: print s1out
5737	quit(-1)
5738
5739	otexf.write('\\end{document}\n')
5740	otexf.close()
5741
5742	try:
5743	with gen.Capturing() as output:
5744	ins = 'pdflatex ' + owdir + '/' + texout
5745	sout = sub.call(ins, shell=True)
5746	except:
5747	print errormsg
5748	print ' ' + ins
5749	print sout
5750	for s1out in output: print s1out
5751	quit(-1)
5752
5753	with gen.Capturing() as output:
5754	sout = sub.call(ins + '>& /dev/null', shell=True)
5755	with gen.Capturing() as output:
5756	sout = sub.call(ins + '>& /dev/null', shell=True)
5757	with gen.Capturing() as output:
5758	sout = sub.call(ins + '>& /dev/null', shell=True)
5759	with gen.Capturing() as output:
5760	sout = sub.call(ins, shell=True)
5761	if dbg:
5762	print sout
5763	for s1out in output: print s1out
5764
5765	sub.call('evince ' + owdir + '/' + texout + '.pdf &', shell=True)
5766

Note: See TracBrowser for help on using the repository browser.

Download in other formats: