Changeset 1032 in lmdz_wrf

Timestamp:

Aug 19, 2016, 12:55:41 PM (9 years ago)

Author:

lfita

Message:

Up to getting written/read difference files

Location:

trunk/tools

Files:

• model_graphics.py (modified) (4 diffs)
• model_graphics_template.dat (modified) (2 diffs)

trunk/tools/model_graphics.py ¶

@@ -1613 +1613 @@
     return
 
+def get_differences_var(vc,kdiff,db):
+    """ Function to provide the operations to make for each variable from kdiff ('DIFFOP_' or 'DIFFVAR_') dictionary
+      vc= kdiff dictionary, dictionary with all the parameters started with kdiff
+      kdiff= kind of differences 'op' or 'var'
+    """
+    fname = 'get_differences_var'
+
+    if kdiff == 'op':
+        diffn = 'DIFFOP'
+    elif kdiff == 'var':
+        diffn = 'DIFFVAR'
+    else:
+         print errmsg
+         print '  ' + fname + ": differences '" + kdiff + "' not ready !!"
+         quit(-1)
+    LD = len(diffn + '_')
+
+    iop = 1
+    # list of operations by difference
+    doopers = {}
+    # operations by variable
+    operationsvar = {}
+    # individual operations by variable (not repeated)
+    indivoperationsvar = {}
+
+    # Variables with a calculation whic requires vertical interpolation to all the 
+    #   file (operations stating by [kdiff]'_pinterp')
+    LVp = len(diffn + '_pinterp')
+    varglobalp = []
+
+    for oper in vc.keys():
+        Loper = len(oper)
+        opn = oper[LD:Loper+1]
+        vns = vc[oper].split(':')
+        ops1 = opn.split('+')
+        doopers[opn] = ops1
+        for vn in vns:
+            if not operationsvar.has_key(vn):
+                operationsvar[vn] = [opn]
+                indivoperationsvar[vn] = ops1
+            else:
+                opers = operationsvar[vn]
+                opers.append(opn)
+                operationsvar[vn] = opers
+                opers1 = indivoperationsvar[vn]
+                for op1 in ops1:
+                    if not gen.searchInlist(opers1, op1):
+                        opers1.append(op1)
+                indivoperationsvar[vn] = opers1
+
+        if oper[0:LVp] == diffn + '_pinterp':
+            for vn in vns:
+                if not gen.searchInlist(varglobalp,vn): varglobalp.append(vn)
+
+        iop = iop + 1
+
+    if db:
+        print '  operations to make _______'
+        gen.printing_dictionary(doopers)
+        print '  operations by variable _______'
+        gen.printing_dictionary(operationsvar)
+        print '  individual operations by variable _______'
+        gen.printing_dictionary(indivoperationsvar)
+        print '  variables with the vertical interpolation for all the file _______'
+        print '    #', varglobalp
+
+    return doopers, operationsvar, indivoperationsvar, varglobalp
+
+def diffvarop_listconstruct(config, mods, exps, odir, debug):
+    """ Function to create the list of differences to compute and draw
+      config= Configuration of the experiment
+      mods= list with the models to use
+      exps= list with the experiments to use
+      odir= output differences folder
+    """
+    fname='diffvarop_listconstruct'
+  
+    diffop = gen.get_specdictionary_HMT(config, H='DIFFOP_')
+    diffvar = gen.get_specdictionary_HMT(config, H='DIFFVAR_')
+
+    if debug:
+        print "  'op' differences ________"
+        gen.printing_dictionary(diffop)
+        print "  'var' differences ________"
+        gen.printing_dictionary(diffvar)
+
+    # Getting variable characteristics from each model/experiment
+    idir1 = config['ofold'] + '/' + mods[0] + '/' + exps[0]
+    idir2 = config['ofold'] + '/' + mods[1] + '/' + exps[1]
+
+    files1, testfiles1, allcompvar1, Nvar1= read_varcomp_file(idir1+'/varcompute.inf')
+    files2, testfiles2, allcompvar2, Nvar2= read_varcomp_file(idir2+'/varcompute.inf')
+
+    # Getting 'op' differences characteristics
+    doops, opvar, indivopvar, varglobalp = get_differences_var(diffop, 'op', debug)
+
+    diffops = {}
+    Ndiffop = 0
+    for vn in opvar.keys():
+        vnops = opvar[vn]
+        for op in vnops:
+            vnop = vn + '_' + op
+            if not allcompvar1.has_key(vnop):
+                print warnmsg
+                print '  ' + fname + ": DIFFOP variable+operation '" + vnop +        \
+                  "' in '" + mods[0] + '/' + exps[0] + "' was not computed !!"
+                print '    skipping it!'
+                break
+            if not allcompvar2.has_key(vnop):
+                print warnmsg
+                print '  ' + fname + ": DIFFOP variable+operation '" + vnop +        \
+                  "' in '" + mods[1] + '/' + exps[1] + "' was not computed !!"
+                print '    skipping it!'
+                break
+            vals1 = allcompvar1[vnop]
+            vals2 = allcompvar2[vnop]
+
+            headerf1 = vals1[0]
+            headerf2 = vals2[0]
+
+            diffops[vnop] = [headerf1, headerf2]
+
+            Ndiffop = Ndiffop + 1        
+
+    # Getting 'var' differences characteristics
+    doops, opvar, indivopvar, varglobalp = get_differences_var(diffvar, 'var', debug)
+
+    diffvars = {}
+    Ndiffvar = 0
+    for vn in opvar.keys():
+        vnops = opvar[vn]
+        for op in vnops:
+            vnop = vn + '_' + op
+            if not allcompvar1.has_key(vnop):
+                print warnmsg
+                print '  ' + fname + ": DIFFVAR variable+operation '" + vnop +       \
+                  "' in '" + mods[0] + '/' + exps[0] + "' was not computed !!"
+                print '    skipping it!'
+                break
+            if not allcompvar2.has_key(vnop):
+                print warnmsg
+                print '  ' + fname + ": DIFFVAR variable+operation '" + vnop +       \
+                  "' in '" + mods[1] + '/' + exps[1] + "' was not computed !!"
+                print '    skipping it!'
+                break
+            vals1 = allcompvar1[vnop]
+            vals2 = allcompvar2[vnop]
+
+            headerf1 = vals1[0]
+            headerf2 = vals2[0]
+
+            diffvars[vnop] = [headerf1, headerf2]
+
+            Ndiffvar = Ndiffvar + 1        
+
+    Sopdiffs = gen.dictKeysVals_stringList(diffops)
+    Svardiffs = gen.dictKeysVals_stringList(diffvars)
+
+    # Outwritting the op diffferences file to avoid next time (if it is not filescratch!)
+    objf = open(odir + '/diffop.inf', 'w')
+    objf.write('differences: ' + Sopdiffs + '\n')
+    objf.write('Ndiff: ' + str(Ndiffop) + '\n')
+    objf.close() 
+
+    # Outwritting the var diffferences file to avoid next time (if it is not filescratch!)
+    objf = open(odir + '/diffvar.inf', 'w')
+    objf.write('differences: ' + Svardiffs + '\n')
+    objf.write('Ndiff: ' + str(Ndiffvar) + '\n')
+    objf.close() 
+
+    return diffops, diffvars, Ndiffvar, Ndiffop
+
+def read_diff_file(difffile):
+    """ Function to read the file with the information about the differences
+      difffile= file with the information
+    """
+    fname = 'read_diff_file'
+
+    if not os.path.isfile(difffile):
+        print errormsg
+        print '  ' + fname + ": differences file '" + difffile + "' does not exist !!"
+        quit(-1)
+
+    objf = open(difffile, 'r')
+    for line in objf:
+        if line[0:1] != '#' and len(line) > 1:
+            values = line.split(' ')
+            if values[0] == 'differences:':
+                diffs = gen.stringList_dictKeysVals(values[1])
+            elif values[0] == 'Ndiff:':
+                Ndiffs = int(values[1])
+
+    objf.close()
+
+    return diffs, Ndiffs
+
 # Files with information about the configuration of the script
 inffiles = ['varcompute.inf', 'all_computevars.inf', 'all_statsvars.inf']
@@ -1788 +1984 @@
 # Experiments loop
 ##
+    # There are two kind of differences:
+    #   DIFFop: differences between operations of each given variable. 
+    #     Differences are computed directly from the last stage of the operation
+    #   DIFFvar: operations of differences between variables
+    #     First are computed the differences from the initial variable file
+    #     and then operations are made
+    # NOTE: remember that: meanvar2 - meanvar1 = mean(var2 - var1)
+    difffiles = ['diffop.inf', 'diffvar.inf']
+
     Nexps = len(exps)
     for exp1 in exps[0:Nexps-1]:
@@ -1795 +2000 @@
             sout = sub.call('mkdir -p ' + owdir, shell=True)
 
-            diffvarcompf = owdir + '/diffvarcompute.inf'
-            if diffscratch:
-                sub.call('rm -rf' + owdir +' >& /dev/null', shell=True)
-                sub.call('rm ' + diffvarcompf +' >& /dev/null', shell=True)
-                objf = open(owdir+'/all_diffcomputevars.inf','w')
-                objf.write("## Computation and drawing of differences between '" +   \
-                  exp2 + "'-'" + exp1 + "'\n")
-                objf.close()
-
-            if adddiffs:
-                sub.call('rm ' + diffvarcompf +' >& /dev/null', shell=True)
-
+            # Removing files with list of differences if should be started from scratch or add differences
+            for fdiff in difffiles:
+                difff = owdir + '/' + fdiff
+                if diffscratch:
+                    sub.call('rm -rf' + owdir +' >& /dev/null', shell=True)
+                    sub.call('rm ' + diffvarcompf +' >& /dev/null', shell=True)
+                    objf = open(owdir+'/all_' + fdiff,'w')
+                    objf.write("## Computation and drawing of differences " +        \
+                      "between '" + exp2 + "'-'" + exp1 + "'\n")
+                    objf.close()
+
+                if adddiffs:
+                    sub.call('rm ' + difff +' >& /dev/null', shell=True)
+
+            # op and var differences
+            diffvarcompf = owdir + '/' + difffiles[0]
             if not os.path.isfile(diffvarcompf):
-                alldiffcompvar, Ndiffs = diffcompvars_listconstruct(cnf, Modinf,     \
-                  allcompvar, owdir, dbg)
+                alldiffop, alldiffvar,Nopdiffs,Nvardiffs=diffvarop_listconstruct(cnf,\
+                  [mod, mod], [exp1, exp2], owdir, dbg)
             else: 
                 print warnmsg
-                print '  ' + main + ": getting differences to compute already from file !!"
-                alldiffcompvar, Ndiffs = read_diffvarcomp_file(diffvarcompf)
+                print '  ' + main + ": getting 'op' and 'var'differences to " +      \
+                  "compute already from file !!"
+                alldiffop, Nopdiffops = read_diff_file(diffvarcompf)
+                alldiffvar, Nopdiffvars = read_diff_file(owdir+'/'+difffiles[1])
 
         # End of avoiding to repeat all the experiment search
@@ -1822 +2033 @@
         if dbg:
             print 'Differences to compute _______'
-            gen.printing_dictionary(alldiffcompvar)
+            gen.printing_dictionary(alldiffop)
+            gen.printing_dictionary(alldiffvar)
 
     quit()

trunk/tools/model_graphics_template.dat ¶

@@ -42 +42 @@
 
 # Debug
-debug = false
+debug = true
 
 # Folder with the files
@@ -108 +108 @@
 #varcombo = ta;pinterp@last@xmean:ua;pinterp@last@xmean:va;pinterp@last@xmean:hur;pinterp@last@xmean:ws;pinterp@turb@xmean:hus;pinterp@turb@xmean
 
-vardiff = tas|last:tas|xmean:tas|tmean:wss|last:wss|xmean:wss|tmean:ua|pinterp@last@xmean:va|pinterp@last@xmean
+# Differences to compute
+#   These differences will be computed among experiments and among models (and model-experiments)
+#  ':' separated list of [var]|[op]
+#  DIFFOP_[op]: differences of the [op] of [var]. NOTE: [op] must exist for each experiment
+#    Differences are computed directly from the last stage of the operation
+#  DIFFVAR_[op]: [op] of the [var] differences [var]. NOTE: [var] must exist for each experiment
+#    First are computed the differences from the initial variable file and then operations are made
+# NOTE: remember that: meanvar2 - meanvar1 = mean(var2 - var1)
+DIFFOP_last = hfls:hfss
+DIFFOP_xmean = uas:vas
+DIFFOP_tmean = hfss:hfls
+DIFFOP_pinterp+tmean+xmean = ua:va
+
+DIFFVAR_last = hfls:hfss
+DIFFVAR_xmean = uas:vas
+DIFFVAR_tmean = hfss:hfls
+DIFFVAR_pinterp+tmean+xmean = ua:va
 
 # Operations which attach a 'surname' at the end of the variable name