Changeset 3363

Timestamp:

Jun 10, 2024, 4:24:43 PM (7 months ago)

Author:

jbclement

Message:

PEM:

• The PEM can now stop itself cleanly before the SLURM time limit for the job is reached and it continues the simulation with a new cycle.
• Update of "jobPEM.slurm" in the deftank to guarantee enough memory space to run the job.
• Few minor cleanings.

JBC

Location:

trunk/LMDZ.COMMON/libf/evolution

Files:

• changelog.txt (modified) (1 diff)
• deftank/jobPEM.slurm (modified) (2 diffs)
• deftank/launchPEM.sh (modified) (2 diffs)
• deftank/visu_layering.py (moved) (moved from trunk/LMDZ.COMMON/libf/evolution/deftank/output_layering.py)
• pem.F90 (modified) (8 diffs)
• read_data_PCM_mod.F90 (modified) (3 diffs)

trunk/LMDZ.COMMON/libf/evolution/changelog.txt

@@ -348 +348 @@
 The launching script can now operate a relaunch from any already computed PCM/PEM run. The starting point is asked to the user interactively on the terminal.
 Some features might not work very well yet since not every stituation has not been tested.
+
+== 10/06/2024 == JBC
+- The PEM can now stop itself cleanly before the SLURM time limit for the job is reached and it continues the simulation with a new cycle.
+- Update of "jobPEM.slurm" in the deftank to guarantee enough memory space to run the job.
+- Few minor cleanings.

trunk/LMDZ.COMMON/libf/evolution/deftank/jobPEM.slurm

@@ -5 +5 @@
 #SBATCH --constraint=GENOA
 ### Number of Nodes to use
-#SBATCH --nodes=1
+#SBATCH --nodes=4 # to run with enough memory
 #SBATCH --ntasks-per-node=1
 #SBATCH --cpus-per-task=1
@@ -61 +61 @@
 fi
 
-./launchPEM.sh new
+# Launch the next cycle
+#if [ "$(awk 'END{print $NF}' info_PEM.txt)" -eq 7 ]; then
+#    read i_myear n_myear convert_years iPCM iPEM nPCM nPCM_ini < info_PEM.txt
+#    ./launchPEM.sh cont # Continue the PEM run if it stopped because of job time limit
+#else
+    ./launchPEM.sh new
+#fi

trunk/LMDZ.COMMON/libf/evolution/deftank/launchPEM.sh

@@ -64 +64 @@
             errlaunch
         fi
+        echo "The relaunch is initialized with a specific previous successful run."
         while true; do
             echo "Do you want to relaunch from a 'PCM' or 'PEM' run?"
@@ -106 +107 @@
             relaunchPEM
         fi
+
+    # Continuing the PEM run
+    elif [ $1 = "cont" ]; then
+        exec >> log_launchPEM.txt 2>&1
+        echo
+        echo "This is a continuation of the previous PEM run."
+        date
+        submitPEM
+
+    # Default case: error
     else
         echo "Error: given argument '$1' for the launching script is unknown!"

trunk/LMDZ.COMMON/libf/evolution/pem.F90

@@ -228 +228 @@
 
 ! Some variables for the PEM run
-real, parameter :: year_step = 1 ! timestep for the pem
-integer         :: year_iter     ! number of iteration
-integer         :: year_iter_max ! maximum number of iterations before stopping
-integer         :: i_myear       ! Global number of Martian years of the chained simulations
-integer         :: n_myear       ! Maximum number of Martian years of the chained simulations
-real            :: timestep      ! timestep [s]
+real, parameter :: year_step = 1   ! Timestep for the pem
+integer         :: year_iter       ! Number of iteration
+integer         :: year_iter_max   ! Maximum number of iterations before stopping
+integer         :: i_myear         ! Global number of Martian years of the chained simulations
+integer         :: n_myear         ! Maximum number of Martian years of the chained simulations
+real            :: timestep        ! Timestep [s]
+character(20)   :: job_id          ! Job id provided as argument passed on the command line when the program was invoked
+integer(kind=8) :: cr              ! Number of clock ticks per second (count rate)
+integer(kind=8) :: c1, c2          ! Counts of processor clock
+character(100)  :: chtimelimit     ! Time limit for the PEM job outputted by the SLURM command
+real            :: timelimit       ! Time limit for the PEM job in seconds
+real, parameter :: antetime = 1200 ! Anticipation time to prevent reaching the time time limit: 1200 s = 20 min by default
+integer         :: cstat, days, hours, minutes, seconds
+character(1)    :: sep
 
 #ifdef CPP_STD
@@ -274 +282 @@
 integer :: i, l, ig, nnq, t, islope, ig_loop, islope_loop, isoil, icap
 
+! Elapsed time with system clock
+call system_clock(count_rate = cr)
+call system_clock(c1)
+timelimit = 86400 ! 86400 seconds = 24 h by default
+if (command_argument_count() > 0) then
+    ! Read the job id passed as argument to the program
+    call get_command_argument(1,job_id)
+    ! Execute the system command
+    call execute_command_line('squeue -j '//trim(job_id)//' -h --Format TimeLimit > tmp_cmdout.txt',cmdstat = cstat)
+    if (cstat > 0) then
+        error stop 'pem: command execution failed!'
+    else if (cstat < 0) then
+        error stop 'pem: command execution not supported!'
+    endif
+    ! Read the output
+    open(1,file = 'tmp_cmdout.txt',status = 'old')
+    read(1,'(a)') chtimelimit
+    close(1)
+    chtimelimit = trim(chtimelimit)
+    call execute_command_line('rm tmp_cmdout.txt',cmdstat = cstat)
+    if (cstat > 0) then
+        error stop 'pem: command execution failed!'
+    else if (cstat < 0) then
+        error stop 'pem: command execution not supported!'
+    endif
+    ! Assume 'chtimelimit' format is "D-HH:MM:SS" or "HH:MM:SS"
+    if (index(chtimelimit,'-') > 0) then
+        read(chtimelimit, '(i1,a1,i2,a1,i2,a1,i2)') days, sep, hours, sep, minutes, sep, seconds
+        timelimit = days*86400 + hours*3600 + minutes*60 + seconds
+    else
+        read(chtimelimit,'(i2,a1,i2,a1,i2)') hours, sep, minutes, sep, seconds
+        timelimit = hours*3600 + minutes*60 + seconds
+    endif
+endif
+
 ! Parallel variables
 #ifndef CPP_STD
@@ -310 +353 @@
 
     therestart1D = .false. ! Default value
-    inquire(file = 'start1D_evol.txt',exist = therestart1D)
+    inquire(file = 'start1D.txt',exist = therestart1D)
     if (.not. therestart1D) then
-        write(*,*) 'There is no "start1D_evol.txt" file!'
+        write(*,*) 'There is no "start1D.txt" file!'
         error stop 'Initialization cannot be done for the 1D PEM.'
     endif
@@ -322 +365 @@
     endif
 
-    call init_testphys1d('start1D_evol.txt','startfi.nc',therestart1D,therestartfi,ngrid,nlayer,610.,nq,q, &
-                         time_0,ps(1),ucov,vcov,teta,ndt,ptif,pks,dtphys,zqsat,dq,dqdyn,day0,day,gru,grv,w,     &
+    call init_testphys1d('start1D.txt','startfi.nc',therestart1D,therestartfi,ngrid,nlayer,610.,nq,q,        &
+                         time_0,ps(1),ucov,vcov,teta,ndt,ptif,pks,dtphys,zqsat,dq,dqdyn,day0,day,gru,grv,w,  &
                          play,plev,latitude,longitude,cell_area,atm_wat_profile,atm_wat_tau)
     ps(2) = ps(1)
@@ -505 +548 @@
 write(*,*) "Downloading data Y1..."
 call read_data_PCM("data_PCM_Y1.nc",timelen,iim,jjm_value,ngrid,nslope,vmr_co2_PCM,ps_timeseries,min_co2_ice(:,:,1),min_h2o_ice(:,:,1), &
-                   tsurf_avg_yr1,tsoil_ave,tsurf_PCM_timeseries,tsoil_PCM_timeseries,q_co2_PEM_phys,q_h2o_PEM_phys,                      &
+                   tsurf_avg_yr1,tsoil_ave,tsurf_PCM_timeseries,tsoil_PCM_timeseries,q_co2_PEM_phys,q_h2o_PEM_phys,                     &
                    co2_ice_PCM,watersurf_density_ave,watersoil_density_timeseries)
 write(*,*) "Downloading data Y1 done!"
@@ -958 +1001 @@
 !    II_g Checking the stopping criterion
 !------------------------
+    call system_clock(c2)
+    if (timelimit - real((c2 - c1)/cr) <= antetime) stopPEM = 7
     call stopping_crit_h2o_ice(cell_area,h2oice_ini_surf,ini_h2oice_sublim,h2o_ice,stopPEM,ngrid)
     call stopping_crit_co2(cell_area,co2ice_ini_surf,ini_co2ice_sublim,co2_ice,stopPEM,ngrid,global_avg_press_PCM,global_avg_press_new,nslope)
@@ -981 +1026 @@
             case(6)
                 write(*,*) "STOPPING because maximum number of Martian years to be simulated is reached:", stopPEM
+            case(7)
+                write(*,*) "STOPPING because the time limit for the PEM job will be reached soon:", stopPEM
             case default
                 write(*,*) "STOPPING with unknown stopping criterion code:", stopPEM
@@ -1104 +1151 @@
     write(*,*) "restart.nc has been written"
 #else
-    call writerestart1D('restart1D_evol.txt',ps(1),tsurf(1,:),nlayer,size(tsurf,2),teta,ucov,vcov,nq,noms,qsurf(1,:,:),q)
-    write(*,*) "restart1D_evol.txt has been written"
+    call writerestart1D('restart1D.txt',ps(1),tsurf(1,:),nlayer,size(tsurf,2),teta,ucov,vcov,nq,noms,qsurf(1,:,:),q)
+    write(*,*) "restart1D.txt has been written"
 #endif
 

trunk/LMDZ.COMMON/libf/evolution/read_data_PCM_mod.F90

@@ -81 +81 @@
 call error_msg(NF90_OPEN(filename,NF90_NOWRITE,fID),"open",filename)
 
-! Dowload the data from the file
+! Download the data from the file
 call get_var3("ps",ps_PCM)
 write(*,*) "Data for surface pressure downloaded!"
@@ -172 +172 @@
 #endif
 endif
+
+! Close the NetCDF file
+write(*,*) "Closing "//filename//"..."
+call error_msg(nf90_close(fID),"close",filename)
 
 ! Compute the minimum over the year for each point
@@ -336 +340 @@
     case('inq');   msg="Field <"//trim(nam)//"> is missing"
     case('get');   msg="Reading failed for <"//trim(nam)//">"
+    case('put');   msg="Writing failed for <"//trim(nam)//">"
     case('open');  msg="File opening failed for <"//trim(nam)//">"
     case('close'); msg="File closing failed for <"//trim(nam)//">"