Ignore:
Timestamp:
Oct 25, 2013, 10:12:38 AM (11 years ago)
Author:
Ehouarn Millour
Message:

Getting XIOS to work in MPI, OpenMP and mixed MPI/OpenMP modes. Use "-io xios" option with makelmdz or makelmdz_fcm to enable XIOS (this doesn't deactivate IOIPSL outputs; to suppress IOIPSL outputs, add cpp flag CPP_NO_IOIPSL).
EM

Location:
LMDZ5/trunk/libf/phydev
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • LMDZ5/trunk/libf/phydev/iophy.F90

    r1883 r1897  
    11!
    2 ! $Header$
     2! $Id: $
    33!
    44module iophy
    … …  
    4040
    4141  subroutine init_iophy_new(rlat,rlon)
    42   USE dimphy
    43   USE mod_phys_lmdz_para
    44   USE mod_grid_phy_lmdz
    45   USE ioipsl
     42  USE dimphy, only: klon
     43  USE mod_phys_lmdz_para, only: gather, bcast, &
     44                                jj_nb, jj_begin, jj_end, ii_begin, ii_end, &
     45                                mpi_size, mpi_rank, klon_mpi, &
     46                                is_sequential, is_south_pole
     47  USE mod_grid_phy_lmdz, only: nbp_lon, nbp_lat, klon_glo
     48#ifdef CPP_IOIPSL
     49  USE ioipsl, only: flio_dom_set
     50#endif
     51#ifdef CPP_XIOS
     52  use wxios, only: wxios_domain_param
     53#endif
    4654  implicit none
    47   include 'dimensions.h'   
     55  include 'dimensions.h'
     56  include 'iniprint.h'
    4857    real,dimension(klon),intent(in) :: rlon
    4958    real,dimension(klon),intent(in) :: rlat
    … …  
    6069    INTEGER,DIMENSION(2) :: dhe
    6170    INTEGER :: i   
     71    integer :: data_ibegin,data_iend
    6272
    6373    CALL gather(rlat,rlat_glo)
    … …  
    109119#endif
    110120#ifdef CPP_XIOS
    111     !Pour els soucis en MPI, réglage du masque:
     121    ! Set values for the mask:
    112122    IF (mpi_rank == 0) THEN
    113123        data_ibegin = 0
    … …  
    122132    END IF
    123133
    124     WRITE(*,*) "TOTO mpirank=",mpi_rank,"iibeg=",ii_begin , "jjbeg=",jj_begin,"jjnb=",jj_nb,"jjend=",jj_end
    125 
    126     !On initialise le domaine xios, maintenant que tout est connu:
    127     !SUBROUTINE wxios_domain_param(dom_id, is_sequential, ni, nj, ni_glo, nj_glo,        &
    128     !                                ibegin, iend, jbegin, jend,                         &
    129     !                                data_ni, data_ibegin,                               &
    130     !                                io_lat, io_lon)
     134    if (prt_level>=10) then
     135      write(lunout,*) "init_iophy_new: mpirank=",mpi_rank," iibegin=",ii_begin , " ii_end=",ii_end," jjbegin=",jj_begin," jj_nb=",jj_nb," jj_end=",jj_end
     136      write(lunout,*) "init_iophy_new: mpirank=",mpi_rank," nbp_lon=",nbp_lon," nbp_lat=",nbp_lat
     137      write(lunout,*) "init_iophy_new: mpirank=",mpi_rank," data_ibegin=",data_ibegin," data_iend=",data_iend
     138      write(lunout,*) "init_iophy_new: mpirank=",mpi_rank," data_ibegin=",data_ibegin," data_iend=",data_iend
     139      write(lunout,*) "init_iophy_new: mpirank=",mpi_rank," is_south_pole=",is_south_pole
     140    endif
     141
     142    ! Initialize the XIOS domain coreesponding to this process:
    131143    CALL wxios_domain_param("dom_glo", is_sequential, nbp_lon, jj_nb, nbp_lon, nbp_lat, &
    132144                            1, nbp_lon, ii_begin, ii_end, jj_begin, jj_end,             &
    133145                            klon_mpi+2*(nbp_lon-1), data_ibegin, data_iend,             &
    134                             io_lat, io_lon)
     146                            io_lat, io_lon,is_south_pole,mpi_rank)
    135147#endif
    136148!$OMP END MASTER
    … …  
    141153 
    142154  subroutine histbeg_phy(name,itau0,zjulian,dtime,nhori,nid_day)
    143   USE dimphy
    144   USE mod_phys_lmdz_para
    145   use ioipsl
    146   use write_field
     155  USE mod_phys_lmdz_para, only: is_sequential, jj_begin, jj_end, jj_nb
     156  use ioipsl, only: histbeg
    147157  implicit none
    148158  include 'dimensions.h'
    … …  
    173183! SUBROUTINE histbeg_phyxios(name,itau0,zjulian,dtime,ffreq,lev,nhori,nid_day)
    174184 SUBROUTINE histbeg_phyxios(name,ffreq,lev)
    175   USE dimphy
    176   USE mod_phys_lmdz_para
    177 !  use ioipsl
    178   use write_field
     185  USE mod_phys_lmdz_para, only: is_using_mpi, is_mpi_root
     186  use wxios, only: wxios_add_file
    179187  IMPLICIT NONE
    180188  include 'dimensions.h'
    … …  
    206214 
    207215  subroutine histwrite2d_phy(nid,lpoint,name,itau,field)
    208   USE dimphy
    209   USE mod_phys_lmdz_para
    210   USE ioipsl
     216  USE dimphy, only: klon
     217  USE mod_phys_lmdz_para, only: Gather_omp, grid1Dto2D_mpi, &
     218                                is_sequential, klon_mpi_begin, klon_mpi_end, &
     219                                jj_nb, klon_mpi
     220  USE ioipsl, only: histwrite
    211221  implicit none
    212222  include 'dimensions.h'
    … …  
    263273
    264274  subroutine histwrite3d_phy(nid,lpoint,name,itau,field)
    265   USE dimphy
    266   USE mod_phys_lmdz_para
    267 
    268   use ioipsl
     275  USE dimphy, only: klon
     276  USE mod_phys_lmdz_para, only: Gather_omp, grid1Dto2D_mpi, &
     277                                is_sequential, klon_mpi_begin, klon_mpi_end, &
     278                                jj_nb, klon_mpi
     279  USE ioipsl, only: histwrite
    269280  implicit none
    270281  include 'dimensions.h'
    … …  
    283294    IF (size(field,1)/=klon) CALL abort_gcm('iophy::histwrite3d','Field first dimension not equal to klon',1)
    284295    nlev=size(field,2)
    285 
    286 !   print*,'hist3d_phy mpi_rank npstn=',mpi_rank,npstn
    287 
    288 !   DO ip=1, npstn
    289 !    print*,'hist3d_phy mpi_rank nptabij',mpi_rank,nptabij(ip)
    290 !   ENDDO
    291296
    292297    CALL Gather_omp(field,buffer_omp)
    … …  
    332337#ifdef CPP_XIOS
    333338  SUBROUTINE histwrite2d_xios(field_name,field)
    334   USE dimphy
    335   USE mod_phys_lmdz_para
    336   USE wxios
     339  USE dimphy, only: klon
     340  USE mod_phys_lmdz_para, only: gather_omp, grid1Dto2D_mpi, &
     341                                jj_nb, klon_mpi
     342  USE wxios, only: wxios_write_2D
    337343
    338344
    … …  
    345351     
    346352    REAL,DIMENSION(klon_mpi) :: buffer_omp
    347     INTEGER, allocatable, DIMENSION(:) :: index2d
    348353    REAL :: Field2d(iim,jj_nb)
    349354
    350     INTEGER :: ip
    351     REAL, ALLOCATABLE, DIMENSION(:) :: fieldok
    352 
    353     IF (prt_level >= 9) WRITE(lunout,*)'Begin histrwrite2d_xios ',field_name
    354 
    355     !Et sinon on.... écrit
     355    IF (prt_level >= 10) WRITE(lunout,*)'Begin histrwrite2d_xios ',trim(field_name)
     356
    356357    IF (SIZE(field)/=klon) CALL abort_gcm('iophy::histwrite2d_xios','Field first DIMENSION not equal to klon',1)
    357358   
    … …  
    360361    CALL grid1Dto2D_mpi(buffer_omp,Field2d)
    361362   
    362 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    363 !ATTENTION, STATIONS PAS GEREES !
    364 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    365     !IF(.NOT.clef_stations(iff)) THEN
    366     IF (.TRUE.) THEN
    367         ALLOCATE(index2d(iim*jj_nb))
    368         ALLOCATE(fieldok(iim*jj_nb))
    369 
    370 
    371         CALL wxios_write_2D(field_name, Field2d)
    372 
    373     ELSE
    374         ALLOCATE(fieldok(npstn))
    375         ALLOCATE(index2d(npstn))
    376 
    377         IF (is_sequential) THEN
    378             DO ip=1, npstn
    379                 fieldok(ip)=buffer_omp(nptabij(ip))
    380             ENDDO
    381         ELSE
    382             DO ip=1, npstn
    383                 PRINT*,'histwrite2d_xios is_sequential npstn ip namenptabij',npstn,ip,field_name,nptabij(ip)
    384                 IF(nptabij(ip).GE.klon_mpi_begin.AND. &
    385                 nptabij(ip).LE.klon_mpi_end) THEN
    386                     fieldok(ip)=buffer_omp(nptabij(ip)-klon_mpi_begin+1)
    387                 ENDIF
    388             ENDDO
    389         ENDIF
    390 
    391     ENDIF
    392                  
    393     deallocate(index2d)
    394     deallocate(fieldok)
     363    CALL wxios_write_2D(field_name, Field2d)
    395364!$OMP END MASTER   
    396365
    397   IF (prt_level >= 9) WRITE(lunout,*)'End histrwrite2d_xios ',field_name
     366    IF (prt_level >= 10) WRITE(lunout,*)'End histrwrite2d_xios ',trim(field_name)
    398367  END SUBROUTINE histwrite2d_xios
    399368#endif
    … …  
    404373#ifdef CPP_XIOS
    405374  SUBROUTINE histwrite3d_xios(field_name, field)
    406   USE dimphy
    407   USE mod_phys_lmdz_para
    408   USE wxios
     375  USE dimphy, only: klon, klev
     376  USE mod_phys_lmdz_para, only: gather_omp, grid1Dto2D_mpi, &
     377                                jj_nb, klon_mpi
     378  USE wxios, only: wxios_write_3D
    409379
    410380
    … …  
    419389    REAL :: Field3d(iim,jj_nb,SIZE(field,2))
    420390    INTEGER :: ip, n, nlev
    421     INTEGER, ALLOCATABLE, DIMENSION(:) :: index3d
    422     REAL,ALLOCATABLE, DIMENSION(:,:) :: fieldok
    423 
    424   IF (prt_level >= 9) write(lunout,*)'Begin histrwrite3d_xios ',field_name
     391
     392  IF (prt_level >= 10) write(lunout,*)'Begin histrwrite3d_xios ',trim(field_name)
    425393
    426394    !Et on.... écrit
    … …  
    433401    CALL grid1Dto2D_mpi(buffer_omp,field3d)
    434402
    435     IF(.TRUE.)THEN
    436         ALLOCATE(index3d(iim*jj_nb*nlev))
    437         ALLOCATE(fieldok(iim*jj_nb,nlev))
    438         CALL wxios_write_3D(field_name, Field3d(:,:,1:klev))
    439                        
    440     ELSE
    441         nlev=size(field,2)
    442         ALLOCATE(index3d(npstn*nlev))
    443         ALLOCATE(fieldok(npstn,nlev))
    444 
    445         IF (is_sequential) THEN
    446             DO n=1, nlev
    447                 DO ip=1, npstn
    448                     fieldok(ip,n)=buffer_omp(nptabij(ip),n)
    449                 ENDDO
    450             ENDDO
    451         ELSE
    452             DO n=1, nlev
    453                 DO ip=1, npstn
    454                     IF(nptabij(ip).GE.klon_mpi_begin.AND. &
    455                     nptabij(ip).LE.klon_mpi_end) THEN
    456                         fieldok(ip,n)=buffer_omp(nptabij(ip)-klon_mpi_begin+1,n)
    457                     ENDIF
    458                 ENDDO
    459             ENDDO
    460         ENDIF
    461     ENDIF
    462     deallocate(index3d)
    463     deallocate(fieldok)
     403    CALL wxios_write_3D(field_name, Field3d(:,:,1:klev))
    464404!$OMP END MASTER   
    465405
    466   IF (prt_level >= 9) write(lunout,*)'End histrwrite3d_xios ',field_name
     406    IF (prt_level >= 10) write(lunout,*)'End histrwrite3d_xios ',trim(field_name)
    467407  END SUBROUTINE histwrite3d_xios
    468408#endif

  • LMDZ5/trunk/libf/phydev/physiq.F90

    r1882 r1897  
    2121
    2222#ifdef CPP_XIOS
    23       USE wxios
     23      USE wxios, only: wxios_add_vaxis, wxios_set_timestep, wxios_closedef, &
     24                       wxios_update_calendar, histwrite_phy
    2425#endif
    2526
    … …  
    168169
    169170
    170 !print*,'PHYDEV: itau=',itau
     171print*,'PHYDEV: itau=',itau
    171172
    172173! write some outputs:
    … …  
    186187    !Increment XIOS time
    187188    CALL wxios_update_calendar(itau)
     189!$OMP END MASTER
     190!$OMP BARRIER
    188191
    189192    !Send fields to XIOS:
    … …  
    192195    CALL histwrite_phy("v",v)
    193196    CALL histwrite_phy("ps",paprs(:,1))
    194 !$OMP END MASTER
    195197#endif
    196198
    … …  
    200202endif
    201203
    202 end
     204end subroutine physiq
Note: See TracChangeset for help on using the changeset viewer.