| 1 | /* #define LEARN_BCAST */ |
|---|
| 2 | /*********************************************************************** |
|---|
| 3 | |
|---|
| 4 | COPYRIGHT |
|---|
| 5 | |
|---|
| 6 | The following is a notice of limited availability of the code and |
|---|
| 7 | Government license and disclaimer which must be included in the |
|---|
| 8 | prologue of the code and in all source listings of the code. |
|---|
| 9 | |
|---|
| 10 | Copyright notice |
|---|
| 11 | (c) 1977 University of Chicago |
|---|
| 12 | |
|---|
| 13 | Permission is hereby granted to use, reproduce, prepare |
|---|
| 14 | derivative works, and to redistribute to others at no charge. If |
|---|
| 15 | you distribute a copy or copies of the Software, or you modify a |
|---|
| 16 | copy or copies of the Software or any portion of it, thus forming |
|---|
| 17 | a work based on the Software and make and/or distribute copies of |
|---|
| 18 | such work, you must meet the following conditions: |
|---|
| 19 | |
|---|
| 20 | a) If you make a copy of the Software (modified or verbatim) |
|---|
| 21 | it must include the copyright notice and Government |
|---|
| 22 | license and disclaimer. |
|---|
| 23 | |
|---|
| 24 | b) You must cause the modified Software to carry prominent |
|---|
| 25 | notices stating that you changed specified portions of |
|---|
| 26 | the Software. |
|---|
| 27 | |
|---|
| 28 | This software was authored by: |
|---|
| 29 | |
|---|
| 30 | Argonne National Laboratory |
|---|
| 31 | J. Michalakes: (630) 252-6646; email: michalak@mcs.anl.gov |
|---|
| 32 | Mathematics and Computer Science Division |
|---|
| 33 | Argonne National Laboratory, Argonne, IL 60439 |
|---|
| 34 | |
|---|
| 35 | ARGONNE NATIONAL LABORATORY (ANL), WITH FACILITIES IN THE STATES |
|---|
| 36 | OF ILLINOIS AND IDAHO, IS OWNED BY THE UNITED STATES GOVERNMENT, |
|---|
| 37 | AND OPERATED BY THE UNIVERSITY OF CHICAGO UNDER PROVISION OF A |
|---|
| 38 | CONTRACT WITH THE DEPARTMENT OF ENERGY. |
|---|
| 39 | |
|---|
| 40 | GOVERNMENT LICENSE AND DISCLAIMER |
|---|
| 41 | |
|---|
| 42 | This computer code material was prepared, in part, as an account |
|---|
| 43 | of work sponsored by an agency of the United States Government. |
|---|
| 44 | The Government is granted for itself and others acting on its |
|---|
| 45 | behalf a paid-up, nonexclusive, irrevocable worldwide license in |
|---|
| 46 | this data to reproduce, prepare derivative works, distribute |
|---|
| 47 | copies to the public, perform publicly and display publicly, and |
|---|
| 48 | to permit others to do so. NEITHER THE UNITED STATES GOVERNMENT |
|---|
| 49 | NOR ANY AGENCY THEREOF, NOR THE UNIVERSITY OF CHICAGO, NOR ANY OF |
|---|
| 50 | THEIR EMPLOYEES, MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR |
|---|
| 51 | ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE ACCURACY, |
|---|
| 52 | COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, APPARATUS, |
|---|
| 53 | PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD |
|---|
| 54 | NOT INFRINGE PRIVATELY OWNED RIGHTS. |
|---|
| 55 | |
|---|
| 56 | ***************************************************************************/ |
|---|
| 57 | |
|---|
| 58 | #define MOD_9707 |
|---|
| 59 | |
|---|
| 60 | #include <stdio.h> |
|---|
| 61 | #include <stdlib.h> |
|---|
| 62 | #include "mpi.h" |
|---|
| 63 | #include "rsl_lite.h" |
|---|
| 64 | |
|---|
| 65 | char mess[4096] ; |
|---|
| 66 | |
|---|
| 67 | typedef struct bcast_point_desc { |
|---|
| 68 | int ig ; |
|---|
| 69 | int jg ; |
|---|
| 70 | } bcast_point_desc_t ; |
|---|
| 71 | |
|---|
| 72 | |
|---|
| 73 | static destroy_par_info ( p ) |
|---|
| 74 | char * p ; |
|---|
| 75 | { |
|---|
| 76 | if ( p != NULL ) RSL_FREE( p ) ; |
|---|
| 77 | } |
|---|
| 78 | |
|---|
| 79 | static rsl_list_t *Xlist, *Xp, *Xprev ; |
|---|
| 80 | static rsl_list_t *stage ; |
|---|
| 81 | static int stage_len = 0 ; /* 96/3/15 */ |
|---|
| 82 | |
|---|
| 83 | static int Sendbufsize ; |
|---|
| 84 | static int Sendbufcurs ; |
|---|
| 85 | static char *Sendbuf ; |
|---|
| 86 | static int Sdisplacements[RSL_MAXPROC] ; |
|---|
| 87 | static int Ssizes[RSL_MAXPROC] ; |
|---|
| 88 | |
|---|
| 89 | static int Recsizeindex ; |
|---|
| 90 | |
|---|
| 91 | static int Rbufsize ; |
|---|
| 92 | static int Rbufcurs ; |
|---|
| 93 | static int Rpointcurs ; |
|---|
| 94 | static char *Recvbuf ; |
|---|
| 95 | static int Rdisplacements[RSL_MAXPROC+1] ; |
|---|
| 96 | static int Rsizes[RSL_MAXPROC] ; |
|---|
| 97 | static int Rreclen ; |
|---|
| 98 | |
|---|
| 99 | static int s_d ; |
|---|
| 100 | static int s_nst ; |
|---|
| 101 | static int s_msize ; |
|---|
| 102 | static int s_idim ; |
|---|
| 103 | static int s_jdim ; |
|---|
| 104 | static int s_idim_nst ; |
|---|
| 105 | static int s_jdim_nst ; |
|---|
| 106 | static int s_irax_n ; |
|---|
| 107 | static int s_irax_m ; |
|---|
| 108 | static int s_ntasks_x ; |
|---|
| 109 | static int s_ntasks_y ; |
|---|
| 110 | static rsl_list_t **Plist ; |
|---|
| 111 | static int Psize[RSL_MAXPROC] ; |
|---|
| 112 | static char *s_parent_msgs ; |
|---|
| 113 | static int s_parent_msgs_curs ; |
|---|
| 114 | static int s_remaining ; /* number of bytes left in a parent message before |
|---|
| 115 | the next point descriptor */ |
|---|
| 116 | |
|---|
| 117 | /* add a field to a message outgoing for the specified child domain cell */ |
|---|
| 118 | /* relies on rsl_ready_bcast having been called already */ |
|---|
| 119 | /* sends are specified in terms of coarse domain */ |
|---|
| 120 | |
|---|
| 121 | static int s_i, s_j, s_ig, s_jg, s_cm, s_cn, |
|---|
| 122 | s_nig, s_njg ; |
|---|
| 123 | |
|---|
| 124 | static int Pcurs ; |
|---|
| 125 | static rsl_list_t *Pptr ; |
|---|
| 126 | |
|---|
| 127 | #ifdef LEARN_BCAST |
|---|
| 128 | static int s_putmsg = 0 ; |
|---|
| 129 | #endif |
|---|
| 130 | |
|---|
| 131 | /* parent->nest */ |
|---|
| 132 | RSL_LITE_TO_CHILD_INFO ( Fcomm, msize_p, |
|---|
| 133 | cips_p, cipe_p, cjps_p, cjpe_p, /* patch dims of SOURCE DOMAIN */ |
|---|
| 134 | iids_p, iide_p, ijds_p, ijde_p, /* domain dims of INTERMEDIATE DOMAIN */ |
|---|
| 135 | nids_p, nide_p, njds_p, njde_p, /* domain dims of CHILD DOMAIN */ |
|---|
| 136 | pgr_p, shw_p , /* nest ratio and stencil half width */ |
|---|
| 137 | ntasks_x_p , ntasks_y_p , /* proc counts in x and y */ |
|---|
| 138 | icoord_p, jcoord_p, |
|---|
| 139 | idim_cd_p, jdim_cd_p, |
|---|
| 140 | ig_p, jg_p, |
|---|
| 141 | retval_p ) |
|---|
| 142 | |
|---|
| 143 | int_p |
|---|
| 144 | Fcomm /* Fortran version of MPI communicator */ |
|---|
| 145 | ,cips_p, cipe_p, cjps_p, cjpe_p /* (i) c.d. patch dims */ |
|---|
| 146 | ,iids_p, iide_p, ijds_p, ijde_p /* (i) n.n. global dims */ |
|---|
| 147 | ,nids_p, nide_p, njds_p, njde_p /* (i) n.n. global dims */ |
|---|
| 148 | ,pgr_p /* nesting ratio */ |
|---|
| 149 | ,ntasks_x_p , ntasks_y_p /* proc counts in x and y */ |
|---|
| 150 | ,icoord_p /* i coordinate of nest in cd */ |
|---|
| 151 | ,jcoord_p /* j coordinate of nest in cd */ |
|---|
| 152 | ,shw_p /* stencil half width */ |
|---|
| 153 | ,idim_cd_p /* i width of nest in cd */ |
|---|
| 154 | ,jdim_cd_p /* j width of nest in cd */ |
|---|
| 155 | ,msize_p /* (I) Message size in bytes. */ |
|---|
| 156 | ,ig_p /* (O) Global N index of parent domain point. */ |
|---|
| 157 | ,jg_p /* (O) Global N index of parent domain point. */ |
|---|
| 158 | ,retval_p ; /* (O) =1 if a valid point returned; =0 (zero) otherwise. */ |
|---|
| 159 | { |
|---|
| 160 | int P, Px, Py ; |
|---|
| 161 | |
|---|
| 162 | rsl_list_t *q ; |
|---|
| 163 | int *r ; |
|---|
| 164 | int i, j, ni, nj ; |
|---|
| 165 | int coords[2] ; |
|---|
| 166 | MPI_Comm *comm, dummy_comm ; |
|---|
| 167 | |
|---|
| 168 | comm = &dummy_comm ; |
|---|
| 169 | *comm = MPI_Comm_f2c( *Fcomm ) ; |
|---|
| 170 | |
|---|
| 171 | if ( Plist == NULL ) { |
|---|
| 172 | s_ntasks_x = *ntasks_x_p ; |
|---|
| 173 | s_ntasks_y = *ntasks_y_p ; |
|---|
| 174 | /* construct Plist */ |
|---|
| 175 | Sendbufsize = 0 ; |
|---|
| 176 | Plist = RSL_MALLOC( rsl_list_t * , s_ntasks_x * s_ntasks_y ) ; /* big enough for nest points */ |
|---|
| 177 | for ( j = 0 ; j < s_ntasks_x * s_ntasks_y ; j++ ) { |
|---|
| 178 | Plist[j] = NULL ; |
|---|
| 179 | Sdisplacements[j] = 0 ; |
|---|
| 180 | Ssizes[j] = 0 ; |
|---|
| 181 | } |
|---|
| 182 | for ( j = *cjps_p ; j <= *cjpe_p ; j++ ) |
|---|
| 183 | { |
|---|
| 184 | for ( i = *cips_p ; i <= *cipe_p ; i++ ) |
|---|
| 185 | { |
|---|
| 186 | if ( ( *jcoord_p <= j && j <= *jcoord_p+*jdim_cd_p-1 ) && ( *icoord_p <= i && i <= *icoord_p+*idim_cd_p-1 ) ) { |
|---|
| 187 | ni = ( i - (*icoord_p + *shw_p) ) * *pgr_p + 1 + 1 ; /* add 1 to give center point */ |
|---|
| 188 | nj = ( j - (*jcoord_p + *shw_p) ) * *pgr_p + 1 + 1 ; |
|---|
| 189 | |
|---|
| 190 | TASK_FOR_POINT ( &ni, &nj, nids_p, nide_p, njds_p, njde_p, &s_ntasks_x, &s_ntasks_y, &Px, &Py ) ; |
|---|
| 191 | coords[1] = Px ; coords[0] = Py ; |
|---|
| 192 | MPI_Cart_rank( *comm, coords, &P ) ; |
|---|
| 193 | |
|---|
| 194 | q = RSL_MALLOC( rsl_list_t , 1 ) ; |
|---|
| 195 | q->info1 = i ; |
|---|
| 196 | q->info2 = j ; |
|---|
| 197 | q->next = Plist[P] ; |
|---|
| 198 | Plist[P] = q ; |
|---|
| 199 | Sendbufsize += *msize_p + 3 * sizeof( int ) ; /* point data plus 3 ints for i, j, and size */ |
|---|
| 200 | } |
|---|
| 201 | } |
|---|
| 202 | } |
|---|
| 203 | Sendbuf = RSL_MALLOC( char , Sendbufsize ) ; |
|---|
| 204 | Sendbufcurs = 0 ; |
|---|
| 205 | Recsizeindex = -1 ; |
|---|
| 206 | Pcurs = -1 ; |
|---|
| 207 | Pptr = NULL ; |
|---|
| 208 | } |
|---|
| 209 | |
|---|
| 210 | if ( Pptr != NULL ) { |
|---|
| 211 | Pptr = Pptr->next ; |
|---|
| 212 | } |
|---|
| 213 | |
|---|
| 214 | if ( Recsizeindex >= 0 ) { |
|---|
| 215 | r = (int *) &(Sendbuf[Recsizeindex]) ; |
|---|
| 216 | *r = Sendbufcurs - Recsizeindex + 2 * sizeof(int) ; |
|---|
| 217 | Ssizes[Pcurs] += *r ; |
|---|
| 218 | } |
|---|
| 219 | |
|---|
| 220 | while ( Pptr == NULL ) { |
|---|
| 221 | Pcurs++ ; |
|---|
| 222 | while ( Pcurs < s_ntasks_x * s_ntasks_y && Plist[Pcurs] == NULL ) Pcurs++ ; |
|---|
| 223 | if ( Pcurs < s_ntasks_x * s_ntasks_y ) { |
|---|
| 224 | Sdisplacements[Pcurs] = Sendbufcurs ; |
|---|
| 225 | Ssizes[Pcurs] = 0 ; |
|---|
| 226 | Pptr = Plist[Pcurs] ; |
|---|
| 227 | } else { |
|---|
| 228 | *retval_p = 0 ; |
|---|
| 229 | #if 0 |
|---|
| 230 | fprintf(stderr,"TO _INFO: %d %d %d \n",*ig_p,*jg_p, *retval_p) ; |
|---|
| 231 | #endif |
|---|
| 232 | return ; /* done */ |
|---|
| 233 | } |
|---|
| 234 | } |
|---|
| 235 | |
|---|
| 236 | *ig_p = Pptr->info1 ; |
|---|
| 237 | *jg_p = Pptr->info2 ; |
|---|
| 238 | |
|---|
| 239 | r = (int *) &(Sendbuf[Sendbufcurs]) ; |
|---|
| 240 | *r++ = Pptr->info1 ; Sendbufcurs += sizeof(int) ; /* ig to buffer */ |
|---|
| 241 | *r++ = Pptr->info2 ; Sendbufcurs += sizeof(int) ; /* jg to buffer */ |
|---|
| 242 | Recsizeindex = Sendbufcurs ; |
|---|
| 243 | *r++ = 0 ; Sendbufcurs += sizeof(int) ; /* store start for size */ |
|---|
| 244 | *retval_p = 1 ; |
|---|
| 245 | |
|---|
| 246 | #if 0 |
|---|
| 247 | fprintf(stderr,"TO INFO: %d %d %d \n",*ig_p,*jg_p, *retval_p) ; |
|---|
| 248 | #endif |
|---|
| 249 | |
|---|
| 250 | return ; |
|---|
| 251 | } |
|---|
| 252 | |
|---|
| 253 | /********************************************/ |
|---|
| 254 | |
|---|
| 255 | /* nest->parent */ |
|---|
| 256 | RSL_LITE_TO_PARENT_INFO ( Fcomm, msize_p, |
|---|
| 257 | nips_p, nipe_p, njps_p, njpe_p, /* patch dims of SOURCE DOMAIN (CHILD) */ |
|---|
| 258 | cids_p, cide_p, cjds_p, cjde_p, /* domain dims of TARGET DOMAIN (PARENT) */ |
|---|
| 259 | ntasks_x_p , ntasks_y_p , /* proc counts in x and y */ |
|---|
| 260 | icoord_p, jcoord_p, |
|---|
| 261 | idim_cd_p, jdim_cd_p, |
|---|
| 262 | ig_p, jg_p, |
|---|
| 263 | retval_p ) |
|---|
| 264 | int_p |
|---|
| 265 | Fcomm /* Fortran version of MPI communicator */ |
|---|
| 266 | ,nips_p, nipe_p, njps_p, njpe_p /* (i) n.d. patch dims */ |
|---|
| 267 | ,cids_p, cide_p, cjds_p, cjde_p /* (i) n.n. global dims */ |
|---|
| 268 | ,ntasks_x_p , ntasks_y_p /* proc counts in x and y */ |
|---|
| 269 | ,icoord_p /* i coordinate of nest in cd */ |
|---|
| 270 | ,jcoord_p /* j coordinate of nest in cd */ |
|---|
| 271 | ,idim_cd_p /* i width of nest in cd */ |
|---|
| 272 | ,jdim_cd_p /* j width of nest in cd */ |
|---|
| 273 | ,msize_p /* (I) Message size in bytes. */ |
|---|
| 274 | ,ig_p /* (O) Global N index of parent domain point. */ |
|---|
| 275 | ,jg_p /* (O) Global N index of parent domain point. */ |
|---|
| 276 | ,retval_p ; /* (O) =1 if a valid point returned; =0 (zero) otherwise. */ |
|---|
| 277 | { |
|---|
| 278 | int P, Px, Py ; |
|---|
| 279 | rsl_list_t *q ; |
|---|
| 280 | int *r ; |
|---|
| 281 | int i, j ; |
|---|
| 282 | int coords[2] ; |
|---|
| 283 | MPI_Comm *comm, dummy_comm ; |
|---|
| 284 | |
|---|
| 285 | comm = &dummy_comm ; |
|---|
| 286 | *comm = MPI_Comm_f2c( *Fcomm ) ; |
|---|
| 287 | |
|---|
| 288 | if ( Plist == NULL ) { |
|---|
| 289 | s_ntasks_x = *ntasks_x_p ; |
|---|
| 290 | s_ntasks_y = *ntasks_y_p ; |
|---|
| 291 | /* construct Plist */ |
|---|
| 292 | Sendbufsize = 0 ; |
|---|
| 293 | Plist = RSL_MALLOC( rsl_list_t * , s_ntasks_x * s_ntasks_y ) ; |
|---|
| 294 | for ( j = 0 ; j < s_ntasks_x * s_ntasks_y ; j++ ) { |
|---|
| 295 | Plist[j] = NULL ; |
|---|
| 296 | Sdisplacements[j] = 0 ; |
|---|
| 297 | Ssizes[j] = 0 ; |
|---|
| 298 | } |
|---|
| 299 | for ( j = *njps_p ; j <= *njpe_p ; j++ ) |
|---|
| 300 | { |
|---|
| 301 | for ( i = *nips_p ; i <= *nipe_p ; i++ ) |
|---|
| 302 | { |
|---|
| 303 | if ( ( *jcoord_p <= j && j <= *jcoord_p+*jdim_cd_p-1 ) && ( *icoord_p <= i && i <= *icoord_p+*idim_cd_p-1 ) ) { |
|---|
| 304 | TASK_FOR_POINT ( &i, &j, cids_p, cide_p, cjds_p, cjde_p, &s_ntasks_x, &s_ntasks_y, &Px, &Py ) ; |
|---|
| 305 | coords[1] = Px ; coords[0] = Py ; |
|---|
| 306 | MPI_Cart_rank( *comm, coords, &P ) ; |
|---|
| 307 | q = RSL_MALLOC( rsl_list_t , 1 ) ; |
|---|
| 308 | q->info1 = i ; |
|---|
| 309 | q->info2 = j ; |
|---|
| 310 | q->next = Plist[P] ; |
|---|
| 311 | Plist[P] = q ; |
|---|
| 312 | Sendbufsize += *msize_p + 3 * sizeof( int ) ; /* point data plus 3 ints for i, j, and size */ |
|---|
| 313 | } |
|---|
| 314 | } |
|---|
| 315 | } |
|---|
| 316 | Sendbuf = RSL_MALLOC( char , Sendbufsize ) ; |
|---|
| 317 | Sendbufcurs = 0 ; |
|---|
| 318 | Recsizeindex = -1 ; |
|---|
| 319 | Pcurs = -1 ; |
|---|
| 320 | Pptr = NULL ; |
|---|
| 321 | } |
|---|
| 322 | if ( Pptr != NULL ) { |
|---|
| 323 | Pptr = Pptr->next ; |
|---|
| 324 | } |
|---|
| 325 | |
|---|
| 326 | if ( Recsizeindex >= 0 ) { |
|---|
| 327 | r = (int *) &(Sendbuf[Recsizeindex]) ; |
|---|
| 328 | *r = Sendbufcurs - Recsizeindex + 2 * sizeof(int) ; |
|---|
| 329 | Ssizes[Pcurs] += *r ; |
|---|
| 330 | } |
|---|
| 331 | |
|---|
| 332 | while ( Pptr == NULL ) { |
|---|
| 333 | Pcurs++ ; |
|---|
| 334 | while ( Pcurs < s_ntasks_x * s_ntasks_y && Plist[Pcurs] == NULL ) Pcurs++ ; |
|---|
| 335 | if ( Pcurs < s_ntasks_x * s_ntasks_y ) { |
|---|
| 336 | Sdisplacements[Pcurs] = Sendbufcurs ; |
|---|
| 337 | Ssizes[Pcurs] = 0 ; |
|---|
| 338 | Pptr = Plist[Pcurs] ; |
|---|
| 339 | } else { |
|---|
| 340 | *retval_p = 0 ; |
|---|
| 341 | return ; /* done */ |
|---|
| 342 | } |
|---|
| 343 | } |
|---|
| 344 | |
|---|
| 345 | *ig_p = Pptr->info1 ; |
|---|
| 346 | *jg_p = Pptr->info2 ; |
|---|
| 347 | |
|---|
| 348 | r = (int *) &(Sendbuf[Sendbufcurs]) ; |
|---|
| 349 | *r++ = Pptr->info1 ; Sendbufcurs += sizeof(int) ; /* ig to buffer */ |
|---|
| 350 | *r++ = Pptr->info2 ; Sendbufcurs += sizeof(int) ; /* jg to buffer */ |
|---|
| 351 | Recsizeindex = Sendbufcurs ; |
|---|
| 352 | *r++ = 0 ; Sendbufcurs += sizeof(int) ; /* store start for size */ |
|---|
| 353 | *retval_p = 1 ; |
|---|
| 354 | |
|---|
| 355 | return ; |
|---|
| 356 | } |
|---|
| 357 | |
|---|
| 358 | |
|---|
| 359 | /********************************************/ |
|---|
| 360 | |
|---|
| 361 | /*@ |
|---|
| 362 | RSL_TO_CHILD_MSG -- Pack force data into a message for a nest point. |
|---|
| 363 | |
|---|
| 364 | @*/ |
|---|
| 365 | |
|---|
| 366 | /* parent->nest */ |
|---|
| 367 | RSL_LITE_TO_CHILD_MSG ( nbuf_p, buf ) |
|---|
| 368 | int_p |
|---|
| 369 | nbuf_p ; /* (I) Number of bytes to be packed. */ |
|---|
| 370 | char * |
|---|
| 371 | buf ; /* (I) Buffer containing the data to be packed. */ |
|---|
| 372 | { |
|---|
| 373 | rsl_lite_to_peerpoint_msg ( nbuf_p, buf ) ; |
|---|
| 374 | } |
|---|
| 375 | |
|---|
| 376 | /* nest->parent */ |
|---|
| 377 | RSL_LITE_TO_PARENT_MSG ( nbuf_p, buf ) |
|---|
| 378 | int_p |
|---|
| 379 | nbuf_p ; /* (I) Number of bytes to be packed. */ |
|---|
| 380 | char * |
|---|
| 381 | buf ; /* (I) Buffer containing the data to be packed. */ |
|---|
| 382 | { |
|---|
| 383 | rsl_lite_to_peerpoint_msg ( nbuf_p, buf ) ; |
|---|
| 384 | } |
|---|
| 385 | |
|---|
| 386 | /* common code */ |
|---|
| 387 | rsl_lite_to_peerpoint_msg ( nbuf_p, buf ) |
|---|
| 388 | int_p |
|---|
| 389 | nbuf_p ; /* (I) Number of bytes to be packed. */ |
|---|
| 390 | char * |
|---|
| 391 | buf ; /* (I) Buffer containing the data to be packed. */ |
|---|
| 392 | { |
|---|
| 393 | int nbuf ; |
|---|
| 394 | int *p, *q ; |
|---|
| 395 | char *c, *d ; |
|---|
| 396 | int i ; |
|---|
| 397 | |
|---|
| 398 | RSL_TEST_ERR(buf==NULL,"2nd argument is NULL. Field allocated?") ; |
|---|
| 399 | |
|---|
| 400 | nbuf = *nbuf_p ; |
|---|
| 401 | |
|---|
| 402 | if ( Sendbufcurs + nbuf >= Sendbufsize ) { |
|---|
| 403 | sprintf(mess,"RSL_LITE_TO_CHILD_MSG: Sendbufcurs + nbuf (%d) would exceed Sendbufsize (%d)\n", |
|---|
| 404 | Sendbufcurs + nbuf , Sendbufsize ) ; |
|---|
| 405 | RSL_TEST_ERR(1,mess) ; |
|---|
| 406 | } |
|---|
| 407 | |
|---|
| 408 | if ( nbuf % sizeof(int) == 0 ) { |
|---|
| 409 | for ( p = (int *)buf, q = (int *) &(Sendbuf[Sendbufcurs]), i = 0 ; i < nbuf ; i += sizeof(int) ) |
|---|
| 410 | { |
|---|
| 411 | *q++ = *p++ ; |
|---|
| 412 | } |
|---|
| 413 | } |
|---|
| 414 | else |
|---|
| 415 | { |
|---|
| 416 | for ( c = buf, d = &(Sendbuf[Sendbufcurs]), i = 0 ; i < nbuf ; i++ ) |
|---|
| 417 | { |
|---|
| 418 | *d++ = *c++ ; |
|---|
| 419 | } |
|---|
| 420 | } |
|---|
| 421 | |
|---|
| 422 | Sendbufcurs += nbuf ; |
|---|
| 423 | |
|---|
| 424 | } |
|---|
| 425 | |
|---|
| 426 | /********************************************/ |
|---|
| 427 | |
|---|
| 428 | /* parent->nest */ |
|---|
| 429 | RSL_LITE_BCAST_MSGS ( mytask_p, ntasks_p, Fcomm ) |
|---|
| 430 | int_p mytask_p, ntasks_p, Fcomm ; |
|---|
| 431 | { |
|---|
| 432 | MPI_Comm comm ; |
|---|
| 433 | |
|---|
| 434 | comm = MPI_Comm_f2c( *Fcomm ) ; |
|---|
| 435 | rsl_lite_allgather_msgs ( mytask_p, ntasks_p, comm ) ; |
|---|
| 436 | } |
|---|
| 437 | |
|---|
| 438 | /* nest->parent */ |
|---|
| 439 | RSL_LITE_MERGE_MSGS ( mytask_p, ntasks_p, Fcomm ) |
|---|
| 440 | int_p mytask_p, ntasks_p, Fcomm ; |
|---|
| 441 | { |
|---|
| 442 | MPI_Comm comm ; |
|---|
| 443 | |
|---|
| 444 | comm = MPI_Comm_f2c( *Fcomm ) ; |
|---|
| 445 | rsl_lite_allgather_msgs ( mytask_p, ntasks_p, comm ) ; |
|---|
| 446 | } |
|---|
| 447 | |
|---|
| 448 | /* common code */ |
|---|
| 449 | rsl_lite_allgather_msgs ( mytask_p, ntasks_p, comm ) |
|---|
| 450 | int_p mytask_p, ntasks_p ; |
|---|
| 451 | MPI_Comm comm ; |
|---|
| 452 | { |
|---|
| 453 | int P ; |
|---|
| 454 | char *work ; |
|---|
| 455 | int * r ; |
|---|
| 456 | bcast_point_desc_t pdesc ; |
|---|
| 457 | int curs ; |
|---|
| 458 | int msglen, mdest, mtag ; |
|---|
| 459 | int ntasks, mytask ; |
|---|
| 460 | int ii, i, j ; |
|---|
| 461 | int ig, jg ; |
|---|
| 462 | int *Psize_all ; |
|---|
| 463 | int *sp, *bp ; |
|---|
| 464 | int rc ; |
|---|
| 465 | |
|---|
| 466 | ntasks = *ntasks_p ; |
|---|
| 467 | mytask = *mytask_p ; |
|---|
| 468 | |
|---|
| 469 | RSL_TEST_ERR( Plist == NULL, |
|---|
| 470 | "RSL_BCAST_MSGS: rsl_to_child_info not called first" ) ; |
|---|
| 471 | |
|---|
| 472 | RSL_TEST_ERR( ntasks == RSL_MAXPROC , |
|---|
| 473 | "RSL_BCAST_MSGS: raise the compile time value of MAXPROC" ) ; |
|---|
| 474 | |
|---|
| 475 | Psize_all = RSL_MALLOC( int, ntasks * ntasks ) ; |
|---|
| 476 | |
|---|
| 477 | MPI_Allgather( Ssizes, ntasks, MPI_INT , Psize_all, ntasks, MPI_INT, comm ) ; |
|---|
| 478 | |
|---|
| 479 | for ( j = 0 ; j < ntasks ; j++ ) |
|---|
| 480 | Rsizes[j] = 0 ; |
|---|
| 481 | |
|---|
| 482 | for ( j = 0 ; j < ntasks ; j++ ) |
|---|
| 483 | { |
|---|
| 484 | Rsizes[j] += Psize_all[ INDEX_2( j , mytask , ntasks ) ] ; |
|---|
| 485 | } |
|---|
| 486 | |
|---|
| 487 | for ( Rbufsize = 0, P = 0, Rdisplacements[0] = 0 ; P < ntasks ; P++ ) |
|---|
| 488 | { |
|---|
| 489 | Rdisplacements[P+1] = Rsizes[P] + Rdisplacements[P] ; |
|---|
| 490 | Rbufsize += Rsizes[P] ; |
|---|
| 491 | } |
|---|
| 492 | |
|---|
| 493 | /* this will be freed later */ |
|---|
| 494 | |
|---|
| 495 | Recvbuf = RSL_MALLOC( char , Rbufsize + 3 * sizeof(int) ) ; /* for sentinal record */ |
|---|
| 496 | Rbufcurs = 0 ; |
|---|
| 497 | Rreclen = 0 ; |
|---|
| 498 | |
|---|
| 499 | rc = MPI_Alltoallv ( Sendbuf, Ssizes, Sdisplacements, MPI_BYTE , |
|---|
| 500 | Recvbuf, Rsizes, Rdisplacements, MPI_BYTE , comm ) ; |
|---|
| 501 | |
|---|
| 502 | /* add sentinel to the end of Recvbuf */ |
|---|
| 503 | |
|---|
| 504 | r = (int *)&(Recvbuf[Rbufsize + 2 * sizeof(int)]) ; |
|---|
| 505 | *r = RSL_INVALID ; |
|---|
| 506 | |
|---|
| 507 | RSL_FREE( Sendbuf ) ; |
|---|
| 508 | RSL_FREE( Psize_all ) ; |
|---|
| 509 | |
|---|
| 510 | for ( j = 0 ; j < *ntasks_p ; j++ ) { |
|---|
| 511 | destroy_list ( &(Plist[j]), NULL ) ; |
|---|
| 512 | } |
|---|
| 513 | RSL_FREE( Plist ) ; |
|---|
| 514 | Plist = NULL ; |
|---|
| 515 | |
|---|
| 516 | } |
|---|
| 517 | |
|---|
| 518 | /********************************************/ |
|---|
| 519 | |
|---|
| 520 | /* parent->nest */ |
|---|
| 521 | RSL_LITE_FROM_PARENT_INFO ( ig_p, jg_p, retval_p ) |
|---|
| 522 | int_p |
|---|
| 523 | ig_p /* (O) Global index in M dimension of nest. */ |
|---|
| 524 | ,jg_p /* (O) Global index in N dimension of nest. */ |
|---|
| 525 | ,retval_p ; /* (O) Return value; =1 valid point, =0 done. */ |
|---|
| 526 | { |
|---|
| 527 | rsl_lite_from_peerpoint_info ( ig_p, jg_p, retval_p ) ; |
|---|
| 528 | } |
|---|
| 529 | |
|---|
| 530 | /* nest->parent */ |
|---|
| 531 | RSL_LITE_FROM_CHILD_INFO ( ig_p, jg_p, retval_p ) |
|---|
| 532 | int_p |
|---|
| 533 | ig_p /* (O) Global index in M dimension of nest. */ |
|---|
| 534 | ,jg_p /* (O) Global index in N dimension of nest. */ |
|---|
| 535 | ,retval_p ; /* (O) Return value; =1 valid point, =0 done. */ |
|---|
| 536 | { |
|---|
| 537 | rsl_lite_from_peerpoint_info ( ig_p, jg_p, retval_p ) ; |
|---|
| 538 | } |
|---|
| 539 | |
|---|
| 540 | /* common code */ |
|---|
| 541 | rsl_lite_from_peerpoint_info ( ig_p, jg_p, retval_p ) |
|---|
| 542 | int_p |
|---|
| 543 | ig_p /* (O) Global index in M dimension of nest. */ |
|---|
| 544 | ,jg_p /* (O) Global index in N dimension of nest. */ |
|---|
| 545 | ,retval_p ; /* (O) Return value; =1 valid point, =0 done. */ |
|---|
| 546 | { |
|---|
| 547 | int ii ; |
|---|
| 548 | |
|---|
| 549 | Rbufcurs = Rbufcurs + Rreclen ; |
|---|
| 550 | Rpointcurs = 0 ; |
|---|
| 551 | *ig_p = *(int *)&( Recvbuf[Rbufcurs + Rpointcurs ] ) ; Rpointcurs += sizeof(int) ; |
|---|
| 552 | *jg_p = *(int *)&( Recvbuf[Rbufcurs + Rpointcurs ] ) ; Rpointcurs += sizeof(int) ; |
|---|
| 553 | /* read sentinel */ |
|---|
| 554 | Rreclen = *(int *)&( Recvbuf[Rbufcurs + Rpointcurs ] ) ; Rpointcurs += sizeof(int) ; |
|---|
| 555 | *retval_p = 1 ; |
|---|
| 556 | if ( Rreclen == RSL_INVALID ) { |
|---|
| 557 | *retval_p = 0 ; |
|---|
| 558 | RSL_FREE( Recvbuf ) ; |
|---|
| 559 | } |
|---|
| 560 | |
|---|
| 561 | #if 0 |
|---|
| 562 | fprintf(stderr,"FROM INFO: %d %d %d %d %d %d\n",*ig_p,*jg_p,Rreclen, Rpointcurs, Rbufcurs + Rpointcurs, *retval_p) ; |
|---|
| 563 | #endif |
|---|
| 564 | return ; |
|---|
| 565 | } |
|---|
| 566 | |
|---|
| 567 | /********************************************/ |
|---|
| 568 | |
|---|
| 569 | /* parent->nest */ |
|---|
| 570 | RSL_LITE_FROM_PARENT_MSG ( len_p, buf ) |
|---|
| 571 | int_p |
|---|
| 572 | len_p ; /* (I) Number of bytes to unpack. */ |
|---|
| 573 | int * |
|---|
| 574 | buf ; /* (O) Destination buffer. */ |
|---|
| 575 | { |
|---|
| 576 | rsl_lite_from_peerpoint_msg ( len_p, buf ) ; |
|---|
| 577 | } |
|---|
| 578 | |
|---|
| 579 | /* nest->parent */ |
|---|
| 580 | RSL_LITE_FROM_CHILD_MSG ( len_p, buf ) |
|---|
| 581 | int_p |
|---|
| 582 | len_p ; /* (I) Number of bytes to unpack. */ |
|---|
| 583 | int * |
|---|
| 584 | buf ; /* (O) Destination buffer. */ |
|---|
| 585 | { |
|---|
| 586 | rsl_lite_from_peerpoint_msg ( len_p, buf ) ; |
|---|
| 587 | } |
|---|
| 588 | |
|---|
| 589 | /* common code */ |
|---|
| 590 | rsl_lite_from_peerpoint_msg ( len_p, buf ) |
|---|
| 591 | int_p |
|---|
| 592 | len_p ; /* (I) Number of bytes to unpack. */ |
|---|
| 593 | int * |
|---|
| 594 | buf ; /* (O) Destination buffer. */ |
|---|
| 595 | { |
|---|
| 596 | int *p, *q ; |
|---|
| 597 | char *c, *d ; |
|---|
| 598 | int i ; |
|---|
| 599 | |
|---|
| 600 | if ( *len_p % sizeof(int) == 0 ) { |
|---|
| 601 | for ( p = (int *)&(Recvbuf[Rbufcurs+Rpointcurs]), q = buf , i = 0 ; i < *len_p ; i += sizeof(int) ) |
|---|
| 602 | { |
|---|
| 603 | *q++ = *p++ ; |
|---|
| 604 | } |
|---|
| 605 | } else { |
|---|
| 606 | for ( c = &(Recvbuf[Rbufcurs+Rpointcurs]), d = (char *) buf , i = 0 ; i < *len_p ; i++ ) |
|---|
| 607 | { |
|---|
| 608 | *d++ = *c++ ; |
|---|
| 609 | } |
|---|
| 610 | } |
|---|
| 611 | |
|---|
| 612 | Rpointcurs += *len_p ; |
|---|
| 613 | } |
|---|
| 614 | |
|---|
| 615 | /********************************************/ |
|---|
| 616 | |
|---|
| 617 | destroy_list( list, dfcn ) |
|---|
| 618 | rsl_list_t ** list ; /* pointer to pointer to list */ |
|---|
| 619 | int (*dfcn)() ; /* pointer to function for destroying |
|---|
| 620 | the data field of the list */ |
|---|
| 621 | { |
|---|
| 622 | rsl_list_t *p, *trash ; |
|---|
| 623 | if ( list == NULL ) return(0) ; |
|---|
| 624 | if ( *list == NULL ) return(0) ; |
|---|
| 625 | for ( p = *list ; p != NULL ; ) |
|---|
| 626 | { |
|---|
| 627 | if ( dfcn != NULL ) (*dfcn)( p->data ) ; |
|---|
| 628 | trash = p ; |
|---|
| 629 | p = p->next ; |
|---|
| 630 | RSL_FREE( trash ) ; |
|---|
| 631 | } |
|---|
| 632 | *list = NULL ; |
|---|
| 633 | return(0) ; |
|---|
| 634 | } |
|---|
| 635 | |
|---|
| 636 | /********************************************/ |
|---|