Actual source code: da3.c

  1: #define PETSCDM_DLL
  2: /*
  3:    Code for manipulating distributed regular 3d arrays in parallel.
  4:    File created by Peter Mell  7/14/95
  5:  */

 7:  #include ../src/dm/da/daimpl.h

 11: PetscErrorCode DAView_3d(DA da,PetscViewer viewer)
 12: {
 14:   PetscMPIInt    rank;
 15:   PetscTruth     iascii,isdraw;

 18:   MPI_Comm_rank(((PetscObject)da)->comm,&rank);

 20:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
 21:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_DRAW,&isdraw);
 22:   if (iascii) {
 23:     PetscViewerFormat format;

 25:     PetscViewerGetFormat(viewer, &format);
 26:     if (format != PETSC_VIEWER_ASCII_VTK && format != PETSC_VIEWER_ASCII_VTK_CELL) {
 27:       PetscViewerASCIISynchronizedPrintf(viewer,"Processor [%d] M %D N %D P %D m %D n %D p %D w %D s %D\n",
 28:                                                 rank,da->M,da->N,da->P,da->m,da->n,da->p,da->w,da->s);
 29:       PetscViewerASCIISynchronizedPrintf(viewer,"X range of indices: %D %D, Y range of indices: %D %D, Z range of indices: %D %D\n",
 30:                                                 da->xs,da->xe,da->ys,da->ye,da->zs,da->ze);
 31: #if !defined(PETSC_USE_COMPLEX)
 32:       if (da->coordinates) {
 33:         PetscInt  last;
 34:         PetscReal *coors;
 35:         VecGetArray(da->coordinates,&coors);
 36:         VecGetLocalSize(da->coordinates,&last);
 37:         last = last - 3;
 38:         PetscViewerASCIISynchronizedPrintf(viewer,"Lower left corner %G %G %G : Upper right %G %G %G\n",
 39:                                                   coors[0],coors[1],coors[2],coors[last],coors[last+1],coors[last+2]);
 40:         VecRestoreArray(da->coordinates,&coors);
 41:       }
 42: #endif
 43:       PetscViewerFlush(viewer);
 44:     }
 45:   } else if (isdraw) {
 46:     PetscDraw       draw;
 47:     PetscReal     ymin = -1.0,ymax = (PetscReal)da->N;
 48:     PetscReal     xmin = -1.0,xmax = (PetscReal)((da->M+2)*da->P),x,y,ycoord,xcoord;
 49:     PetscInt        k,plane,base,*idx;
 50:     char       node[10];
 51:     PetscTruth isnull;

 53:     PetscViewerDrawGetDraw(viewer,0,&draw);
 54:     PetscDrawIsNull(draw,&isnull); if (isnull) return(0);
 55:     PetscDrawSetCoordinates(draw,xmin,ymin,xmax,ymax);
 56:     PetscDrawSynchronizedClear(draw);

 58:     /* first processor draw all node lines */
 59:     if (!rank) {
 60:       for (k=0; k<da->P; k++) {
 61:         ymin = 0.0; ymax = (PetscReal)(da->N - 1);
 62:         for (xmin=(PetscReal)(k*(da->M+1)); xmin<(PetscReal)(da->M+(k*(da->M+1))); xmin++) {
 63:           PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_BLACK);
 64:         }
 65: 
 66:         xmin = (PetscReal)(k*(da->M+1)); xmax = xmin + (PetscReal)(da->M - 1);
 67:         for (ymin=0; ymin<(PetscReal)da->N; ymin++) {
 68:           PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_BLACK);
 69:         }
 70:       }
 71:     }
 72:     PetscDrawSynchronizedFlush(draw);
 73:     PetscDrawPause(draw);

 75:     for (k=0; k<da->P; k++) {  /*Go through and draw for each plane*/
 76:       if ((k >= da->zs) && (k < da->ze)) {
 77:         /* draw my box */
 78:         ymin = da->ys;
 79:         ymax = da->ye - 1;
 80:         xmin = da->xs/da->w    + (da->M+1)*k;
 81:         xmax =(da->xe-1)/da->w + (da->M+1)*k;

 83:         PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_RED);
 84:         PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_RED);
 85:         PetscDrawLine(draw,xmin,ymax,xmax,ymax,PETSC_DRAW_RED);
 86:         PetscDrawLine(draw,xmax,ymin,xmax,ymax,PETSC_DRAW_RED);

 88:         xmin = da->xs/da->w;
 89:         xmax =(da->xe-1)/da->w;

 91:         /* put in numbers*/
 92:         base = (da->base+(da->xe-da->xs)*(da->ye-da->ys)*(k-da->zs))/da->w;

 94:         /* Identify which processor owns the box */
 95:         sprintf(node,"%d",rank);
 96:         PetscDrawString(draw,xmin+(da->M+1)*k+.2,ymin+.3,PETSC_DRAW_RED,node);

 98:         for (y=ymin; y<=ymax; y++) {
 99:           for (x=xmin+(da->M+1)*k; x<=xmax+(da->M+1)*k; x++) {
100:             sprintf(node,"%d",(int)base++);
101:             PetscDrawString(draw,x,y,PETSC_DRAW_BLACK,node);
102:           }
103:         }
104: 
105:       }
106:     }
107:     PetscDrawSynchronizedFlush(draw);
108:     PetscDrawPause(draw);

110:     for (k=0-da->s; k<da->P+da->s; k++) {
111:       /* Go through and draw for each plane */
112:       if ((k >= da->Zs) && (k < da->Ze)) {
113: 
114:         /* overlay ghost numbers, useful for error checking */
115:         base = (da->Xe-da->Xs)*(da->Ye-da->Ys)*(k-da->Zs); idx = da->idx;
116:         plane=k;
117:         /* Keep z wrap around points on the dradrawg */
118:         if (k<0)    { plane=da->P+k; }
119:         if (k>=da->P) { plane=k-da->P; }
120:         ymin = da->Ys; ymax = da->Ye;
121:         xmin = (da->M+1)*plane*da->w;
122:         xmax = (da->M+1)*plane*da->w+da->M*da->w;
123:         for (y=ymin; y<ymax; y++) {
124:           for (x=xmin+da->Xs; x<xmin+da->Xe; x+=da->w) {
125:             sprintf(node,"%d",(int)(idx[base]/da->w));
126:             ycoord = y;
127:             /*Keep y wrap around points on drawing */
128:             if (y<0)      { ycoord = da->N+y; }

130:             if (y>=da->N) { ycoord = y-da->N; }
131:             xcoord = x;   /* Keep x wrap points on drawing */

133:             if (x<xmin)  { xcoord = xmax - (xmin-x); }
134:             if (x>=xmax) { xcoord = xmin + (x-xmax); }
135:             PetscDrawString(draw,xcoord/da->w,ycoord,PETSC_DRAW_BLUE,node);
136:             base+=da->w;
137:           }
138:         }
139:       }
140:     }
141:     PetscDrawSynchronizedFlush(draw);
142:     PetscDrawPause(draw);
143:   } else {
144:     SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for DA 3d",((PetscObject)viewer)->type_name);
145:   }
146:   return(0);
147: }

149: #if 0
150: EXTERN PetscErrorCode DAPublish_Petsc(PetscObject);
151: #endif

155: /*@C
156:    DACreate3d - Creates an object that will manage the communication of three-dimensional 
157:    regular array data that is distributed across some processors.

159:    Collective on MPI_Comm

161:    Input Parameters:
162: +  comm - MPI communicator
163: .  wrap - type of periodicity the array should have, if any.  Use one
164:           of DA_NONPERIODIC, DA_XPERIODIC, DA_YPERIODIC, DA_XYPERIODIC, DA_XYZPERIODIC, DA_XZPERIODIC, or DA_YZPERIODIC.
165: .  stencil_type - Type of stencil (DA_STENCIL_STAR or DA_STENCIL_BOX)
166: .  M,N,P - global dimension in each direction of the array (use -M, -N, and or -P to indicate that it may be set to a different value 
167:             from the command line with -da_grid_x <M> -da_grid_y <N> -da_grid_z <P>)
168: .  m,n,p - corresponding number of processors in each dimension 
169:            (or PETSC_DECIDE to have calculated)
170: .  dof - number of degrees of freedom per node
171: .  lx, ly, lz - arrays containing the number of nodes in each cell along
172:           the x, y, and z coordinates, or PETSC_NULL. If non-null, these
173:           must be of length as m,n,p and the corresponding
174:           m,n, or p cannot be PETSC_DECIDE. Sum of the lx[] entries must be M, sum of
175:           the ly[] must N, sum of the lz[] must be P
176: -  s - stencil width

178:    Output Parameter:
179: .  inra - the resulting distributed array object

181:    Options Database Key:
182: +  -da_view - Calls DAView() at the conclusion of DACreate3d()
183: .  -da_grid_x <nx> - number of grid points in x direction, if M < 0
184: .  -da_grid_y <ny> - number of grid points in y direction, if N < 0
185: .  -da_grid_z <nz> - number of grid points in z direction, if P < 0
186: .  -da_processors_x <MX> number of processors in x direction
187: .  -da_processors_y <MY> number of processors in y direction
188: .  -da_processors_z <MZ> number of processors in z direction
189: .  -da_refine_x - refinement ratio in x direction
190: .  -da_refine_y - refinement ratio in y direction
191: -  -da_refine_y - refinement ratio in z direction

193:    Level: beginner

195:    Notes:
196:    The stencil type DA_STENCIL_STAR with width 1 corresponds to the 
197:    standard 7-pt stencil, while DA_STENCIL_BOX with width 1 denotes
198:    the standard 27-pt stencil.

200:    The array data itself is NOT stored in the DA, it is stored in Vec objects;
201:    The appropriate vector objects can be obtained with calls to DACreateGlobalVector()
202:    and DACreateLocalVector() and calls to VecDuplicate() if more are needed.

204: .keywords: distributed array, create, three-dimensional

206: .seealso: DADestroy(), DAView(), DACreate1d(), DACreate2d(), DAGlobalToLocalBegin(), DAGetRefinementFactor(),
207:           DAGlobalToLocalEnd(), DALocalToGlobal(), DALocalToLocalBegin(), DALocalToLocalEnd(), DASetRefinementFactor(),
208:           DAGetInfo(), DACreateGlobalVector(), DACreateLocalVector(), DACreateNaturalVector(), DALoad(), DAView(), DAGetOwnershipRanges()

210: @*/
211: PetscErrorCode  DACreate3d(MPI_Comm comm,DAPeriodicType wrap,DAStencilType stencil_type,PetscInt M,
212:                PetscInt N,PetscInt P,PetscInt m,PetscInt n,PetscInt p,PetscInt dof,PetscInt s,const PetscInt lx[],const PetscInt ly[],const PetscInt lz[],DA *inra)
213: {
215:   PetscMPIInt    rank,size;
216:   PetscInt       xs = 0,xe,ys = 0,ye,zs = 0,ze,x = 0,y = 0,z = 0,Xs,Xe,Ys,Ye,Zs,Ze,start,end,pm;
217:   PetscInt       left,up,down,bottom,top,i,j,k,*idx,nn,*flx = 0,*fly = 0,*flz = 0;
218:   PetscInt       n0,n1,n2,n3,n4,n5,n6,n7,n8,n9,n10,n11,n12,n14;
219:   PetscInt       n15,n16,n17,n18,n19,n20,n21,n22,n23,n24,n25,n26;
220:   PetscInt       *bases,*ldims,x_t,y_t,z_t,s_t,base,count,s_x,s_y,s_z;
221:   PetscInt       tM = M,tN = N,tP = P;
222:   PetscInt       sn0 = 0,sn1 = 0,sn2 = 0,sn3 = 0,sn5 = 0,sn6 = 0,sn7 = 0;
223:   PetscInt       sn8 = 0,sn9 = 0,sn11 = 0,sn15 = 0,sn24 = 0,sn25 = 0,sn26 = 0;
224:   PetscInt       sn17 = 0,sn18 = 0,sn19 = 0,sn20 = 0,sn21 = 0,sn23 = 0,refine_x = 2, refine_y = 2, refine_z = 2;
225:   DA             da;
226:   Vec            local,global;
227:   VecScatter     ltog,gtol;
228:   IS             to,from;

232:   *inra = 0;
233: #ifndef PETSC_USE_DYNAMIC_LIBRARIES
234:   DMInitializePackage(PETSC_NULL);
235: #endif

237:   if (dof < 1) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Must have 1 or more degrees of freedom per node: %D",dof);
238:   if (s < 0) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Stencil width cannot be negative: %D",s);

240:   PetscOptionsBegin(comm,PETSC_NULL,"3d DA Options","DA");
241:     if (M < 0){
242:       tM   = -M;
243:       PetscOptionsInt("-da_grid_x","Number of grid points in x direction","DACreate3d",tM,&tM,PETSC_NULL);
244:     }
245:     if (N < 0){
246:       tN   = -N;
247:       PetscOptionsInt("-da_grid_y","Number of grid points in y direction","DACreate3d",tN,&tN,PETSC_NULL);
248:     }
249:     if (P < 0){
250:       tP   = -P;
251:       PetscOptionsInt("-da_grid_z","Number of grid points in z direction","DACreate3d",tP,&tP,PETSC_NULL);
252:     }
253:     PetscOptionsInt("-da_processors_x","Number of processors in x direction","DACreate3d",m,&m,PETSC_NULL);
254:     PetscOptionsInt("-da_processors_y","Number of processors in y direction","DACreate3d",n,&n,PETSC_NULL);
255:     PetscOptionsInt("-da_processors_z","Number of processors in z direction","DACreate3d",p,&p,PETSC_NULL);
256:     PetscOptionsInt("-da_refine_x","Refinement ratio in x direction","DASetRefinementFactor",refine_x,&refine_x,PETSC_NULL);
257:     PetscOptionsInt("-da_refine_y","Refinement ratio in y direction","DASetRefinementFactor",refine_y,&refine_y,PETSC_NULL);
258:     PetscOptionsInt("-da_refine_z","Refinement ratio in z direction","DASetRefinementFactor",refine_z,&refine_z,PETSC_NULL);
259:   PetscOptionsEnd();
260:   M = tM; N = tN; P = tP;

262:   PetscHeaderCreate(da,_p_DA,struct _DAOps,DM_COOKIE,0,"DM",comm,DADestroy,DAView);
263:   PetscObjectChangeTypeName((PetscObject)da,"DA");
264:   da->ops->globaltolocalbegin = DAGlobalToLocalBegin;
265:   da->ops->globaltolocalend   = DAGlobalToLocalEnd;
266:   da->ops->localtoglobal      = DALocalToGlobal;
267:   da->ops->createglobalvector = DACreateGlobalVector;
268:   da->ops->createlocalvector  = DACreateLocalVector;
269:   da->ops->getinterpolation   = DAGetInterpolation;
270:   da->ops->getcoloring        = DAGetColoring;
271:   da->ops->getmatrix          = DAGetMatrix;
272:   da->ops->refine             = DARefine;
273:   da->ops->coarsen            = DACoarsen;
274:   da->ops->getaggregates      = DAGetAggregates;
275:   da->ops->destroy            = DADestroy;

277:   da->dim        = 3;
278:   da->interptype = DA_Q1;
279:   da->refine_x   = refine_x;
280:   da->refine_y   = refine_y;
281:   da->refine_z   = refine_z;
282:   PetscMalloc(dof*sizeof(char*),&da->fieldname);
283:   PetscMemzero(da->fieldname,dof*sizeof(char*));

285:   MPI_Comm_size(comm,&size);
286:   MPI_Comm_rank(comm,&rank);

288:   if (m != PETSC_DECIDE) {
289:     if (m < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in X direction: %D",m);}
290:     else if (m > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in X direction: %D %d",m,size);}
291:   }
292:   if (n != PETSC_DECIDE) {
293:     if (n < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in Y direction: %D",n);}
294:     else if (n > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in Y direction: %D %d",n,size);}
295:   }
296:   if (p != PETSC_DECIDE) {
297:     if (p < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in Z direction: %D",p);}
298:     else if (p > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in Z direction: %D %d",p,size);}
299:   }

301:   /* Partition the array among the processors */
302:   if (m == PETSC_DECIDE && n != PETSC_DECIDE && p != PETSC_DECIDE) {
303:     m = size/(n*p);
304:   } else if (m != PETSC_DECIDE && n == PETSC_DECIDE && p != PETSC_DECIDE) {
305:     n = size/(m*p);
306:   } else if (m != PETSC_DECIDE && n != PETSC_DECIDE && p == PETSC_DECIDE) {
307:     p = size/(m*n);
308:   } else if (m == PETSC_DECIDE && n == PETSC_DECIDE && p != PETSC_DECIDE) {
309:     /* try for squarish distribution */
310:     m = (int)(0.5 + sqrt(((PetscReal)M)*((PetscReal)size)/((PetscReal)N*p)));
311:     if (!m) m = 1;
312:     while (m > 0) {
313:       n = size/(m*p);
314:       if (m*n*p == size) break;
315:       m--;
316:     }
317:     if (!m) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad p value: p = %D",p);
318:     if (M > N && m < n) {PetscInt _m = m; m = n; n = _m;}
319:   } else if (m == PETSC_DECIDE && n != PETSC_DECIDE && p == PETSC_DECIDE) {
320:     /* try for squarish distribution */
321:     m = (int)(0.5 + sqrt(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n)));
322:     if (!m) m = 1;
323:     while (m > 0) {
324:       p = size/(m*n);
325:       if (m*n*p == size) break;
326:       m--;
327:     }
328:     if (!m) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad n value: n = %D",n);
329:     if (M > P && m < p) {PetscInt _m = m; m = p; p = _m;}
330:   } else if (m != PETSC_DECIDE && n == PETSC_DECIDE && p == PETSC_DECIDE) {
331:     /* try for squarish distribution */
332:     n = (int)(0.5 + sqrt(((PetscReal)N)*((PetscReal)size)/((PetscReal)P*m)));
333:     if (!n) n = 1;
334:     while (n > 0) {
335:       p = size/(m*n);
336:       if (m*n*p == size) break;
337:       n--;
338:     }
339:     if (!n) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad m value: m = %D",n);
340:     if (N > P && n < p) {PetscInt _n = n; n = p; p = _n;}
341:   } else if (m == PETSC_DECIDE && n == PETSC_DECIDE && p == PETSC_DECIDE) {
342:     /* try for squarish distribution */
343:     n = (PetscInt)(0.5 + pow(((PetscReal)N*N)*((PetscReal)size)/((PetscReal)P*M),(PetscReal)(1./3.)));
344:     if (!n) n = 1;
345:     while (n > 0) {
346:       pm = size/n;
347:       if (n*pm == size) break;
348:       n--;
349:     }
350:     if (!n) n = 1;
351:     m = (PetscInt)(0.5 + sqrt(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n)));
352:     if (!m) m = 1;
353:     while (m > 0) {
354:       p = size/(m*n);
355:       if (m*n*p == size) break;
356:       m--;
357:     }
358:     if (M > P && m < p) {PetscInt _m = m; m = p; p = _m;}
359:   } else if (m*n*p != size) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Given Bad partition");

361:   if (m*n*p != size) SETERRQ(PETSC_ERR_PLIB,"Could not find good partition");
362:   if (M < m) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in x direction is too fine! %D %D",M,m);
363:   if (N < n) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in y direction is too fine! %D %D",N,n);
364:   if (P < p) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in z direction is too fine! %D %D",P,p);

366:   /* 
367:      Determine locally owned region 
368:      [x, y, or z]s is the first local node number, [x, y, z] is the number of local nodes 
369:   */

371:   PetscMalloc(m*sizeof(PetscInt),&flx);
372:   if (lx) { /* user decided distribution */
373:     PetscMemcpy(flx,lx,m*sizeof(PetscInt));
374:   } else {
375:     for (i=0; i<m; i++) {
376:       flx[i] = M/m + ((M % m) > (i % m));
377:     }
378:   }
379:   x  = flx[rank % m];
380:   xs = 0;
381:   for (i=0; i<(rank%m); i++) { xs += flx[i];}
382:   if (m > 1 && x < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Column width is too thin for stencil! %D %D",x,s);

384:   PetscMalloc(n*sizeof(PetscInt),&fly);
385:   if (ly) { /* user decided distribution */
386:     PetscMemcpy(fly,ly,n*sizeof(PetscInt));
387:   } else {
388:     for (i=0; i<n; i++) {
389:       fly[i] = N/n + ((N % n) > (i % n));
390:     }
391:   }
392:   y  = fly[(rank % (m*n))/m];
393:   if (n > 1 && y < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Row width is too thin for stencil! %D %D",y,s);
394:   ys = 0;
395:   for (i=0; i<(rank % (m*n))/m; i++) { ys += fly[i];}

397:   PetscMalloc(p*sizeof(PetscInt),&flz);
398:   if (lz) { /* user decided distribution */
399:     PetscMemcpy(flz,lz,p*sizeof(PetscInt));
400:   } else {
401:     for (i=0; i<p; i++) {
402:       flz[i] = P/p + ((P % p) > (i % p));
403:     }
404:   }
405:   z  = flz[rank/(m*n)];
406:   if (p > 1 && z < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Plane width is too thin for stencil! %D %D",z,s);
407:   zs = 0;
408:   for (i=0; i<(rank/(m*n)); i++) { zs += flz[i];}
409:   ye = ys + y;
410:   xe = xs + x;
411:   ze = zs + z;

413:   /* determine ghost region */
414:   /* Assume No Periodicity */
415:   if (xs-s > 0) Xs = xs - s; else Xs = 0;
416:   if (ys-s > 0) Ys = ys - s; else Ys = 0;
417:   if (zs-s > 0) Zs = zs - s; else Zs = 0;
418:   if (xe+s <= M) Xe = xe + s; else Xe = M;
419:   if (ye+s <= N) Ye = ye + s; else Ye = N;
420:   if (ze+s <= P) Ze = ze + s; else Ze = P;

422:   /* X Periodic */
423:   if (DAXPeriodic(wrap)){
424:     Xs = xs - s;
425:     Xe = xe + s;
426:   }

428:   /* Y Periodic */
429:   if (DAYPeriodic(wrap)){
430:     Ys = ys - s;
431:     Ye = ye + s;
432:   }

434:   /* Z Periodic */
435:   if (DAZPeriodic(wrap)){
436:     Zs = zs - s;
437:     Ze = ze + s;
438:   }

440:   /* Resize all X parameters to reflect w */
441:   x   *= dof;
442:   xs  *= dof;
443:   xe  *= dof;
444:   Xs  *= dof;
445:   Xe  *= dof;
446:   s_x  = s*dof;
447:   s_y  = s;
448:   s_z  = s;

450:   /* determine starting point of each processor */
451:   nn       = x*y*z;
452:   PetscMalloc((2*size+1)*sizeof(PetscInt),&bases);
453:   ldims    = (PetscInt*)(bases+size+1);
454:   MPI_Allgather(&nn,1,MPIU_INT,ldims,1,MPIU_INT,comm);
455:   bases[0] = 0;
456:   for (i=1; i<=size; i++) {
457:     bases[i] = ldims[i-1];
458:   }
459:   for (i=1; i<=size; i++) {
460:     bases[i] += bases[i-1];
461:   }

463:   /* allocate the base parallel and sequential vectors */
464:   da->Nlocal = x*y*z;
465:   VecCreateMPIWithArray(comm,da->Nlocal,PETSC_DECIDE,0,&global);
466:   VecSetBlockSize(global,dof);
467:   da->nlocal = (Xe-Xs)*(Ye-Ys)*(Ze-Zs);
468:   VecCreateSeqWithArray(MPI_COMM_SELF,da->nlocal,0,&local);
469:   VecSetBlockSize(local,dof);

471:   /* generate appropriate vector scatters */
472:   /* local to global inserts non-ghost point region into global */
473:   VecGetOwnershipRange(global,&start,&end);
474:   ISCreateStride(comm,x*y*z,start,1,&to);

476:   left   = xs - Xs;
477:   bottom = ys - Ys; top = bottom + y;
478:   down   = zs - Zs; up  = down + z;
479:   count  = x*(top-bottom)*(up-down);
480:   PetscMalloc(count*sizeof(PetscInt)/dof,&idx);
481:   count  = 0;
482:   for (i=down; i<up; i++) {
483:     for (j=bottom; j<top; j++) {
484:       for (k=0; k<x; k += dof) {
485:         idx[count++] = (left+j*(Xe-Xs))+i*(Xe-Xs)*(Ye-Ys) + k;
486:       }
487:     }
488:   }
489:   ISCreateBlock(comm,dof,count,idx,&from);
490:   PetscFree(idx);

492:   VecScatterCreate(local,from,global,to,&ltog);
493:   PetscLogObjectParent(da,to);
494:   PetscLogObjectParent(da,from);
495:   PetscLogObjectParent(da,ltog);
496:   ISDestroy(from);
497:   ISDestroy(to);

499:   /* global to local must include ghost points */
500:   if (stencil_type == DA_STENCIL_BOX) {
501:     ISCreateStride(comm,(Xe-Xs)*(Ye-Ys)*(Ze-Zs),0,1,&to);
502:   } else {
503:     /* This is way ugly! We need to list the funny cross type region */
504:     /* the bottom chunck */
505:     left   = xs - Xs;
506:     bottom = ys - Ys; top = bottom + y;
507:     down   = zs - Zs;   up  = down + z;
508:     count  = down*(top-bottom)*x + (up-down)*(bottom*x  + (top-bottom)*(Xe-Xs) + (Ye-Ys-top)*x) + (Ze-Zs-up)*(top-bottom)*x;
509:     PetscMalloc(count*sizeof(PetscInt)/dof,&idx);
510:     count  = 0;
511:     for (i=0; i<down; i++) {
512:       for (j=bottom; j<top; j++) {
513:         for (k=0; k<x; k += dof) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
514:       }
515:     }
516:     /* the middle piece */
517:     for (i=down; i<up; i++) {
518:       /* front */
519:       for (j=0; j<bottom; j++) {
520:         for (k=0; k<x; k += dof) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
521:       }
522:       /* middle */
523:       for (j=bottom; j<top; j++) {
524:         for (k=0; k<Xe-Xs; k += dof) idx[count++] = j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
525:       }
526:       /* back */
527:       for (j=top; j<Ye-Ys; j++) {
528:         for (k=0; k<x; k += dof) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
529:       }
530:     }
531:     /* the top piece */
532:     for (i=up; i<Ze-Zs; i++) {
533:       for (j=bottom; j<top; j++) {
534:         for (k=0; k<x; k += dof) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
535:       }
536:     }
537:     ISCreateBlock(comm,dof,count,idx,&to);
538:     PetscFree(idx);
539:   }

541:   /* determine who lies on each side of use stored in    n24 n25 n26
542:                                                          n21 n22 n23
543:                                                          n18 n19 n20

545:                                                          n15 n16 n17
546:                                                          n12     n14
547:                                                          n9  n10 n11

549:                                                          n6  n7  n8
550:                                                          n3  n4  n5
551:                                                          n0  n1  n2
552:   */
553: 
554:   /* Solve for X,Y, and Z Periodic Case First, Then Modify Solution */
555: 
556:   /* Assume Nodes are Internal to the Cube */
557: 
558:   n0  = rank - m*n - m - 1;
559:   n1  = rank - m*n - m;
560:   n2  = rank - m*n - m + 1;
561:   n3  = rank - m*n -1;
562:   n4  = rank - m*n;
563:   n5  = rank - m*n + 1;
564:   n6  = rank - m*n + m - 1;
565:   n7  = rank - m*n + m;
566:   n8  = rank - m*n + m + 1;

568:   n9  = rank - m - 1;
569:   n10 = rank - m;
570:   n11 = rank - m + 1;
571:   n12 = rank - 1;
572:   n14 = rank + 1;
573:   n15 = rank + m - 1;
574:   n16 = rank + m;
575:   n17 = rank + m + 1;

577:   n18 = rank + m*n - m - 1;
578:   n19 = rank + m*n - m;
579:   n20 = rank + m*n - m + 1;
580:   n21 = rank + m*n - 1;
581:   n22 = rank + m*n;
582:   n23 = rank + m*n + 1;
583:   n24 = rank + m*n + m - 1;
584:   n25 = rank + m*n + m;
585:   n26 = rank + m*n + m + 1;

587:   /* Assume Pieces are on Faces of Cube */

589:   if (xs == 0) { /* First assume not corner or edge */
590:     n0  = rank       -1 - (m*n);
591:     n3  = rank + m   -1 - (m*n);
592:     n6  = rank + 2*m -1 - (m*n);
593:     n9  = rank       -1;
594:     n12 = rank + m   -1;
595:     n15 = rank + 2*m -1;
596:     n18 = rank       -1 + (m*n);
597:     n21 = rank + m   -1 + (m*n);
598:     n24 = rank + 2*m -1 + (m*n);
599:    }

601:   if (xe == M*dof) { /* First assume not corner or edge */
602:     n2  = rank -2*m +1 - (m*n);
603:     n5  = rank - m  +1 - (m*n);
604:     n8  = rank      +1 - (m*n);
605:     n11 = rank -2*m +1;
606:     n14 = rank - m  +1;
607:     n17 = rank      +1;
608:     n20 = rank -2*m +1 + (m*n);
609:     n23 = rank - m  +1 + (m*n);
610:     n26 = rank      +1 + (m*n);
611:   }

613:   if (ys==0) { /* First assume not corner or edge */
614:     n0  = rank + m * (n-1) -1 - (m*n);
615:     n1  = rank + m * (n-1)    - (m*n);
616:     n2  = rank + m * (n-1) +1 - (m*n);
617:     n9  = rank + m * (n-1) -1;
618:     n10 = rank + m * (n-1);
619:     n11 = rank + m * (n-1) +1;
620:     n18 = rank + m * (n-1) -1 + (m*n);
621:     n19 = rank + m * (n-1)    + (m*n);
622:     n20 = rank + m * (n-1) +1 + (m*n);
623:   }

625:   if (ye == N) { /* First assume not corner or edge */
626:     n6  = rank - m * (n-1) -1 - (m*n);
627:     n7  = rank - m * (n-1)    - (m*n);
628:     n8  = rank - m * (n-1) +1 - (m*n);
629:     n15 = rank - m * (n-1) -1;
630:     n16 = rank - m * (n-1);
631:     n17 = rank - m * (n-1) +1;
632:     n24 = rank - m * (n-1) -1 + (m*n);
633:     n25 = rank - m * (n-1)    + (m*n);
634:     n26 = rank - m * (n-1) +1 + (m*n);
635:   }
636: 
637:   if (zs == 0) { /* First assume not corner or edge */
638:     n0 = size - (m*n) + rank - m - 1;
639:     n1 = size - (m*n) + rank - m;
640:     n2 = size - (m*n) + rank - m + 1;
641:     n3 = size - (m*n) + rank - 1;
642:     n4 = size - (m*n) + rank;
643:     n5 = size - (m*n) + rank + 1;
644:     n6 = size - (m*n) + rank + m - 1;
645:     n7 = size - (m*n) + rank + m ;
646:     n8 = size - (m*n) + rank + m + 1;
647:   }

649:   if (ze == P) { /* First assume not corner or edge */
650:     n18 = (m*n) - (size-rank) - m - 1;
651:     n19 = (m*n) - (size-rank) - m;
652:     n20 = (m*n) - (size-rank) - m + 1;
653:     n21 = (m*n) - (size-rank) - 1;
654:     n22 = (m*n) - (size-rank);
655:     n23 = (m*n) - (size-rank) + 1;
656:     n24 = (m*n) - (size-rank) + m - 1;
657:     n25 = (m*n) - (size-rank) + m;
658:     n26 = (m*n) - (size-rank) + m + 1;
659:   }

661:   if ((xs==0) && (zs==0)) { /* Assume an edge, not corner */
662:     n0 = size - m*n + rank + m-1 - m;
663:     n3 = size - m*n + rank + m-1;
664:     n6 = size - m*n + rank + m-1 + m;
665:   }
666: 
667:   if ((xs==0) && (ze==P)) { /* Assume an edge, not corner */
668:     n18 = m*n - (size - rank) + m-1 - m;
669:     n21 = m*n - (size - rank) + m-1;
670:     n24 = m*n - (size - rank) + m-1 + m;
671:   }

673:   if ((xs==0) && (ys==0)) { /* Assume an edge, not corner */
674:     n0  = rank + m*n -1 - m*n;
675:     n9  = rank + m*n -1;
676:     n18 = rank + m*n -1 + m*n;
677:   }

679:   if ((xs==0) && (ye==N)) { /* Assume an edge, not corner */
680:     n6  = rank - m*(n-1) + m-1 - m*n;
681:     n15 = rank - m*(n-1) + m-1;
682:     n24 = rank - m*(n-1) + m-1 + m*n;
683:   }

685:   if ((xe==M*dof) && (zs==0)) { /* Assume an edge, not corner */
686:     n2 = size - (m*n-rank) - (m-1) - m;
687:     n5 = size - (m*n-rank) - (m-1);
688:     n8 = size - (m*n-rank) - (m-1) + m;
689:   }

691:   if ((xe==M*dof) && (ze==P)) { /* Assume an edge, not corner */
692:     n20 = m*n - (size - rank) - (m-1) - m;
693:     n23 = m*n - (size - rank) - (m-1);
694:     n26 = m*n - (size - rank) - (m-1) + m;
695:   }

697:   if ((xe==M*dof) && (ys==0)) { /* Assume an edge, not corner */
698:     n2  = rank + m*(n-1) - (m-1) - m*n;
699:     n11 = rank + m*(n-1) - (m-1);
700:     n20 = rank + m*(n-1) - (m-1) + m*n;
701:   }

703:   if ((xe==M*dof) && (ye==N)) { /* Assume an edge, not corner */
704:     n8  = rank - m*n +1 - m*n;
705:     n17 = rank - m*n +1;
706:     n26 = rank - m*n +1 + m*n;
707:   }

709:   if ((ys==0) && (zs==0)) { /* Assume an edge, not corner */
710:     n0 = size - m + rank -1;
711:     n1 = size - m + rank;
712:     n2 = size - m + rank +1;
713:   }

715:   if ((ys==0) && (ze==P)) { /* Assume an edge, not corner */
716:     n18 = m*n - (size - rank) + m*(n-1) -1;
717:     n19 = m*n - (size - rank) + m*(n-1);
718:     n20 = m*n - (size - rank) + m*(n-1) +1;
719:   }

721:   if ((ye==N) && (zs==0)) { /* Assume an edge, not corner */
722:     n6 = size - (m*n-rank) - m * (n-1) -1;
723:     n7 = size - (m*n-rank) - m * (n-1);
724:     n8 = size - (m*n-rank) - m * (n-1) +1;
725:   }

727:   if ((ye==N) && (ze==P)) { /* Assume an edge, not corner */
728:     n24 = rank - (size-m) -1;
729:     n25 = rank - (size-m);
730:     n26 = rank - (size-m) +1;
731:   }

733:   /* Check for Corners */
734:   if ((xs==0)   && (ys==0) && (zs==0)) { n0  = size -1;}
735:   if ((xs==0)   && (ys==0) && (ze==P)) { n18 = m*n-1;}
736:   if ((xs==0)   && (ye==N) && (zs==0)) { n6  = (size-1)-m*(n-1);}
737:   if ((xs==0)   && (ye==N) && (ze==P)) { n24 = m-1;}
738:   if ((xe==M*dof) && (ys==0) && (zs==0)) { n2  = size-m;}
739:   if ((xe==M*dof) && (ys==0) && (ze==P)) { n20 = m*n-m;}
740:   if ((xe==M*dof) && (ye==N) && (zs==0)) { n8  = size-m*n;}
741:   if ((xe==M*dof) && (ye==N) && (ze==P)) { n26 = 0;}

743:   /* Check for when not X,Y, and Z Periodic */

745:   /* If not X periodic */
746:   if ((wrap != DA_XPERIODIC)  && (wrap != DA_XYPERIODIC) &&
747:      (wrap != DA_XZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
748:     if (xs==0)   {n0  = n3  = n6  = n9  = n12 = n15 = n18 = n21 = n24 = -2;}
749:     if (xe==M*dof) {n2  = n5  = n8  = n11 = n14 = n17 = n20 = n23 = n26 = -2;}
750:   }

752:   /* If not Y periodic */
753:   if ((wrap != DA_YPERIODIC)  && (wrap != DA_XYPERIODIC) &&
754:       (wrap != DA_YZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
755:     if (ys==0)   {n0  = n1  = n2  = n9  = n10 = n11 = n18 = n19 = n20 = -2;}
756:     if (ye==N)   {n6  = n7  = n8  = n15 = n16 = n17 = n24 = n25 = n26 = -2;}
757:   }

759:   /* If not Z periodic */
760:   if ((wrap != DA_ZPERIODIC)  && (wrap != DA_XZPERIODIC) &&
761:       (wrap != DA_YZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
762:     if (zs==0)   {n0  = n1  = n2  = n3  = n4  = n5  = n6  = n7  = n8  = -2;}
763:     if (ze==P)   {n18 = n19 = n20 = n21 = n22 = n23 = n24 = n25 = n26 = -2;}
764:   }

766:   PetscMalloc(27*sizeof(PetscInt),&da->neighbors);
767:   da->neighbors[0] = n0;
768:   da->neighbors[1] = n1;
769:   da->neighbors[2] = n2;
770:   da->neighbors[3] = n3;
771:   da->neighbors[4] = n4;
772:   da->neighbors[5] = n5;
773:   da->neighbors[6] = n6;
774:   da->neighbors[7] = n7;
775:   da->neighbors[8] = n8;
776:   da->neighbors[9] = n9;
777:   da->neighbors[10] = n10;
778:   da->neighbors[11] = n11;
779:   da->neighbors[12] = n12;
780:   da->neighbors[13] = rank;
781:   da->neighbors[14] = n14;
782:   da->neighbors[15] = n15;
783:   da->neighbors[16] = n16;
784:   da->neighbors[17] = n17;
785:   da->neighbors[18] = n18;
786:   da->neighbors[19] = n19;
787:   da->neighbors[20] = n20;
788:   da->neighbors[21] = n21;
789:   da->neighbors[22] = n22;
790:   da->neighbors[23] = n23;
791:   da->neighbors[24] = n24;
792:   da->neighbors[25] = n25;
793:   da->neighbors[26] = n26;

795:   /* If star stencil then delete the corner neighbors */
796:   if (stencil_type == DA_STENCIL_STAR) {
797:      /* save information about corner neighbors */
798:      sn0 = n0; sn1 = n1; sn2 = n2; sn3 = n3; sn5 = n5; sn6 = n6; sn7 = n7;
799:      sn8 = n8; sn9 = n9; sn11 = n11; sn15 = n15; sn17 = n17; sn18 = n18;
800:      sn19 = n19; sn20 = n20; sn21 = n21; sn23 = n23; sn24 = n24; sn25 = n25;
801:      sn26 = n26;
802:      n0  = n1  = n2  = n3  = n5  = n6  = n7  = n8  = n9  = n11 =
803:      n15 = n17 = n18 = n19 = n20 = n21 = n23 = n24 = n25 = n26 = -1;
804:   }


807:   PetscMalloc((Xe-Xs)*(Ye-Ys)*(Ze-Zs)*sizeof(PetscInt),&idx);
808:   PetscLogObjectMemory(da,(Xe-Xs)*(Ye-Ys)*(Ze-Zs)*sizeof(PetscInt));

810:   nn = 0;

812:   /* Bottom Level */
813:   for (k=0; k<s_z; k++) {
814:     for (i=1; i<=s_y; i++) {
815:       if (n0 >= 0) { /* left below */
816:         x_t = flx[n0 % m]*dof;
817:         y_t = fly[(n0 % (m*n))/m];
818:         z_t = flz[n0 / (m*n)];
819:         s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t - s_x - (s_z-k-1)*x_t*y_t;
820:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
821:       }
822:       if (n1 >= 0) { /* directly below */
823:         x_t = x;
824:         y_t = fly[(n1 % (m*n))/m];
825:         z_t = flz[n1 / (m*n)];
826:         s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
827:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
828:       }
829:       if (n2 >= 0) { /* right below */
830:         x_t = flx[n2 % m]*dof;
831:         y_t = fly[(n2 % (m*n))/m];
832:         z_t = flz[n2 / (m*n)];
833:         s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
834:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
835:       }
836:     }

838:     for (i=0; i<y; i++) {
839:       if (n3 >= 0) { /* directly left */
840:         x_t = flx[n3 % m]*dof;
841:         y_t = y;
842:         z_t = flz[n3 / (m*n)];
843:         s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
844:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
845:       }

847:       if (n4 >= 0) { /* middle */
848:         x_t = x;
849:         y_t = y;
850:         z_t = flz[n4 / (m*n)];
851:         s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
852:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
853:       }

855:       if (n5 >= 0) { /* directly right */
856:         x_t = flx[n5 % m]*dof;
857:         y_t = y;
858:         z_t = flz[n5 / (m*n)];
859:         s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
860:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
861:       }
862:     }

864:     for (i=1; i<=s_y; i++) {
865:       if (n6 >= 0) { /* left above */
866:         x_t = flx[n6 % m]*dof;
867:         y_t = fly[(n6 % (m*n))/m];
868:         z_t = flz[n6 / (m*n)];
869:         s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
870:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
871:       }
872:       if (n7 >= 0) { /* directly above */
873:         x_t = x;
874:         y_t = fly[(n7 % (m*n))/m];
875:         z_t = flz[n7 / (m*n)];
876:         s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
877:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
878:       }
879:       if (n8 >= 0) { /* right above */
880:         x_t = flx[n8 % m]*dof;
881:         y_t = fly[(n8 % (m*n))/m];
882:         z_t = flz[n8 / (m*n)];
883:         s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
884:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
885:       }
886:     }
887:   }

889:   /* Middle Level */
890:   for (k=0; k<z; k++) {
891:     for (i=1; i<=s_y; i++) {
892:       if (n9 >= 0) { /* left below */
893:         x_t = flx[n9 % m]*dof;
894:         y_t = fly[(n9 % (m*n))/m];
895:         /* z_t = z; */
896:         s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
897:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
898:       }
899:       if (n10 >= 0) { /* directly below */
900:         x_t = x;
901:         y_t = fly[(n10 % (m*n))/m];
902:         /* z_t = z; */
903:         s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
904:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
905:       }
906:       if (n11 >= 0) { /* right below */
907:         x_t = flx[n11 % m]*dof;
908:         y_t = fly[(n11 % (m*n))/m];
909:         /* z_t = z; */
910:         s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
911:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
912:       }
913:     }

915:     for (i=0; i<y; i++) {
916:       if (n12 >= 0) { /* directly left */
917:         x_t = flx[n12 % m]*dof;
918:         y_t = y;
919:         /* z_t = z; */
920:         s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
921:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
922:       }

924:       /* Interior */
925:       s_t = bases[rank] + i*x + k*x*y;
926:       for (j=0; j<x; j++) { idx[nn++] = s_t++;}

928:       if (n14 >= 0) { /* directly right */
929:         x_t = flx[n14 % m]*dof;
930:         y_t = y;
931:         /* z_t = z; */
932:         s_t = bases[n14] + i*x_t + k*x_t*y_t;
933:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
934:       }
935:     }

937:     for (i=1; i<=s_y; i++) {
938:       if (n15 >= 0) { /* left above */
939:         x_t = flx[n15 % m]*dof;
940:         y_t = fly[(n15 % (m*n))/m];
941:         /* z_t = z; */
942:         s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
943:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
944:       }
945:       if (n16 >= 0) { /* directly above */
946:         x_t = x;
947:         y_t = fly[(n16 % (m*n))/m];
948:         /* z_t = z; */
949:         s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
950:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
951:       }
952:       if (n17 >= 0) { /* right above */
953:         x_t = flx[n17 % m]*dof;
954:         y_t = fly[(n17 % (m*n))/m];
955:         /* z_t = z; */
956:         s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
957:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
958:       }
959:     }
960:   }
961: 
962:   /* Upper Level */
963:   for (k=0; k<s_z; k++) {
964:     for (i=1; i<=s_y; i++) {
965:       if (n18 >= 0) { /* left below */
966:         x_t = flx[n18 % m]*dof;
967:         y_t = fly[(n18 % (m*n))/m];
968:         /* z_t = flz[n18 / (m*n)]; */
969:         s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
970:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
971:       }
972:       if (n19 >= 0) { /* directly below */
973:         x_t = x;
974:         y_t = fly[(n19 % (m*n))/m];
975:         /* z_t = flz[n19 / (m*n)]; */
976:         s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
977:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
978:       }
979:       if (n20 >= 0) { /* right below */
980:         x_t = flx[n20 % m]*dof;
981:         y_t = fly[(n20 % (m*n))/m];
982:         /* z_t = flz[n20 / (m*n)]; */
983:         s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
984:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
985:       }
986:     }

988:     for (i=0; i<y; i++) {
989:       if (n21 >= 0) { /* directly left */
990:         x_t = flx[n21 % m]*dof;
991:         y_t = y;
992:         /* z_t = flz[n21 / (m*n)]; */
993:         s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
994:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
995:       }

997:       if (n22 >= 0) { /* middle */
998:         x_t = x;
999:         y_t = y;
1000:         /* z_t = flz[n22 / (m*n)]; */
1001:         s_t = bases[n22] + i*x_t + k*x_t*y_t;
1002:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1003:       }

1005:       if (n23 >= 0) { /* directly right */
1006:         x_t = flx[n23 % m]*dof;
1007:         y_t = y;
1008:         /* z_t = flz[n23 / (m*n)]; */
1009:         s_t = bases[n23] + i*x_t + k*x_t*y_t;
1010:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1011:       }
1012:     }

1014:     for (i=1; i<=s_y; i++) {
1015:       if (n24 >= 0) { /* left above */
1016:         x_t = flx[n24 % m]*dof;
1017:         y_t = fly[(n24 % (m*n))/m];
1018:         /* z_t = flz[n24 / (m*n)]; */
1019:         s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
1020:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1021:       }
1022:       if (n25 >= 0) { /* directly above */
1023:         x_t = x;
1024:         y_t = fly[(n25 % (m*n))/m];
1025:         /* z_t = flz[n25 / (m*n)]; */
1026:         s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1027:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1028:       }
1029:       if (n26 >= 0) { /* right above */
1030:         x_t = flx[n26 % m]*dof;
1031:         y_t = fly[(n26 % (m*n))/m];
1032:         /* z_t = flz[n26 / (m*n)]; */
1033:         s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1034:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1035:       }
1036:     }
1037:   }
1038:   base = bases[rank];
1039:   {
1040:     PetscInt nnn = nn/dof,*iidx;
1041:     PetscMalloc(nnn*sizeof(PetscInt),&iidx);
1042:     for (i=0; i<nnn; i++) {
1043:       iidx[i] = idx[dof*i];
1044:     }
1045:     ISCreateBlock(comm,dof,nnn,iidx,&from);
1046:     PetscFree(iidx);
1047:   }
1048:   VecScatterCreate(global,from,local,to,&gtol);
1049:   PetscLogObjectParent(da,gtol);
1050:   PetscLogObjectParent(da,to);
1051:   PetscLogObjectParent(da,from);
1052:   ISDestroy(to);
1053:   ISDestroy(from);
1054:   da->stencil_type = stencil_type;
1055:   da->M  = M;  da->N  = N; da->P = P;
1056:   da->m  = m;  da->n  = n; da->p = p;
1057:   da->w  = dof;  da->s  = s;
1058:   da->xs = xs; da->xe = xe; da->ys = ys; da->ye = ye; da->zs = zs; da->ze = ze;
1059:   da->Xs = Xs; da->Xe = Xe; da->Ys = Ys; da->Ye = Ye; da->Zs = Zs; da->Ze = Ze;

1061:   VecDestroy(local);
1062:   VecDestroy(global);

1064:   if (stencil_type == DA_STENCIL_STAR) {
1065:     /*
1066:         Recompute the local to global mappings, this time keeping the 
1067:       information about the cross corner processor numbers.
1068:     */
1069:     n0  = sn0;  n1  = sn1;  n2  = sn2;  n3  = sn3;  n5  = sn5;  n6  = sn6; n7 = sn7;
1070:     n8  = sn8;  n9  = sn9;  n11 = sn11; n15 = sn15; n17 = sn17; n18 = sn18;
1071:     n19 = sn19; n20 = sn20; n21 = sn21; n23 = sn23; n24 = sn24; n25 = sn25;
1072:     n26 = sn26;

1074:     nn = 0;

1076:     /* Bottom Level */
1077:     for (k=0; k<s_z; k++) {
1078:       for (i=1; i<=s_y; i++) {
1079:         if (n0 >= 0) { /* left below */
1080:           x_t = flx[n0 % m]*dof;
1081:           y_t = fly[(n0 % (m*n))/m];
1082:           z_t = flz[n0 / (m*n)];
1083:           s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t - s_x - (s_z-k-1)*x_t*y_t;
1084:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1085:         }
1086:         if (n1 >= 0) { /* directly below */
1087:           x_t = x;
1088:           y_t = fly[(n1 % (m*n))/m];
1089:           z_t = flz[n1 / (m*n)];
1090:           s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1091:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1092:         }
1093:         if (n2 >= 0) { /* right below */
1094:           x_t = flx[n2 % m]*dof;
1095:           y_t = fly[(n2 % (m*n))/m];
1096:           z_t = flz[n2 / (m*n)];
1097:           s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1098:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1099:         }
1100:       }

1102:       for (i=0; i<y; i++) {
1103:         if (n3 >= 0) { /* directly left */
1104:           x_t = flx[n3 % m]*dof;
1105:           y_t = y;
1106:           z_t = flz[n3 / (m*n)];
1107:           s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1108:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1109:         }

1111:         if (n4 >= 0) { /* middle */
1112:           x_t = x;
1113:           y_t = y;
1114:           z_t = flz[n4 / (m*n)];
1115:           s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1116:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1117:         }

1119:         if (n5 >= 0) { /* directly right */
1120:           x_t = flx[n5 % m]*dof;
1121:           y_t = y;
1122:           z_t = flz[n5 / (m*n)];
1123:           s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1124:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1125:         }
1126:       }

1128:       for (i=1; i<=s_y; i++) {
1129:         if (n6 >= 0) { /* left above */
1130:           x_t = flx[n6 % m]*dof;
1131:           y_t = fly[(n6 % (m*n))/m];
1132:           z_t = flz[n6 / (m*n)];
1133:           s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1134:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1135:         }
1136:         if (n7 >= 0) { /* directly above */
1137:           x_t = x;
1138:           y_t = fly[(n7 % (m*n))/m];
1139:           z_t = flz[n7 / (m*n)];
1140:           s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1141:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1142:         }
1143:         if (n8 >= 0) { /* right above */
1144:           x_t = flx[n8 % m]*dof;
1145:           y_t = fly[(n8 % (m*n))/m];
1146:           z_t = flz[n8 / (m*n)];
1147:           s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1148:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1149:         }
1150:       }
1151:     }

1153:     /* Middle Level */
1154:     for (k=0; k<z; k++) {
1155:       for (i=1; i<=s_y; i++) {
1156:         if (n9 >= 0) { /* left below */
1157:           x_t = flx[n9 % m]*dof;
1158:           y_t = fly[(n9 % (m*n))/m];
1159:           /* z_t = z; */
1160:           s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1161:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1162:         }
1163:         if (n10 >= 0) { /* directly below */
1164:           x_t = x;
1165:           y_t = fly[(n10 % (m*n))/m];
1166:           /* z_t = z; */
1167:           s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1168:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1169:         }
1170:         if (n11 >= 0) { /* right below */
1171:           x_t = flx[n11 % m]*dof;
1172:           y_t = fly[(n11 % (m*n))/m];
1173:           /* z_t = z; */
1174:           s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1175:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1176:         }
1177:       }

1179:       for (i=0; i<y; i++) {
1180:         if (n12 >= 0) { /* directly left */
1181:           x_t = flx[n12 % m]*dof;
1182:           y_t = y;
1183:           /* z_t = z; */
1184:           s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
1185:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1186:         }

1188:         /* Interior */
1189:         s_t = bases[rank] + i*x + k*x*y;
1190:         for (j=0; j<x; j++) { idx[nn++] = s_t++;}

1192:         if (n14 >= 0) { /* directly right */
1193:           x_t = flx[n14 % m]*dof;
1194:           y_t = y;
1195:           /* z_t = z; */
1196:           s_t = bases[n14] + i*x_t + k*x_t*y_t;
1197:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1198:         }
1199:       }

1201:       for (i=1; i<=s_y; i++) {
1202:         if (n15 >= 0) { /* left above */
1203:           x_t = flx[n15 % m]*dof;
1204:           y_t = fly[(n15 % (m*n))/m];
1205:           /* z_t = z; */
1206:           s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
1207:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1208:         }
1209:         if (n16 >= 0) { /* directly above */
1210:           x_t = x;
1211:           y_t = fly[(n16 % (m*n))/m];
1212:           /* z_t = z; */
1213:           s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
1214:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1215:         }
1216:         if (n17 >= 0) { /* right above */
1217:           x_t = flx[n17 % m]*dof;
1218:           y_t = fly[(n17 % (m*n))/m];
1219:           /* z_t = z; */
1220:           s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
1221:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1222:         }
1223:       }
1224:     }
1225: 
1226:     /* Upper Level */
1227:     for (k=0; k<s_z; k++) {
1228:       for (i=1; i<=s_y; i++) {
1229:         if (n18 >= 0) { /* left below */
1230:           x_t = flx[n18 % m]*dof;
1231:           y_t = fly[(n18 % (m*n))/m];
1232:           /* z_t = flz[n18 / (m*n)]; */
1233:           s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1234:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1235:         }
1236:         if (n19 >= 0) { /* directly below */
1237:           x_t = x;
1238:           y_t = fly[(n19 % (m*n))/m];
1239:           /* z_t = flz[n19 / (m*n)]; */
1240:           s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1241:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1242:         }
1243:         if (n20 >= 0) { /* right below */
1244:           x_t = flx[n20 % m]*dof;
1245:           y_t = fly[(n20 % (m*n))/m];
1246:           /* z_t = flz[n20 / (m*n)]; */
1247:           s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1248:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1249:         }
1250:       }

1252:       for (i=0; i<y; i++) {
1253:         if (n21 >= 0) { /* directly left */
1254:           x_t = flx[n21 % m]*dof;
1255:           y_t = y;
1256:           /* z_t = flz[n21 / (m*n)]; */
1257:           s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
1258:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1259:         }

1261:         if (n22 >= 0) { /* middle */
1262:           x_t = x;
1263:           y_t = y;
1264:           /* z_t = flz[n22 / (m*n)]; */
1265:           s_t = bases[n22] + i*x_t + k*x_t*y_t;
1266:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1267:         }

1269:         if (n23 >= 0) { /* directly right */
1270:           x_t = flx[n23 % m]*dof;
1271:           y_t = y;
1272:           /* z_t = flz[n23 / (m*n)]; */
1273:           s_t = bases[n23] + i*x_t + k*x_t*y_t;
1274:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1275:         }
1276:       }

1278:       for (i=1; i<=s_y; i++) {
1279:         if (n24 >= 0) { /* left above */
1280:           x_t = flx[n24 % m]*dof;
1281:           y_t = fly[(n24 % (m*n))/m];
1282:           /* z_t = flz[n24 / (m*n)]; */
1283:           s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
1284:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1285:         }
1286:         if (n25 >= 0) { /* directly above */
1287:           x_t = x;
1288:           y_t = fly[(n25 % (m*n))/m];
1289:           /* z_t = flz[n25 / (m*n)]; */
1290:           s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1291:           for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1292:         }
1293:         if (n26 >= 0) { /* right above */
1294:           x_t = flx[n26 % m]*dof;
1295:           y_t = fly[(n26 % (m*n))/m];
1296:           /* z_t = flz[n26 / (m*n)]; */
1297:           s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1298:           for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1299:         }
1300:       }
1301:     }
1302:   }
1303:   /* redo idx to include "missing" ghost points */
1304:   /* Solve for X,Y, and Z Periodic Case First, Then Modify Solution */
1305: 
1306:   /* Assume Nodes are Internal to the Cube */
1307: 
1308:   n0  = rank - m*n - m - 1;
1309:   n1  = rank - m*n - m;
1310:   n2  = rank - m*n - m + 1;
1311:   n3  = rank - m*n -1;
1312:   n4  = rank - m*n;
1313:   n5  = rank - m*n + 1;
1314:   n6  = rank - m*n + m - 1;
1315:   n7  = rank - m*n + m;
1316:   n8  = rank - m*n + m + 1;

1318:   n9  = rank - m - 1;
1319:   n10 = rank - m;
1320:   n11 = rank - m + 1;
1321:   n12 = rank - 1;
1322:   n14 = rank + 1;
1323:   n15 = rank + m - 1;
1324:   n16 = rank + m;
1325:   n17 = rank + m + 1;

1327:   n18 = rank + m*n - m - 1;
1328:   n19 = rank + m*n - m;
1329:   n20 = rank + m*n - m + 1;
1330:   n21 = rank + m*n - 1;
1331:   n22 = rank + m*n;
1332:   n23 = rank + m*n + 1;
1333:   n24 = rank + m*n + m - 1;
1334:   n25 = rank + m*n + m;
1335:   n26 = rank + m*n + m + 1;

1337:   /* Assume Pieces are on Faces of Cube */

1339:   if (xs == 0) { /* First assume not corner or edge */
1340:     n0  = rank       -1 - (m*n);
1341:     n3  = rank + m   -1 - (m*n);
1342:     n6  = rank + 2*m -1 - (m*n);
1343:     n9  = rank       -1;
1344:     n12 = rank + m   -1;
1345:     n15 = rank + 2*m -1;
1346:     n18 = rank       -1 + (m*n);
1347:     n21 = rank + m   -1 + (m*n);
1348:     n24 = rank + 2*m -1 + (m*n);
1349:    }

1351:   if (xe == M*dof) { /* First assume not corner or edge */
1352:     n2  = rank -2*m +1 - (m*n);
1353:     n5  = rank - m  +1 - (m*n);
1354:     n8  = rank      +1 - (m*n);
1355:     n11 = rank -2*m +1;
1356:     n14 = rank - m  +1;
1357:     n17 = rank      +1;
1358:     n20 = rank -2*m +1 + (m*n);
1359:     n23 = rank - m  +1 + (m*n);
1360:     n26 = rank      +1 + (m*n);
1361:   }

1363:   if (ys==0) { /* First assume not corner or edge */
1364:     n0  = rank + m * (n-1) -1 - (m*n);
1365:     n1  = rank + m * (n-1)    - (m*n);
1366:     n2  = rank + m * (n-1) +1 - (m*n);
1367:     n9  = rank + m * (n-1) -1;
1368:     n10 = rank + m * (n-1);
1369:     n11 = rank + m * (n-1) +1;
1370:     n18 = rank + m * (n-1) -1 + (m*n);
1371:     n19 = rank + m * (n-1)    + (m*n);
1372:     n20 = rank + m * (n-1) +1 + (m*n);
1373:   }

1375:   if (ye == N) { /* First assume not corner or edge */
1376:     n6  = rank - m * (n-1) -1 - (m*n);
1377:     n7  = rank - m * (n-1)    - (m*n);
1378:     n8  = rank - m * (n-1) +1 - (m*n);
1379:     n15 = rank - m * (n-1) -1;
1380:     n16 = rank - m * (n-1);
1381:     n17 = rank - m * (n-1) +1;
1382:     n24 = rank - m * (n-1) -1 + (m*n);
1383:     n25 = rank - m * (n-1)    + (m*n);
1384:     n26 = rank - m * (n-1) +1 + (m*n);
1385:   }
1386: 
1387:   if (zs == 0) { /* First assume not corner or edge */
1388:     n0 = size - (m*n) + rank - m - 1;
1389:     n1 = size - (m*n) + rank - m;
1390:     n2 = size - (m*n) + rank - m + 1;
1391:     n3 = size - (m*n) + rank - 1;
1392:     n4 = size - (m*n) + rank;
1393:     n5 = size - (m*n) + rank + 1;
1394:     n6 = size - (m*n) + rank + m - 1;
1395:     n7 = size - (m*n) + rank + m ;
1396:     n8 = size - (m*n) + rank + m + 1;
1397:   }

1399:   if (ze == P) { /* First assume not corner or edge */
1400:     n18 = (m*n) - (size-rank) - m - 1;
1401:     n19 = (m*n) - (size-rank) - m;
1402:     n20 = (m*n) - (size-rank) - m + 1;
1403:     n21 = (m*n) - (size-rank) - 1;
1404:     n22 = (m*n) - (size-rank);
1405:     n23 = (m*n) - (size-rank) + 1;
1406:     n24 = (m*n) - (size-rank) + m - 1;
1407:     n25 = (m*n) - (size-rank) + m;
1408:     n26 = (m*n) - (size-rank) + m + 1;
1409:   }

1411:   if ((xs==0) && (zs==0)) { /* Assume an edge, not corner */
1412:     n0 = size - m*n + rank + m-1 - m;
1413:     n3 = size - m*n + rank + m-1;
1414:     n6 = size - m*n + rank + m-1 + m;
1415:   }
1416: 
1417:   if ((xs==0) && (ze==P)) { /* Assume an edge, not corner */
1418:     n18 = m*n - (size - rank) + m-1 - m;
1419:     n21 = m*n - (size - rank) + m-1;
1420:     n24 = m*n - (size - rank) + m-1 + m;
1421:   }

1423:   if ((xs==0) && (ys==0)) { /* Assume an edge, not corner */
1424:     n0  = rank + m*n -1 - m*n;
1425:     n9  = rank + m*n -1;
1426:     n18 = rank + m*n -1 + m*n;
1427:   }

1429:   if ((xs==0) && (ye==N)) { /* Assume an edge, not corner */
1430:     n6  = rank - m*(n-1) + m-1 - m*n;
1431:     n15 = rank - m*(n-1) + m-1;
1432:     n24 = rank - m*(n-1) + m-1 + m*n;
1433:   }

1435:   if ((xe==M*dof) && (zs==0)) { /* Assume an edge, not corner */
1436:     n2 = size - (m*n-rank) - (m-1) - m;
1437:     n5 = size - (m*n-rank) - (m-1);
1438:     n8 = size - (m*n-rank) - (m-1) + m;
1439:   }

1441:   if ((xe==M*dof) && (ze==P)) { /* Assume an edge, not corner */
1442:     n20 = m*n - (size - rank) - (m-1) - m;
1443:     n23 = m*n - (size - rank) - (m-1);
1444:     n26 = m*n - (size - rank) - (m-1) + m;
1445:   }

1447:   if ((xe==M*dof) && (ys==0)) { /* Assume an edge, not corner */
1448:     n2  = rank + m*(n-1) - (m-1) - m*n;
1449:     n11 = rank + m*(n-1) - (m-1);
1450:     n20 = rank + m*(n-1) - (m-1) + m*n;
1451:   }

1453:   if ((xe==M*dof) && (ye==N)) { /* Assume an edge, not corner */
1454:     n8  = rank - m*n +1 - m*n;
1455:     n17 = rank - m*n +1;
1456:     n26 = rank - m*n +1 + m*n;
1457:   }

1459:   if ((ys==0) && (zs==0)) { /* Assume an edge, not corner */
1460:     n0 = size - m + rank -1;
1461:     n1 = size - m + rank;
1462:     n2 = size - m + rank +1;
1463:   }

1465:   if ((ys==0) && (ze==P)) { /* Assume an edge, not corner */
1466:     n18 = m*n - (size - rank) + m*(n-1) -1;
1467:     n19 = m*n - (size - rank) + m*(n-1);
1468:     n20 = m*n - (size - rank) + m*(n-1) +1;
1469:   }

1471:   if ((ye==N) && (zs==0)) { /* Assume an edge, not corner */
1472:     n6 = size - (m*n-rank) - m * (n-1) -1;
1473:     n7 = size - (m*n-rank) - m * (n-1);
1474:     n8 = size - (m*n-rank) - m * (n-1) +1;
1475:   }

1477:   if ((ye==N) && (ze==P)) { /* Assume an edge, not corner */
1478:     n24 = rank - (size-m) -1;
1479:     n25 = rank - (size-m);
1480:     n26 = rank - (size-m) +1;
1481:   }

1483:   /* Check for Corners */
1484:   if ((xs==0)   && (ys==0) && (zs==0)) { n0  = size -1;}
1485:   if ((xs==0)   && (ys==0) && (ze==P)) { n18 = m*n-1;}
1486:   if ((xs==0)   && (ye==N) && (zs==0)) { n6  = (size-1)-m*(n-1);}
1487:   if ((xs==0)   && (ye==N) && (ze==P)) { n24 = m-1;}
1488:   if ((xe==M*dof) && (ys==0) && (zs==0)) { n2  = size-m;}
1489:   if ((xe==M*dof) && (ys==0) && (ze==P)) { n20 = m*n-m;}
1490:   if ((xe==M*dof) && (ye==N) && (zs==0)) { n8  = size-m*n;}
1491:   if ((xe==M*dof) && (ye==N) && (ze==P)) { n26 = 0;}

1493:   /* Check for when not X,Y, and Z Periodic */

1495:   /* If not X periodic */
1496:   if (!DAXPeriodic(wrap)){
1497:     if (xs==0)   {n0  = n3  = n6  = n9  = n12 = n15 = n18 = n21 = n24 = -2;}
1498:     if (xe==M*dof) {n2  = n5  = n8  = n11 = n14 = n17 = n20 = n23 = n26 = -2;}
1499:   }

1501:   /* If not Y periodic */
1502:   if (!DAYPeriodic(wrap)){
1503:     if (ys==0)   {n0  = n1  = n2  = n9  = n10 = n11 = n18 = n19 = n20 = -2;}
1504:     if (ye==N)   {n6  = n7  = n8  = n15 = n16 = n17 = n24 = n25 = n26 = -2;}
1505:   }

1507:   /* If not Z periodic */
1508:   if (!DAZPeriodic(wrap)){
1509:     if (zs==0)   {n0  = n1  = n2  = n3  = n4  = n5  = n6  = n7  = n8  = -2;}
1510:     if (ze==P)   {n18 = n19 = n20 = n21 = n22 = n23 = n24 = n25 = n26 = -2;}
1511:   }

1513:   nn = 0;

1515:   /* Bottom Level */
1516:   for (k=0; k<s_z; k++) {
1517:     for (i=1; i<=s_y; i++) {
1518:       if (n0 >= 0) { /* left below */
1519:         x_t = flx[n0 % m]*dof;
1520:         y_t = fly[(n0 % (m*n))/m];
1521:         z_t = flz[n0 / (m*n)];
1522:         s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t -s_x - (s_z-k-1)*x_t*y_t;
1523:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1524:       }
1525:       if (n1 >= 0) { /* directly below */
1526:         x_t = x;
1527:         y_t = fly[(n1 % (m*n))/m];
1528:         z_t = flz[n1 / (m*n)];
1529:         s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1530:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1531:       }
1532:       if (n2 >= 0) { /* right below */
1533:         x_t = flx[n2 % m]*dof;
1534:         y_t = fly[(n2 % (m*n))/m];
1535:         z_t = flz[n2 / (m*n)];
1536:         s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1537:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1538:       }
1539:     }

1541:     for (i=0; i<y; i++) {
1542:       if (n3 >= 0) { /* directly left */
1543:         x_t = flx[n3 % m]*dof;
1544:         y_t = y;
1545:         z_t = flz[n3 / (m*n)];
1546:         s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1547:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1548:       }

1550:       if (n4 >= 0) { /* middle */
1551:         x_t = x;
1552:         y_t = y;
1553:         z_t = flz[n4 / (m*n)];
1554:         s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1555:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1556:       }

1558:       if (n5 >= 0) { /* directly right */
1559:         x_t = flx[n5 % m]*dof;
1560:         y_t = y;
1561:         z_t = flz[n5 / (m*n)];
1562:         s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1563:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1564:       }
1565:     }

1567:     for (i=1; i<=s_y; i++) {
1568:       if (n6 >= 0) { /* left above */
1569:         x_t = flx[n6 % m]*dof;
1570:         y_t = fly[(n6 % (m*n))/m];
1571:         z_t = flz[n6 / (m*n)];
1572:         s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1573:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1574:       }
1575:       if (n7 >= 0) { /* directly above */
1576:         x_t = x;
1577:         y_t = fly[(n7 % (m*n))/m];
1578:         z_t = flz[n7 / (m*n)];
1579:         s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1580:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1581:       }
1582:       if (n8 >= 0) { /* right above */
1583:         x_t = flx[n8 % m]*dof;
1584:         y_t = fly[(n8 % (m*n))/m];
1585:         z_t = flz[n8 / (m*n)];
1586:         s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1587:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1588:       }
1589:     }
1590:   }

1592:   /* Middle Level */
1593:   for (k=0; k<z; k++) {
1594:     for (i=1; i<=s_y; i++) {
1595:       if (n9 >= 0) { /* left below */
1596:         x_t = flx[n9 % m]*dof;
1597:         y_t = fly[(n9 % (m*n))/m];
1598:         /* z_t = z; */
1599:         s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1600:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1601:       }
1602:       if (n10 >= 0) { /* directly below */
1603:         x_t = x;
1604:         y_t = fly[(n10 % (m*n))/m];
1605:         /* z_t = z; */
1606:         s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1607:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1608:       }
1609:       if (n11 >= 0) { /* right below */
1610:         x_t = flx[n11 % m]*dof;
1611:         y_t = fly[(n11 % (m*n))/m];
1612:         /* z_t = z; */
1613:         s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1614:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1615:       }
1616:     }

1618:     for (i=0; i<y; i++) {
1619:       if (n12 >= 0) { /* directly left */
1620:         x_t = flx[n12 % m]*dof;
1621:         y_t = y;
1622:         /* z_t = z; */
1623:         s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
1624:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1625:       }

1627:       /* Interior */
1628:       s_t = bases[rank] + i*x + k*x*y;
1629:       for (j=0; j<x; j++) { idx[nn++] = s_t++;}

1631:       if (n14 >= 0) { /* directly right */
1632:         x_t = flx[n14 % m]*dof;
1633:         y_t = y;
1634:         /* z_t = z; */
1635:         s_t = bases[n14] + i*x_t + k*x_t*y_t;
1636:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1637:       }
1638:     }

1640:     for (i=1; i<=s_y; i++) {
1641:       if (n15 >= 0) { /* left above */
1642:         x_t = flx[n15 % m]*dof;
1643:         y_t = fly[(n15 % (m*n))/m];
1644:         /* z_t = z; */
1645:         s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
1646:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1647:       }
1648:       if (n16 >= 0) { /* directly above */
1649:         x_t = x;
1650:         y_t = fly[(n16 % (m*n))/m];
1651:         /* z_t = z; */
1652:         s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
1653:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1654:       }
1655:       if (n17 >= 0) { /* right above */
1656:         x_t = flx[n17 % m]*dof;
1657:         y_t = fly[(n17 % (m*n))/m];
1658:         /* z_t = z; */
1659:         s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
1660:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1661:       }
1662:     }
1663:   }
1664: 
1665:   /* Upper Level */
1666:   for (k=0; k<s_z; k++) {
1667:     for (i=1; i<=s_y; i++) {
1668:       if (n18 >= 0) { /* left below */
1669:         x_t = flx[n18 % m]*dof;
1670:         y_t = fly[(n18 % (m*n))/m];
1671:         /* z_t = flz[n18 / (m*n)]; */
1672:         s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1673:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1674:       }
1675:       if (n19 >= 0) { /* directly below */
1676:         x_t = x;
1677:         y_t = fly[(n19 % (m*n))/m];
1678:         /* z_t = flz[n19 / (m*n)]; */
1679:         s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1680:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1681:       }
1682:       if (n20 >= 0) { /* right belodof */
1683:         x_t = flx[n20 % m]*dof;
1684:         y_t = fly[(n20 % (m*n))/m];
1685:         /* z_t = flz[n20 / (m*n)]; */
1686:         s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1687:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1688:       }
1689:     }

1691:     for (i=0; i<y; i++) {
1692:       if (n21 >= 0) { /* directly left */
1693:         x_t = flx[n21 % m]*dof;
1694:         y_t = y;
1695:         /* z_t = flz[n21 / (m*n)]; */
1696:         s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
1697:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1698:       }

1700:       if (n22 >= 0) { /* middle */
1701:         x_t = x;
1702:         y_t = y;
1703:         /* z_t = flz[n22 / (m*n)]; */
1704:         s_t = bases[n22] + i*x_t + k*x_t*y_t;
1705:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1706:       }

1708:       if (n23 >= 0) { /* directly right */
1709:         x_t = flx[n23 % m]*dof;
1710:         y_t = y;
1711:         /* z_t = flz[n23 / (m*n)]; */
1712:         s_t = bases[n23] + i*x_t + k*x_t*y_t;
1713:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1714:       }
1715:     }

1717:     for (i=1; i<=s_y; i++) {
1718:       if (n24 >= 0) { /* left above */
1719:         x_t = flx[n24 % m]*dof;
1720:         y_t = fly[(n24 % (m*n))/m];
1721:         /* z_t = flz[n24 / (m*n)]; */
1722:         s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
1723:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1724:       }
1725:       if (n25 >= 0) { /* directly above */
1726:         x_t = x;
1727:         y_t = fly[(n25 % (m*n))/m];
1728:         /* z_t = flz[n25 / (m*n)]; */
1729:         s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1730:         for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1731:       }
1732:       if (n26 >= 0) { /* right above */
1733:         x_t = flx[n26 % m]*dof;
1734:         y_t = fly[(n26 % (m*n))/m];
1735:         /* z_t = flz[n26 / (m*n)]; */
1736:         s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1737:         for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1738:       }
1739:     }
1740:   }
1741:   PetscFree(bases);
1742:   da->gtol      = gtol;
1743:   da->ltog      = ltog;
1744:   da->idx       = idx;
1745:   da->Nl        = nn;
1746:   da->base      = base;
1747:   da->ops->view = DAView_3d;
1748:   da->wrap      = wrap;
1749:   *inra = da;

1751:   /* 
1752:      Set the local to global ordering in the global vector, this allows use
1753:      of VecSetValuesLocal().
1754:   */
1755:   ISLocalToGlobalMappingCreateNC(comm,nn,idx,&da->ltogmap);
1756:   ISLocalToGlobalMappingBlock(da->ltogmap,da->w,&da->ltogmapb);
1757:   PetscLogObjectParent(da,da->ltogmap);

1759:   da->ltol = PETSC_NULL;
1760:   da->ao   = PETSC_NULL;

1762:   da->lx = flx;
1763:   da->ly = fly;
1764:   da->lz = flz;

1766:   DAView_Private(da);
1767:   return(0);
1768: }

1772: /*@C
1773:    DACreate - Creates an object that will manage the communication of regular array data that is distributed across some processors
1774:        in 1, 2 or 3 dimensions

1776:    Collective on MPI_Comm

1778:    See the manual pages for the routines for each dimension.

1780:    Level: beginner

1782:    
1783: .keywords: distributed array, create, three-dimensional

1785: .seealso: DACreate1d(), DACreate2d(), DACreate3d(), DAGetOwnershipRanges()

1787: @*/
1788: PetscErrorCode  DACreate(MPI_Comm comm,PetscInt dim,DAPeriodicType wrap,DAStencilType stencil_type,PetscInt M,
1789:          PetscInt N,PetscInt P,PetscInt m,PetscInt n,PetscInt p,PetscInt dof,PetscInt s,const PetscInt lx[],const PetscInt ly[],const PetscInt lz[],DA *inra)
1790: {

1793:   if (dim == 3) {
1794:     DACreate3d(comm,wrap,stencil_type,M,N,P,m,n,p,dof,s,lx,ly,lz,inra);
1795:   } else if (dim == 2) {
1796:     DACreate2d(comm,wrap,stencil_type,M,N,m,n,dof,s,lx,ly,inra);
1797:   } else if (dim == 1) {
1798:     DACreate1d(comm,wrap,M,dof,s,lx,inra);
1799:   }
1800:   return(0);
1801: }