Actual source code: da3.c
1: #define PETSCDM_DLL
2: /*
3: Code for manipulating distributed regular 3d arrays in parallel.
4: File created by Peter Mell 7/14/95
5: */
7: #include ../src/dm/da/daimpl.h
11: PetscErrorCode DAView_3d(DA da,PetscViewer viewer)
12: {
14: PetscMPIInt rank;
15: PetscTruth iascii,isdraw;
18: MPI_Comm_rank(((PetscObject)da)->comm,&rank);
20: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
21: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_DRAW,&isdraw);
22: if (iascii) {
23: PetscViewerFormat format;
25: PetscViewerGetFormat(viewer, &format);
26: if (format != PETSC_VIEWER_ASCII_VTK && format != PETSC_VIEWER_ASCII_VTK_CELL) {
27: PetscViewerASCIISynchronizedPrintf(viewer,"Processor [%d] M %D N %D P %D m %D n %D p %D w %D s %D\n",
28: rank,da->M,da->N,da->P,da->m,da->n,da->p,da->w,da->s);
29: PetscViewerASCIISynchronizedPrintf(viewer,"X range of indices: %D %D, Y range of indices: %D %D, Z range of indices: %D %D\n",
30: da->xs,da->xe,da->ys,da->ye,da->zs,da->ze);
31: #if !defined(PETSC_USE_COMPLEX)
32: if (da->coordinates) {
33: PetscInt last;
34: PetscReal *coors;
35: VecGetArray(da->coordinates,&coors);
36: VecGetLocalSize(da->coordinates,&last);
37: last = last - 3;
38: PetscViewerASCIISynchronizedPrintf(viewer,"Lower left corner %G %G %G : Upper right %G %G %G\n",
39: coors[0],coors[1],coors[2],coors[last],coors[last+1],coors[last+2]);
40: VecRestoreArray(da->coordinates,&coors);
41: }
42: #endif
43: PetscViewerFlush(viewer);
44: }
45: } else if (isdraw) {
46: PetscDraw draw;
47: PetscReal ymin = -1.0,ymax = (PetscReal)da->N;
48: PetscReal xmin = -1.0,xmax = (PetscReal)((da->M+2)*da->P),x,y,ycoord,xcoord;
49: PetscInt k,plane,base,*idx;
50: char node[10];
51: PetscTruth isnull;
53: PetscViewerDrawGetDraw(viewer,0,&draw);
54: PetscDrawIsNull(draw,&isnull); if (isnull) return(0);
55: PetscDrawSetCoordinates(draw,xmin,ymin,xmax,ymax);
56: PetscDrawSynchronizedClear(draw);
58: /* first processor draw all node lines */
59: if (!rank) {
60: for (k=0; k<da->P; k++) {
61: ymin = 0.0; ymax = (PetscReal)(da->N - 1);
62: for (xmin=(PetscReal)(k*(da->M+1)); xmin<(PetscReal)(da->M+(k*(da->M+1))); xmin++) {
63: PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_BLACK);
64: }
65:
66: xmin = (PetscReal)(k*(da->M+1)); xmax = xmin + (PetscReal)(da->M - 1);
67: for (ymin=0; ymin<(PetscReal)da->N; ymin++) {
68: PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_BLACK);
69: }
70: }
71: }
72: PetscDrawSynchronizedFlush(draw);
73: PetscDrawPause(draw);
75: for (k=0; k<da->P; k++) { /*Go through and draw for each plane*/
76: if ((k >= da->zs) && (k < da->ze)) {
77: /* draw my box */
78: ymin = da->ys;
79: ymax = da->ye - 1;
80: xmin = da->xs/da->w + (da->M+1)*k;
81: xmax =(da->xe-1)/da->w + (da->M+1)*k;
83: PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_RED);
84: PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_RED);
85: PetscDrawLine(draw,xmin,ymax,xmax,ymax,PETSC_DRAW_RED);
86: PetscDrawLine(draw,xmax,ymin,xmax,ymax,PETSC_DRAW_RED);
88: xmin = da->xs/da->w;
89: xmax =(da->xe-1)/da->w;
91: /* put in numbers*/
92: base = (da->base+(da->xe-da->xs)*(da->ye-da->ys)*(k-da->zs))/da->w;
94: /* Identify which processor owns the box */
95: sprintf(node,"%d",rank);
96: PetscDrawString(draw,xmin+(da->M+1)*k+.2,ymin+.3,PETSC_DRAW_RED,node);
98: for (y=ymin; y<=ymax; y++) {
99: for (x=xmin+(da->M+1)*k; x<=xmax+(da->M+1)*k; x++) {
100: sprintf(node,"%d",(int)base++);
101: PetscDrawString(draw,x,y,PETSC_DRAW_BLACK,node);
102: }
103: }
104:
105: }
106: }
107: PetscDrawSynchronizedFlush(draw);
108: PetscDrawPause(draw);
110: for (k=0-da->s; k<da->P+da->s; k++) {
111: /* Go through and draw for each plane */
112: if ((k >= da->Zs) && (k < da->Ze)) {
113:
114: /* overlay ghost numbers, useful for error checking */
115: base = (da->Xe-da->Xs)*(da->Ye-da->Ys)*(k-da->Zs); idx = da->idx;
116: plane=k;
117: /* Keep z wrap around points on the dradrawg */
118: if (k<0) { plane=da->P+k; }
119: if (k>=da->P) { plane=k-da->P; }
120: ymin = da->Ys; ymax = da->Ye;
121: xmin = (da->M+1)*plane*da->w;
122: xmax = (da->M+1)*plane*da->w+da->M*da->w;
123: for (y=ymin; y<ymax; y++) {
124: for (x=xmin+da->Xs; x<xmin+da->Xe; x+=da->w) {
125: sprintf(node,"%d",(int)(idx[base]/da->w));
126: ycoord = y;
127: /*Keep y wrap around points on drawing */
128: if (y<0) { ycoord = da->N+y; }
130: if (y>=da->N) { ycoord = y-da->N; }
131: xcoord = x; /* Keep x wrap points on drawing */
133: if (x<xmin) { xcoord = xmax - (xmin-x); }
134: if (x>=xmax) { xcoord = xmin + (x-xmax); }
135: PetscDrawString(draw,xcoord/da->w,ycoord,PETSC_DRAW_BLUE,node);
136: base+=da->w;
137: }
138: }
139: }
140: }
141: PetscDrawSynchronizedFlush(draw);
142: PetscDrawPause(draw);
143: } else {
144: SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for DA 3d",((PetscObject)viewer)->type_name);
145: }
146: return(0);
147: }
149: #if 0
150: EXTERN PetscErrorCode DAPublish_Petsc(PetscObject);
151: #endif
155: /*@C
156: DACreate3d - Creates an object that will manage the communication of three-dimensional
157: regular array data that is distributed across some processors.
159: Collective on MPI_Comm
161: Input Parameters:
162: + comm - MPI communicator
163: . wrap - type of periodicity the array should have, if any. Use one
164: of DA_NONPERIODIC, DA_XPERIODIC, DA_YPERIODIC, DA_XYPERIODIC, DA_XYZPERIODIC, DA_XZPERIODIC, or DA_YZPERIODIC.
165: . stencil_type - Type of stencil (DA_STENCIL_STAR or DA_STENCIL_BOX)
166: . M,N,P - global dimension in each direction of the array (use -M, -N, and or -P to indicate that it may be set to a different value
167: from the command line with -da_grid_x <M> -da_grid_y <N> -da_grid_z <P>)
168: . m,n,p - corresponding number of processors in each dimension
169: (or PETSC_DECIDE to have calculated)
170: . dof - number of degrees of freedom per node
171: . lx, ly, lz - arrays containing the number of nodes in each cell along
172: the x, y, and z coordinates, or PETSC_NULL. If non-null, these
173: must be of length as m,n,p and the corresponding
174: m,n, or p cannot be PETSC_DECIDE. Sum of the lx[] entries must be M, sum of
175: the ly[] must N, sum of the lz[] must be P
176: - s - stencil width
178: Output Parameter:
179: . inra - the resulting distributed array object
181: Options Database Key:
182: + -da_view - Calls DAView() at the conclusion of DACreate3d()
183: . -da_grid_x <nx> - number of grid points in x direction, if M < 0
184: . -da_grid_y <ny> - number of grid points in y direction, if N < 0
185: . -da_grid_z <nz> - number of grid points in z direction, if P < 0
186: . -da_processors_x <MX> number of processors in x direction
187: . -da_processors_y <MY> number of processors in y direction
188: . -da_processors_z <MZ> number of processors in z direction
189: . -da_refine_x - refinement ratio in x direction
190: . -da_refine_y - refinement ratio in y direction
191: - -da_refine_y - refinement ratio in z direction
193: Level: beginner
195: Notes:
196: The stencil type DA_STENCIL_STAR with width 1 corresponds to the
197: standard 7-pt stencil, while DA_STENCIL_BOX with width 1 denotes
198: the standard 27-pt stencil.
200: The array data itself is NOT stored in the DA, it is stored in Vec objects;
201: The appropriate vector objects can be obtained with calls to DACreateGlobalVector()
202: and DACreateLocalVector() and calls to VecDuplicate() if more are needed.
204: .keywords: distributed array, create, three-dimensional
206: .seealso: DADestroy(), DAView(), DACreate1d(), DACreate2d(), DAGlobalToLocalBegin(), DAGetRefinementFactor(),
207: DAGlobalToLocalEnd(), DALocalToGlobal(), DALocalToLocalBegin(), DALocalToLocalEnd(), DASetRefinementFactor(),
208: DAGetInfo(), DACreateGlobalVector(), DACreateLocalVector(), DACreateNaturalVector(), DALoad(), DAView(), DAGetOwnershipRanges()
210: @*/
211: PetscErrorCode DACreate3d(MPI_Comm comm,DAPeriodicType wrap,DAStencilType stencil_type,PetscInt M,
212: PetscInt N,PetscInt P,PetscInt m,PetscInt n,PetscInt p,PetscInt dof,PetscInt s,const PetscInt lx[],const PetscInt ly[],const PetscInt lz[],DA *inra)
213: {
215: PetscMPIInt rank,size;
216: PetscInt xs = 0,xe,ys = 0,ye,zs = 0,ze,x = 0,y = 0,z = 0,Xs,Xe,Ys,Ye,Zs,Ze,start,end,pm;
217: PetscInt left,up,down,bottom,top,i,j,k,*idx,nn,*flx = 0,*fly = 0,*flz = 0;
218: PetscInt n0,n1,n2,n3,n4,n5,n6,n7,n8,n9,n10,n11,n12,n14;
219: PetscInt n15,n16,n17,n18,n19,n20,n21,n22,n23,n24,n25,n26;
220: PetscInt *bases,*ldims,x_t,y_t,z_t,s_t,base,count,s_x,s_y,s_z;
221: PetscInt tM = M,tN = N,tP = P;
222: PetscInt sn0 = 0,sn1 = 0,sn2 = 0,sn3 = 0,sn5 = 0,sn6 = 0,sn7 = 0;
223: PetscInt sn8 = 0,sn9 = 0,sn11 = 0,sn15 = 0,sn24 = 0,sn25 = 0,sn26 = 0;
224: PetscInt sn17 = 0,sn18 = 0,sn19 = 0,sn20 = 0,sn21 = 0,sn23 = 0,refine_x = 2, refine_y = 2, refine_z = 2;
225: DA da;
226: Vec local,global;
227: VecScatter ltog,gtol;
228: IS to,from;
232: *inra = 0;
233: #ifndef PETSC_USE_DYNAMIC_LIBRARIES
234: DMInitializePackage(PETSC_NULL);
235: #endif
237: if (dof < 1) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Must have 1 or more degrees of freedom per node: %D",dof);
238: if (s < 0) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Stencil width cannot be negative: %D",s);
240: PetscOptionsBegin(comm,PETSC_NULL,"3d DA Options","DA");
241: if (M < 0){
242: tM = -M;
243: PetscOptionsInt("-da_grid_x","Number of grid points in x direction","DACreate3d",tM,&tM,PETSC_NULL);
244: }
245: if (N < 0){
246: tN = -N;
247: PetscOptionsInt("-da_grid_y","Number of grid points in y direction","DACreate3d",tN,&tN,PETSC_NULL);
248: }
249: if (P < 0){
250: tP = -P;
251: PetscOptionsInt("-da_grid_z","Number of grid points in z direction","DACreate3d",tP,&tP,PETSC_NULL);
252: }
253: PetscOptionsInt("-da_processors_x","Number of processors in x direction","DACreate3d",m,&m,PETSC_NULL);
254: PetscOptionsInt("-da_processors_y","Number of processors in y direction","DACreate3d",n,&n,PETSC_NULL);
255: PetscOptionsInt("-da_processors_z","Number of processors in z direction","DACreate3d",p,&p,PETSC_NULL);
256: PetscOptionsInt("-da_refine_x","Refinement ratio in x direction","DASetRefinementFactor",refine_x,&refine_x,PETSC_NULL);
257: PetscOptionsInt("-da_refine_y","Refinement ratio in y direction","DASetRefinementFactor",refine_y,&refine_y,PETSC_NULL);
258: PetscOptionsInt("-da_refine_z","Refinement ratio in z direction","DASetRefinementFactor",refine_z,&refine_z,PETSC_NULL);
259: PetscOptionsEnd();
260: M = tM; N = tN; P = tP;
262: PetscHeaderCreate(da,_p_DA,struct _DAOps,DM_COOKIE,0,"DM",comm,DADestroy,DAView);
263: PetscObjectChangeTypeName((PetscObject)da,"DA");
264: da->ops->globaltolocalbegin = DAGlobalToLocalBegin;
265: da->ops->globaltolocalend = DAGlobalToLocalEnd;
266: da->ops->localtoglobal = DALocalToGlobal;
267: da->ops->createglobalvector = DACreateGlobalVector;
268: da->ops->createlocalvector = DACreateLocalVector;
269: da->ops->getinterpolation = DAGetInterpolation;
270: da->ops->getcoloring = DAGetColoring;
271: da->ops->getmatrix = DAGetMatrix;
272: da->ops->refine = DARefine;
273: da->ops->coarsen = DACoarsen;
274: da->ops->getaggregates = DAGetAggregates;
275: da->ops->destroy = DADestroy;
277: da->dim = 3;
278: da->interptype = DA_Q1;
279: da->refine_x = refine_x;
280: da->refine_y = refine_y;
281: da->refine_z = refine_z;
282: PetscMalloc(dof*sizeof(char*),&da->fieldname);
283: PetscMemzero(da->fieldname,dof*sizeof(char*));
285: MPI_Comm_size(comm,&size);
286: MPI_Comm_rank(comm,&rank);
288: if (m != PETSC_DECIDE) {
289: if (m < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in X direction: %D",m);}
290: else if (m > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in X direction: %D %d",m,size);}
291: }
292: if (n != PETSC_DECIDE) {
293: if (n < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in Y direction: %D",n);}
294: else if (n > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in Y direction: %D %d",n,size);}
295: }
296: if (p != PETSC_DECIDE) {
297: if (p < 1) {SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in Z direction: %D",p);}
298: else if (p > size) {SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in Z direction: %D %d",p,size);}
299: }
301: /* Partition the array among the processors */
302: if (m == PETSC_DECIDE && n != PETSC_DECIDE && p != PETSC_DECIDE) {
303: m = size/(n*p);
304: } else if (m != PETSC_DECIDE && n == PETSC_DECIDE && p != PETSC_DECIDE) {
305: n = size/(m*p);
306: } else if (m != PETSC_DECIDE && n != PETSC_DECIDE && p == PETSC_DECIDE) {
307: p = size/(m*n);
308: } else if (m == PETSC_DECIDE && n == PETSC_DECIDE && p != PETSC_DECIDE) {
309: /* try for squarish distribution */
310: m = (int)(0.5 + sqrt(((PetscReal)M)*((PetscReal)size)/((PetscReal)N*p)));
311: if (!m) m = 1;
312: while (m > 0) {
313: n = size/(m*p);
314: if (m*n*p == size) break;
315: m--;
316: }
317: if (!m) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad p value: p = %D",p);
318: if (M > N && m < n) {PetscInt _m = m; m = n; n = _m;}
319: } else if (m == PETSC_DECIDE && n != PETSC_DECIDE && p == PETSC_DECIDE) {
320: /* try for squarish distribution */
321: m = (int)(0.5 + sqrt(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n)));
322: if (!m) m = 1;
323: while (m > 0) {
324: p = size/(m*n);
325: if (m*n*p == size) break;
326: m--;
327: }
328: if (!m) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad n value: n = %D",n);
329: if (M > P && m < p) {PetscInt _m = m; m = p; p = _m;}
330: } else if (m != PETSC_DECIDE && n == PETSC_DECIDE && p == PETSC_DECIDE) {
331: /* try for squarish distribution */
332: n = (int)(0.5 + sqrt(((PetscReal)N)*((PetscReal)size)/((PetscReal)P*m)));
333: if (!n) n = 1;
334: while (n > 0) {
335: p = size/(m*n);
336: if (m*n*p == size) break;
337: n--;
338: }
339: if (!n) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"bad m value: m = %D",n);
340: if (N > P && n < p) {PetscInt _n = n; n = p; p = _n;}
341: } else if (m == PETSC_DECIDE && n == PETSC_DECIDE && p == PETSC_DECIDE) {
342: /* try for squarish distribution */
343: n = (PetscInt)(0.5 + pow(((PetscReal)N*N)*((PetscReal)size)/((PetscReal)P*M),(PetscReal)(1./3.)));
344: if (!n) n = 1;
345: while (n > 0) {
346: pm = size/n;
347: if (n*pm == size) break;
348: n--;
349: }
350: if (!n) n = 1;
351: m = (PetscInt)(0.5 + sqrt(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n)));
352: if (!m) m = 1;
353: while (m > 0) {
354: p = size/(m*n);
355: if (m*n*p == size) break;
356: m--;
357: }
358: if (M > P && m < p) {PetscInt _m = m; m = p; p = _m;}
359: } else if (m*n*p != size) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Given Bad partition");
361: if (m*n*p != size) SETERRQ(PETSC_ERR_PLIB,"Could not find good partition");
362: if (M < m) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in x direction is too fine! %D %D",M,m);
363: if (N < n) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in y direction is too fine! %D %D",N,n);
364: if (P < p) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Partition in z direction is too fine! %D %D",P,p);
366: /*
367: Determine locally owned region
368: [x, y, or z]s is the first local node number, [x, y, z] is the number of local nodes
369: */
371: PetscMalloc(m*sizeof(PetscInt),&flx);
372: if (lx) { /* user decided distribution */
373: PetscMemcpy(flx,lx,m*sizeof(PetscInt));
374: } else {
375: for (i=0; i<m; i++) {
376: flx[i] = M/m + ((M % m) > (i % m));
377: }
378: }
379: x = flx[rank % m];
380: xs = 0;
381: for (i=0; i<(rank%m); i++) { xs += flx[i];}
382: if (m > 1 && x < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Column width is too thin for stencil! %D %D",x,s);
384: PetscMalloc(n*sizeof(PetscInt),&fly);
385: if (ly) { /* user decided distribution */
386: PetscMemcpy(fly,ly,n*sizeof(PetscInt));
387: } else {
388: for (i=0; i<n; i++) {
389: fly[i] = N/n + ((N % n) > (i % n));
390: }
391: }
392: y = fly[(rank % (m*n))/m];
393: if (n > 1 && y < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Row width is too thin for stencil! %D %D",y,s);
394: ys = 0;
395: for (i=0; i<(rank % (m*n))/m; i++) { ys += fly[i];}
397: PetscMalloc(p*sizeof(PetscInt),&flz);
398: if (lz) { /* user decided distribution */
399: PetscMemcpy(flz,lz,p*sizeof(PetscInt));
400: } else {
401: for (i=0; i<p; i++) {
402: flz[i] = P/p + ((P % p) > (i % p));
403: }
404: }
405: z = flz[rank/(m*n)];
406: if (p > 1 && z < s) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Plane width is too thin for stencil! %D %D",z,s);
407: zs = 0;
408: for (i=0; i<(rank/(m*n)); i++) { zs += flz[i];}
409: ye = ys + y;
410: xe = xs + x;
411: ze = zs + z;
413: /* determine ghost region */
414: /* Assume No Periodicity */
415: if (xs-s > 0) Xs = xs - s; else Xs = 0;
416: if (ys-s > 0) Ys = ys - s; else Ys = 0;
417: if (zs-s > 0) Zs = zs - s; else Zs = 0;
418: if (xe+s <= M) Xe = xe + s; else Xe = M;
419: if (ye+s <= N) Ye = ye + s; else Ye = N;
420: if (ze+s <= P) Ze = ze + s; else Ze = P;
422: /* X Periodic */
423: if (DAXPeriodic(wrap)){
424: Xs = xs - s;
425: Xe = xe + s;
426: }
428: /* Y Periodic */
429: if (DAYPeriodic(wrap)){
430: Ys = ys - s;
431: Ye = ye + s;
432: }
434: /* Z Periodic */
435: if (DAZPeriodic(wrap)){
436: Zs = zs - s;
437: Ze = ze + s;
438: }
440: /* Resize all X parameters to reflect w */
441: x *= dof;
442: xs *= dof;
443: xe *= dof;
444: Xs *= dof;
445: Xe *= dof;
446: s_x = s*dof;
447: s_y = s;
448: s_z = s;
450: /* determine starting point of each processor */
451: nn = x*y*z;
452: PetscMalloc((2*size+1)*sizeof(PetscInt),&bases);
453: ldims = (PetscInt*)(bases+size+1);
454: MPI_Allgather(&nn,1,MPIU_INT,ldims,1,MPIU_INT,comm);
455: bases[0] = 0;
456: for (i=1; i<=size; i++) {
457: bases[i] = ldims[i-1];
458: }
459: for (i=1; i<=size; i++) {
460: bases[i] += bases[i-1];
461: }
463: /* allocate the base parallel and sequential vectors */
464: da->Nlocal = x*y*z;
465: VecCreateMPIWithArray(comm,da->Nlocal,PETSC_DECIDE,0,&global);
466: VecSetBlockSize(global,dof);
467: da->nlocal = (Xe-Xs)*(Ye-Ys)*(Ze-Zs);
468: VecCreateSeqWithArray(MPI_COMM_SELF,da->nlocal,0,&local);
469: VecSetBlockSize(local,dof);
471: /* generate appropriate vector scatters */
472: /* local to global inserts non-ghost point region into global */
473: VecGetOwnershipRange(global,&start,&end);
474: ISCreateStride(comm,x*y*z,start,1,&to);
476: left = xs - Xs;
477: bottom = ys - Ys; top = bottom + y;
478: down = zs - Zs; up = down + z;
479: count = x*(top-bottom)*(up-down);
480: PetscMalloc(count*sizeof(PetscInt)/dof,&idx);
481: count = 0;
482: for (i=down; i<up; i++) {
483: for (j=bottom; j<top; j++) {
484: for (k=0; k<x; k += dof) {
485: idx[count++] = (left+j*(Xe-Xs))+i*(Xe-Xs)*(Ye-Ys) + k;
486: }
487: }
488: }
489: ISCreateBlock(comm,dof,count,idx,&from);
490: PetscFree(idx);
492: VecScatterCreate(local,from,global,to,<og);
493: PetscLogObjectParent(da,to);
494: PetscLogObjectParent(da,from);
495: PetscLogObjectParent(da,ltog);
496: ISDestroy(from);
497: ISDestroy(to);
499: /* global to local must include ghost points */
500: if (stencil_type == DA_STENCIL_BOX) {
501: ISCreateStride(comm,(Xe-Xs)*(Ye-Ys)*(Ze-Zs),0,1,&to);
502: } else {
503: /* This is way ugly! We need to list the funny cross type region */
504: /* the bottom chunck */
505: left = xs - Xs;
506: bottom = ys - Ys; top = bottom + y;
507: down = zs - Zs; up = down + z;
508: count = down*(top-bottom)*x + (up-down)*(bottom*x + (top-bottom)*(Xe-Xs) + (Ye-Ys-top)*x) + (Ze-Zs-up)*(top-bottom)*x;
509: PetscMalloc(count*sizeof(PetscInt)/dof,&idx);
510: count = 0;
511: for (i=0; i<down; i++) {
512: for (j=bottom; j<top; j++) {
513: for (k=0; k<x; k += dof) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
514: }
515: }
516: /* the middle piece */
517: for (i=down; i<up; i++) {
518: /* front */
519: for (j=0; j<bottom; j++) {
520: for (k=0; k<x; k += dof) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
521: }
522: /* middle */
523: for (j=bottom; j<top; j++) {
524: for (k=0; k<Xe-Xs; k += dof) idx[count++] = j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
525: }
526: /* back */
527: for (j=top; j<Ye-Ys; j++) {
528: for (k=0; k<x; k += dof) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
529: }
530: }
531: /* the top piece */
532: for (i=up; i<Ze-Zs; i++) {
533: for (j=bottom; j<top; j++) {
534: for (k=0; k<x; k += dof) idx[count++] = left+j*(Xe-Xs)+i*(Xe-Xs)*(Ye-Ys)+k;
535: }
536: }
537: ISCreateBlock(comm,dof,count,idx,&to);
538: PetscFree(idx);
539: }
541: /* determine who lies on each side of use stored in n24 n25 n26
542: n21 n22 n23
543: n18 n19 n20
545: n15 n16 n17
546: n12 n14
547: n9 n10 n11
549: n6 n7 n8
550: n3 n4 n5
551: n0 n1 n2
552: */
553:
554: /* Solve for X,Y, and Z Periodic Case First, Then Modify Solution */
555:
556: /* Assume Nodes are Internal to the Cube */
557:
558: n0 = rank - m*n - m - 1;
559: n1 = rank - m*n - m;
560: n2 = rank - m*n - m + 1;
561: n3 = rank - m*n -1;
562: n4 = rank - m*n;
563: n5 = rank - m*n + 1;
564: n6 = rank - m*n + m - 1;
565: n7 = rank - m*n + m;
566: n8 = rank - m*n + m + 1;
568: n9 = rank - m - 1;
569: n10 = rank - m;
570: n11 = rank - m + 1;
571: n12 = rank - 1;
572: n14 = rank + 1;
573: n15 = rank + m - 1;
574: n16 = rank + m;
575: n17 = rank + m + 1;
577: n18 = rank + m*n - m - 1;
578: n19 = rank + m*n - m;
579: n20 = rank + m*n - m + 1;
580: n21 = rank + m*n - 1;
581: n22 = rank + m*n;
582: n23 = rank + m*n + 1;
583: n24 = rank + m*n + m - 1;
584: n25 = rank + m*n + m;
585: n26 = rank + m*n + m + 1;
587: /* Assume Pieces are on Faces of Cube */
589: if (xs == 0) { /* First assume not corner or edge */
590: n0 = rank -1 - (m*n);
591: n3 = rank + m -1 - (m*n);
592: n6 = rank + 2*m -1 - (m*n);
593: n9 = rank -1;
594: n12 = rank + m -1;
595: n15 = rank + 2*m -1;
596: n18 = rank -1 + (m*n);
597: n21 = rank + m -1 + (m*n);
598: n24 = rank + 2*m -1 + (m*n);
599: }
601: if (xe == M*dof) { /* First assume not corner or edge */
602: n2 = rank -2*m +1 - (m*n);
603: n5 = rank - m +1 - (m*n);
604: n8 = rank +1 - (m*n);
605: n11 = rank -2*m +1;
606: n14 = rank - m +1;
607: n17 = rank +1;
608: n20 = rank -2*m +1 + (m*n);
609: n23 = rank - m +1 + (m*n);
610: n26 = rank +1 + (m*n);
611: }
613: if (ys==0) { /* First assume not corner or edge */
614: n0 = rank + m * (n-1) -1 - (m*n);
615: n1 = rank + m * (n-1) - (m*n);
616: n2 = rank + m * (n-1) +1 - (m*n);
617: n9 = rank + m * (n-1) -1;
618: n10 = rank + m * (n-1);
619: n11 = rank + m * (n-1) +1;
620: n18 = rank + m * (n-1) -1 + (m*n);
621: n19 = rank + m * (n-1) + (m*n);
622: n20 = rank + m * (n-1) +1 + (m*n);
623: }
625: if (ye == N) { /* First assume not corner or edge */
626: n6 = rank - m * (n-1) -1 - (m*n);
627: n7 = rank - m * (n-1) - (m*n);
628: n8 = rank - m * (n-1) +1 - (m*n);
629: n15 = rank - m * (n-1) -1;
630: n16 = rank - m * (n-1);
631: n17 = rank - m * (n-1) +1;
632: n24 = rank - m * (n-1) -1 + (m*n);
633: n25 = rank - m * (n-1) + (m*n);
634: n26 = rank - m * (n-1) +1 + (m*n);
635: }
636:
637: if (zs == 0) { /* First assume not corner or edge */
638: n0 = size - (m*n) + rank - m - 1;
639: n1 = size - (m*n) + rank - m;
640: n2 = size - (m*n) + rank - m + 1;
641: n3 = size - (m*n) + rank - 1;
642: n4 = size - (m*n) + rank;
643: n5 = size - (m*n) + rank + 1;
644: n6 = size - (m*n) + rank + m - 1;
645: n7 = size - (m*n) + rank + m ;
646: n8 = size - (m*n) + rank + m + 1;
647: }
649: if (ze == P) { /* First assume not corner or edge */
650: n18 = (m*n) - (size-rank) - m - 1;
651: n19 = (m*n) - (size-rank) - m;
652: n20 = (m*n) - (size-rank) - m + 1;
653: n21 = (m*n) - (size-rank) - 1;
654: n22 = (m*n) - (size-rank);
655: n23 = (m*n) - (size-rank) + 1;
656: n24 = (m*n) - (size-rank) + m - 1;
657: n25 = (m*n) - (size-rank) + m;
658: n26 = (m*n) - (size-rank) + m + 1;
659: }
661: if ((xs==0) && (zs==0)) { /* Assume an edge, not corner */
662: n0 = size - m*n + rank + m-1 - m;
663: n3 = size - m*n + rank + m-1;
664: n6 = size - m*n + rank + m-1 + m;
665: }
666:
667: if ((xs==0) && (ze==P)) { /* Assume an edge, not corner */
668: n18 = m*n - (size - rank) + m-1 - m;
669: n21 = m*n - (size - rank) + m-1;
670: n24 = m*n - (size - rank) + m-1 + m;
671: }
673: if ((xs==0) && (ys==0)) { /* Assume an edge, not corner */
674: n0 = rank + m*n -1 - m*n;
675: n9 = rank + m*n -1;
676: n18 = rank + m*n -1 + m*n;
677: }
679: if ((xs==0) && (ye==N)) { /* Assume an edge, not corner */
680: n6 = rank - m*(n-1) + m-1 - m*n;
681: n15 = rank - m*(n-1) + m-1;
682: n24 = rank - m*(n-1) + m-1 + m*n;
683: }
685: if ((xe==M*dof) && (zs==0)) { /* Assume an edge, not corner */
686: n2 = size - (m*n-rank) - (m-1) - m;
687: n5 = size - (m*n-rank) - (m-1);
688: n8 = size - (m*n-rank) - (m-1) + m;
689: }
691: if ((xe==M*dof) && (ze==P)) { /* Assume an edge, not corner */
692: n20 = m*n - (size - rank) - (m-1) - m;
693: n23 = m*n - (size - rank) - (m-1);
694: n26 = m*n - (size - rank) - (m-1) + m;
695: }
697: if ((xe==M*dof) && (ys==0)) { /* Assume an edge, not corner */
698: n2 = rank + m*(n-1) - (m-1) - m*n;
699: n11 = rank + m*(n-1) - (m-1);
700: n20 = rank + m*(n-1) - (m-1) + m*n;
701: }
703: if ((xe==M*dof) && (ye==N)) { /* Assume an edge, not corner */
704: n8 = rank - m*n +1 - m*n;
705: n17 = rank - m*n +1;
706: n26 = rank - m*n +1 + m*n;
707: }
709: if ((ys==0) && (zs==0)) { /* Assume an edge, not corner */
710: n0 = size - m + rank -1;
711: n1 = size - m + rank;
712: n2 = size - m + rank +1;
713: }
715: if ((ys==0) && (ze==P)) { /* Assume an edge, not corner */
716: n18 = m*n - (size - rank) + m*(n-1) -1;
717: n19 = m*n - (size - rank) + m*(n-1);
718: n20 = m*n - (size - rank) + m*(n-1) +1;
719: }
721: if ((ye==N) && (zs==0)) { /* Assume an edge, not corner */
722: n6 = size - (m*n-rank) - m * (n-1) -1;
723: n7 = size - (m*n-rank) - m * (n-1);
724: n8 = size - (m*n-rank) - m * (n-1) +1;
725: }
727: if ((ye==N) && (ze==P)) { /* Assume an edge, not corner */
728: n24 = rank - (size-m) -1;
729: n25 = rank - (size-m);
730: n26 = rank - (size-m) +1;
731: }
733: /* Check for Corners */
734: if ((xs==0) && (ys==0) && (zs==0)) { n0 = size -1;}
735: if ((xs==0) && (ys==0) && (ze==P)) { n18 = m*n-1;}
736: if ((xs==0) && (ye==N) && (zs==0)) { n6 = (size-1)-m*(n-1);}
737: if ((xs==0) && (ye==N) && (ze==P)) { n24 = m-1;}
738: if ((xe==M*dof) && (ys==0) && (zs==0)) { n2 = size-m;}
739: if ((xe==M*dof) && (ys==0) && (ze==P)) { n20 = m*n-m;}
740: if ((xe==M*dof) && (ye==N) && (zs==0)) { n8 = size-m*n;}
741: if ((xe==M*dof) && (ye==N) && (ze==P)) { n26 = 0;}
743: /* Check for when not X,Y, and Z Periodic */
745: /* If not X periodic */
746: if ((wrap != DA_XPERIODIC) && (wrap != DA_XYPERIODIC) &&
747: (wrap != DA_XZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
748: if (xs==0) {n0 = n3 = n6 = n9 = n12 = n15 = n18 = n21 = n24 = -2;}
749: if (xe==M*dof) {n2 = n5 = n8 = n11 = n14 = n17 = n20 = n23 = n26 = -2;}
750: }
752: /* If not Y periodic */
753: if ((wrap != DA_YPERIODIC) && (wrap != DA_XYPERIODIC) &&
754: (wrap != DA_YZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
755: if (ys==0) {n0 = n1 = n2 = n9 = n10 = n11 = n18 = n19 = n20 = -2;}
756: if (ye==N) {n6 = n7 = n8 = n15 = n16 = n17 = n24 = n25 = n26 = -2;}
757: }
759: /* If not Z periodic */
760: if ((wrap != DA_ZPERIODIC) && (wrap != DA_XZPERIODIC) &&
761: (wrap != DA_YZPERIODIC) && (wrap != DA_XYZPERIODIC)) {
762: if (zs==0) {n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = -2;}
763: if (ze==P) {n18 = n19 = n20 = n21 = n22 = n23 = n24 = n25 = n26 = -2;}
764: }
766: PetscMalloc(27*sizeof(PetscInt),&da->neighbors);
767: da->neighbors[0] = n0;
768: da->neighbors[1] = n1;
769: da->neighbors[2] = n2;
770: da->neighbors[3] = n3;
771: da->neighbors[4] = n4;
772: da->neighbors[5] = n5;
773: da->neighbors[6] = n6;
774: da->neighbors[7] = n7;
775: da->neighbors[8] = n8;
776: da->neighbors[9] = n9;
777: da->neighbors[10] = n10;
778: da->neighbors[11] = n11;
779: da->neighbors[12] = n12;
780: da->neighbors[13] = rank;
781: da->neighbors[14] = n14;
782: da->neighbors[15] = n15;
783: da->neighbors[16] = n16;
784: da->neighbors[17] = n17;
785: da->neighbors[18] = n18;
786: da->neighbors[19] = n19;
787: da->neighbors[20] = n20;
788: da->neighbors[21] = n21;
789: da->neighbors[22] = n22;
790: da->neighbors[23] = n23;
791: da->neighbors[24] = n24;
792: da->neighbors[25] = n25;
793: da->neighbors[26] = n26;
795: /* If star stencil then delete the corner neighbors */
796: if (stencil_type == DA_STENCIL_STAR) {
797: /* save information about corner neighbors */
798: sn0 = n0; sn1 = n1; sn2 = n2; sn3 = n3; sn5 = n5; sn6 = n6; sn7 = n7;
799: sn8 = n8; sn9 = n9; sn11 = n11; sn15 = n15; sn17 = n17; sn18 = n18;
800: sn19 = n19; sn20 = n20; sn21 = n21; sn23 = n23; sn24 = n24; sn25 = n25;
801: sn26 = n26;
802: n0 = n1 = n2 = n3 = n5 = n6 = n7 = n8 = n9 = n11 =
803: n15 = n17 = n18 = n19 = n20 = n21 = n23 = n24 = n25 = n26 = -1;
804: }
807: PetscMalloc((Xe-Xs)*(Ye-Ys)*(Ze-Zs)*sizeof(PetscInt),&idx);
808: PetscLogObjectMemory(da,(Xe-Xs)*(Ye-Ys)*(Ze-Zs)*sizeof(PetscInt));
810: nn = 0;
812: /* Bottom Level */
813: for (k=0; k<s_z; k++) {
814: for (i=1; i<=s_y; i++) {
815: if (n0 >= 0) { /* left below */
816: x_t = flx[n0 % m]*dof;
817: y_t = fly[(n0 % (m*n))/m];
818: z_t = flz[n0 / (m*n)];
819: s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t - s_x - (s_z-k-1)*x_t*y_t;
820: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
821: }
822: if (n1 >= 0) { /* directly below */
823: x_t = x;
824: y_t = fly[(n1 % (m*n))/m];
825: z_t = flz[n1 / (m*n)];
826: s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
827: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
828: }
829: if (n2 >= 0) { /* right below */
830: x_t = flx[n2 % m]*dof;
831: y_t = fly[(n2 % (m*n))/m];
832: z_t = flz[n2 / (m*n)];
833: s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
834: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
835: }
836: }
838: for (i=0; i<y; i++) {
839: if (n3 >= 0) { /* directly left */
840: x_t = flx[n3 % m]*dof;
841: y_t = y;
842: z_t = flz[n3 / (m*n)];
843: s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
844: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
845: }
847: if (n4 >= 0) { /* middle */
848: x_t = x;
849: y_t = y;
850: z_t = flz[n4 / (m*n)];
851: s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
852: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
853: }
855: if (n5 >= 0) { /* directly right */
856: x_t = flx[n5 % m]*dof;
857: y_t = y;
858: z_t = flz[n5 / (m*n)];
859: s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
860: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
861: }
862: }
864: for (i=1; i<=s_y; i++) {
865: if (n6 >= 0) { /* left above */
866: x_t = flx[n6 % m]*dof;
867: y_t = fly[(n6 % (m*n))/m];
868: z_t = flz[n6 / (m*n)];
869: s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
870: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
871: }
872: if (n7 >= 0) { /* directly above */
873: x_t = x;
874: y_t = fly[(n7 % (m*n))/m];
875: z_t = flz[n7 / (m*n)];
876: s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
877: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
878: }
879: if (n8 >= 0) { /* right above */
880: x_t = flx[n8 % m]*dof;
881: y_t = fly[(n8 % (m*n))/m];
882: z_t = flz[n8 / (m*n)];
883: s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
884: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
885: }
886: }
887: }
889: /* Middle Level */
890: for (k=0; k<z; k++) {
891: for (i=1; i<=s_y; i++) {
892: if (n9 >= 0) { /* left below */
893: x_t = flx[n9 % m]*dof;
894: y_t = fly[(n9 % (m*n))/m];
895: /* z_t = z; */
896: s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
897: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
898: }
899: if (n10 >= 0) { /* directly below */
900: x_t = x;
901: y_t = fly[(n10 % (m*n))/m];
902: /* z_t = z; */
903: s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
904: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
905: }
906: if (n11 >= 0) { /* right below */
907: x_t = flx[n11 % m]*dof;
908: y_t = fly[(n11 % (m*n))/m];
909: /* z_t = z; */
910: s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
911: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
912: }
913: }
915: for (i=0; i<y; i++) {
916: if (n12 >= 0) { /* directly left */
917: x_t = flx[n12 % m]*dof;
918: y_t = y;
919: /* z_t = z; */
920: s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
921: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
922: }
924: /* Interior */
925: s_t = bases[rank] + i*x + k*x*y;
926: for (j=0; j<x; j++) { idx[nn++] = s_t++;}
928: if (n14 >= 0) { /* directly right */
929: x_t = flx[n14 % m]*dof;
930: y_t = y;
931: /* z_t = z; */
932: s_t = bases[n14] + i*x_t + k*x_t*y_t;
933: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
934: }
935: }
937: for (i=1; i<=s_y; i++) {
938: if (n15 >= 0) { /* left above */
939: x_t = flx[n15 % m]*dof;
940: y_t = fly[(n15 % (m*n))/m];
941: /* z_t = z; */
942: s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
943: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
944: }
945: if (n16 >= 0) { /* directly above */
946: x_t = x;
947: y_t = fly[(n16 % (m*n))/m];
948: /* z_t = z; */
949: s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
950: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
951: }
952: if (n17 >= 0) { /* right above */
953: x_t = flx[n17 % m]*dof;
954: y_t = fly[(n17 % (m*n))/m];
955: /* z_t = z; */
956: s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
957: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
958: }
959: }
960: }
961:
962: /* Upper Level */
963: for (k=0; k<s_z; k++) {
964: for (i=1; i<=s_y; i++) {
965: if (n18 >= 0) { /* left below */
966: x_t = flx[n18 % m]*dof;
967: y_t = fly[(n18 % (m*n))/m];
968: /* z_t = flz[n18 / (m*n)]; */
969: s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
970: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
971: }
972: if (n19 >= 0) { /* directly below */
973: x_t = x;
974: y_t = fly[(n19 % (m*n))/m];
975: /* z_t = flz[n19 / (m*n)]; */
976: s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
977: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
978: }
979: if (n20 >= 0) { /* right below */
980: x_t = flx[n20 % m]*dof;
981: y_t = fly[(n20 % (m*n))/m];
982: /* z_t = flz[n20 / (m*n)]; */
983: s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
984: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
985: }
986: }
988: for (i=0; i<y; i++) {
989: if (n21 >= 0) { /* directly left */
990: x_t = flx[n21 % m]*dof;
991: y_t = y;
992: /* z_t = flz[n21 / (m*n)]; */
993: s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
994: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
995: }
997: if (n22 >= 0) { /* middle */
998: x_t = x;
999: y_t = y;
1000: /* z_t = flz[n22 / (m*n)]; */
1001: s_t = bases[n22] + i*x_t + k*x_t*y_t;
1002: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1003: }
1005: if (n23 >= 0) { /* directly right */
1006: x_t = flx[n23 % m]*dof;
1007: y_t = y;
1008: /* z_t = flz[n23 / (m*n)]; */
1009: s_t = bases[n23] + i*x_t + k*x_t*y_t;
1010: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1011: }
1012: }
1014: for (i=1; i<=s_y; i++) {
1015: if (n24 >= 0) { /* left above */
1016: x_t = flx[n24 % m]*dof;
1017: y_t = fly[(n24 % (m*n))/m];
1018: /* z_t = flz[n24 / (m*n)]; */
1019: s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
1020: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1021: }
1022: if (n25 >= 0) { /* directly above */
1023: x_t = x;
1024: y_t = fly[(n25 % (m*n))/m];
1025: /* z_t = flz[n25 / (m*n)]; */
1026: s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1027: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1028: }
1029: if (n26 >= 0) { /* right above */
1030: x_t = flx[n26 % m]*dof;
1031: y_t = fly[(n26 % (m*n))/m];
1032: /* z_t = flz[n26 / (m*n)]; */
1033: s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1034: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1035: }
1036: }
1037: }
1038: base = bases[rank];
1039: {
1040: PetscInt nnn = nn/dof,*iidx;
1041: PetscMalloc(nnn*sizeof(PetscInt),&iidx);
1042: for (i=0; i<nnn; i++) {
1043: iidx[i] = idx[dof*i];
1044: }
1045: ISCreateBlock(comm,dof,nnn,iidx,&from);
1046: PetscFree(iidx);
1047: }
1048: VecScatterCreate(global,from,local,to,>ol);
1049: PetscLogObjectParent(da,gtol);
1050: PetscLogObjectParent(da,to);
1051: PetscLogObjectParent(da,from);
1052: ISDestroy(to);
1053: ISDestroy(from);
1054: da->stencil_type = stencil_type;
1055: da->M = M; da->N = N; da->P = P;
1056: da->m = m; da->n = n; da->p = p;
1057: da->w = dof; da->s = s;
1058: da->xs = xs; da->xe = xe; da->ys = ys; da->ye = ye; da->zs = zs; da->ze = ze;
1059: da->Xs = Xs; da->Xe = Xe; da->Ys = Ys; da->Ye = Ye; da->Zs = Zs; da->Ze = Ze;
1061: VecDestroy(local);
1062: VecDestroy(global);
1064: if (stencil_type == DA_STENCIL_STAR) {
1065: /*
1066: Recompute the local to global mappings, this time keeping the
1067: information about the cross corner processor numbers.
1068: */
1069: n0 = sn0; n1 = sn1; n2 = sn2; n3 = sn3; n5 = sn5; n6 = sn6; n7 = sn7;
1070: n8 = sn8; n9 = sn9; n11 = sn11; n15 = sn15; n17 = sn17; n18 = sn18;
1071: n19 = sn19; n20 = sn20; n21 = sn21; n23 = sn23; n24 = sn24; n25 = sn25;
1072: n26 = sn26;
1074: nn = 0;
1076: /* Bottom Level */
1077: for (k=0; k<s_z; k++) {
1078: for (i=1; i<=s_y; i++) {
1079: if (n0 >= 0) { /* left below */
1080: x_t = flx[n0 % m]*dof;
1081: y_t = fly[(n0 % (m*n))/m];
1082: z_t = flz[n0 / (m*n)];
1083: s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t - s_x - (s_z-k-1)*x_t*y_t;
1084: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1085: }
1086: if (n1 >= 0) { /* directly below */
1087: x_t = x;
1088: y_t = fly[(n1 % (m*n))/m];
1089: z_t = flz[n1 / (m*n)];
1090: s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1091: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1092: }
1093: if (n2 >= 0) { /* right below */
1094: x_t = flx[n2 % m]*dof;
1095: y_t = fly[(n2 % (m*n))/m];
1096: z_t = flz[n2 / (m*n)];
1097: s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1098: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1099: }
1100: }
1102: for (i=0; i<y; i++) {
1103: if (n3 >= 0) { /* directly left */
1104: x_t = flx[n3 % m]*dof;
1105: y_t = y;
1106: z_t = flz[n3 / (m*n)];
1107: s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1108: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1109: }
1111: if (n4 >= 0) { /* middle */
1112: x_t = x;
1113: y_t = y;
1114: z_t = flz[n4 / (m*n)];
1115: s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1116: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1117: }
1119: if (n5 >= 0) { /* directly right */
1120: x_t = flx[n5 % m]*dof;
1121: y_t = y;
1122: z_t = flz[n5 / (m*n)];
1123: s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1124: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1125: }
1126: }
1128: for (i=1; i<=s_y; i++) {
1129: if (n6 >= 0) { /* left above */
1130: x_t = flx[n6 % m]*dof;
1131: y_t = fly[(n6 % (m*n))/m];
1132: z_t = flz[n6 / (m*n)];
1133: s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1134: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1135: }
1136: if (n7 >= 0) { /* directly above */
1137: x_t = x;
1138: y_t = fly[(n7 % (m*n))/m];
1139: z_t = flz[n7 / (m*n)];
1140: s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1141: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1142: }
1143: if (n8 >= 0) { /* right above */
1144: x_t = flx[n8 % m]*dof;
1145: y_t = fly[(n8 % (m*n))/m];
1146: z_t = flz[n8 / (m*n)];
1147: s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1148: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1149: }
1150: }
1151: }
1153: /* Middle Level */
1154: for (k=0; k<z; k++) {
1155: for (i=1; i<=s_y; i++) {
1156: if (n9 >= 0) { /* left below */
1157: x_t = flx[n9 % m]*dof;
1158: y_t = fly[(n9 % (m*n))/m];
1159: /* z_t = z; */
1160: s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1161: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1162: }
1163: if (n10 >= 0) { /* directly below */
1164: x_t = x;
1165: y_t = fly[(n10 % (m*n))/m];
1166: /* z_t = z; */
1167: s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1168: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1169: }
1170: if (n11 >= 0) { /* right below */
1171: x_t = flx[n11 % m]*dof;
1172: y_t = fly[(n11 % (m*n))/m];
1173: /* z_t = z; */
1174: s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1175: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1176: }
1177: }
1179: for (i=0; i<y; i++) {
1180: if (n12 >= 0) { /* directly left */
1181: x_t = flx[n12 % m]*dof;
1182: y_t = y;
1183: /* z_t = z; */
1184: s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
1185: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1186: }
1188: /* Interior */
1189: s_t = bases[rank] + i*x + k*x*y;
1190: for (j=0; j<x; j++) { idx[nn++] = s_t++;}
1192: if (n14 >= 0) { /* directly right */
1193: x_t = flx[n14 % m]*dof;
1194: y_t = y;
1195: /* z_t = z; */
1196: s_t = bases[n14] + i*x_t + k*x_t*y_t;
1197: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1198: }
1199: }
1201: for (i=1; i<=s_y; i++) {
1202: if (n15 >= 0) { /* left above */
1203: x_t = flx[n15 % m]*dof;
1204: y_t = fly[(n15 % (m*n))/m];
1205: /* z_t = z; */
1206: s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
1207: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1208: }
1209: if (n16 >= 0) { /* directly above */
1210: x_t = x;
1211: y_t = fly[(n16 % (m*n))/m];
1212: /* z_t = z; */
1213: s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
1214: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1215: }
1216: if (n17 >= 0) { /* right above */
1217: x_t = flx[n17 % m]*dof;
1218: y_t = fly[(n17 % (m*n))/m];
1219: /* z_t = z; */
1220: s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
1221: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1222: }
1223: }
1224: }
1225:
1226: /* Upper Level */
1227: for (k=0; k<s_z; k++) {
1228: for (i=1; i<=s_y; i++) {
1229: if (n18 >= 0) { /* left below */
1230: x_t = flx[n18 % m]*dof;
1231: y_t = fly[(n18 % (m*n))/m];
1232: /* z_t = flz[n18 / (m*n)]; */
1233: s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1234: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1235: }
1236: if (n19 >= 0) { /* directly below */
1237: x_t = x;
1238: y_t = fly[(n19 % (m*n))/m];
1239: /* z_t = flz[n19 / (m*n)]; */
1240: s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1241: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1242: }
1243: if (n20 >= 0) { /* right below */
1244: x_t = flx[n20 % m]*dof;
1245: y_t = fly[(n20 % (m*n))/m];
1246: /* z_t = flz[n20 / (m*n)]; */
1247: s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1248: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1249: }
1250: }
1252: for (i=0; i<y; i++) {
1253: if (n21 >= 0) { /* directly left */
1254: x_t = flx[n21 % m]*dof;
1255: y_t = y;
1256: /* z_t = flz[n21 / (m*n)]; */
1257: s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
1258: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1259: }
1261: if (n22 >= 0) { /* middle */
1262: x_t = x;
1263: y_t = y;
1264: /* z_t = flz[n22 / (m*n)]; */
1265: s_t = bases[n22] + i*x_t + k*x_t*y_t;
1266: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1267: }
1269: if (n23 >= 0) { /* directly right */
1270: x_t = flx[n23 % m]*dof;
1271: y_t = y;
1272: /* z_t = flz[n23 / (m*n)]; */
1273: s_t = bases[n23] + i*x_t + k*x_t*y_t;
1274: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1275: }
1276: }
1278: for (i=1; i<=s_y; i++) {
1279: if (n24 >= 0) { /* left above */
1280: x_t = flx[n24 % m]*dof;
1281: y_t = fly[(n24 % (m*n))/m];
1282: /* z_t = flz[n24 / (m*n)]; */
1283: s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
1284: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1285: }
1286: if (n25 >= 0) { /* directly above */
1287: x_t = x;
1288: y_t = fly[(n25 % (m*n))/m];
1289: /* z_t = flz[n25 / (m*n)]; */
1290: s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1291: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1292: }
1293: if (n26 >= 0) { /* right above */
1294: x_t = flx[n26 % m]*dof;
1295: y_t = fly[(n26 % (m*n))/m];
1296: /* z_t = flz[n26 / (m*n)]; */
1297: s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1298: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1299: }
1300: }
1301: }
1302: }
1303: /* redo idx to include "missing" ghost points */
1304: /* Solve for X,Y, and Z Periodic Case First, Then Modify Solution */
1305:
1306: /* Assume Nodes are Internal to the Cube */
1307:
1308: n0 = rank - m*n - m - 1;
1309: n1 = rank - m*n - m;
1310: n2 = rank - m*n - m + 1;
1311: n3 = rank - m*n -1;
1312: n4 = rank - m*n;
1313: n5 = rank - m*n + 1;
1314: n6 = rank - m*n + m - 1;
1315: n7 = rank - m*n + m;
1316: n8 = rank - m*n + m + 1;
1318: n9 = rank - m - 1;
1319: n10 = rank - m;
1320: n11 = rank - m + 1;
1321: n12 = rank - 1;
1322: n14 = rank + 1;
1323: n15 = rank + m - 1;
1324: n16 = rank + m;
1325: n17 = rank + m + 1;
1327: n18 = rank + m*n - m - 1;
1328: n19 = rank + m*n - m;
1329: n20 = rank + m*n - m + 1;
1330: n21 = rank + m*n - 1;
1331: n22 = rank + m*n;
1332: n23 = rank + m*n + 1;
1333: n24 = rank + m*n + m - 1;
1334: n25 = rank + m*n + m;
1335: n26 = rank + m*n + m + 1;
1337: /* Assume Pieces are on Faces of Cube */
1339: if (xs == 0) { /* First assume not corner or edge */
1340: n0 = rank -1 - (m*n);
1341: n3 = rank + m -1 - (m*n);
1342: n6 = rank + 2*m -1 - (m*n);
1343: n9 = rank -1;
1344: n12 = rank + m -1;
1345: n15 = rank + 2*m -1;
1346: n18 = rank -1 + (m*n);
1347: n21 = rank + m -1 + (m*n);
1348: n24 = rank + 2*m -1 + (m*n);
1349: }
1351: if (xe == M*dof) { /* First assume not corner or edge */
1352: n2 = rank -2*m +1 - (m*n);
1353: n5 = rank - m +1 - (m*n);
1354: n8 = rank +1 - (m*n);
1355: n11 = rank -2*m +1;
1356: n14 = rank - m +1;
1357: n17 = rank +1;
1358: n20 = rank -2*m +1 + (m*n);
1359: n23 = rank - m +1 + (m*n);
1360: n26 = rank +1 + (m*n);
1361: }
1363: if (ys==0) { /* First assume not corner or edge */
1364: n0 = rank + m * (n-1) -1 - (m*n);
1365: n1 = rank + m * (n-1) - (m*n);
1366: n2 = rank + m * (n-1) +1 - (m*n);
1367: n9 = rank + m * (n-1) -1;
1368: n10 = rank + m * (n-1);
1369: n11 = rank + m * (n-1) +1;
1370: n18 = rank + m * (n-1) -1 + (m*n);
1371: n19 = rank + m * (n-1) + (m*n);
1372: n20 = rank + m * (n-1) +1 + (m*n);
1373: }
1375: if (ye == N) { /* First assume not corner or edge */
1376: n6 = rank - m * (n-1) -1 - (m*n);
1377: n7 = rank - m * (n-1) - (m*n);
1378: n8 = rank - m * (n-1) +1 - (m*n);
1379: n15 = rank - m * (n-1) -1;
1380: n16 = rank - m * (n-1);
1381: n17 = rank - m * (n-1) +1;
1382: n24 = rank - m * (n-1) -1 + (m*n);
1383: n25 = rank - m * (n-1) + (m*n);
1384: n26 = rank - m * (n-1) +1 + (m*n);
1385: }
1386:
1387: if (zs == 0) { /* First assume not corner or edge */
1388: n0 = size - (m*n) + rank - m - 1;
1389: n1 = size - (m*n) + rank - m;
1390: n2 = size - (m*n) + rank - m + 1;
1391: n3 = size - (m*n) + rank - 1;
1392: n4 = size - (m*n) + rank;
1393: n5 = size - (m*n) + rank + 1;
1394: n6 = size - (m*n) + rank + m - 1;
1395: n7 = size - (m*n) + rank + m ;
1396: n8 = size - (m*n) + rank + m + 1;
1397: }
1399: if (ze == P) { /* First assume not corner or edge */
1400: n18 = (m*n) - (size-rank) - m - 1;
1401: n19 = (m*n) - (size-rank) - m;
1402: n20 = (m*n) - (size-rank) - m + 1;
1403: n21 = (m*n) - (size-rank) - 1;
1404: n22 = (m*n) - (size-rank);
1405: n23 = (m*n) - (size-rank) + 1;
1406: n24 = (m*n) - (size-rank) + m - 1;
1407: n25 = (m*n) - (size-rank) + m;
1408: n26 = (m*n) - (size-rank) + m + 1;
1409: }
1411: if ((xs==0) && (zs==0)) { /* Assume an edge, not corner */
1412: n0 = size - m*n + rank + m-1 - m;
1413: n3 = size - m*n + rank + m-1;
1414: n6 = size - m*n + rank + m-1 + m;
1415: }
1416:
1417: if ((xs==0) && (ze==P)) { /* Assume an edge, not corner */
1418: n18 = m*n - (size - rank) + m-1 - m;
1419: n21 = m*n - (size - rank) + m-1;
1420: n24 = m*n - (size - rank) + m-1 + m;
1421: }
1423: if ((xs==0) && (ys==0)) { /* Assume an edge, not corner */
1424: n0 = rank + m*n -1 - m*n;
1425: n9 = rank + m*n -1;
1426: n18 = rank + m*n -1 + m*n;
1427: }
1429: if ((xs==0) && (ye==N)) { /* Assume an edge, not corner */
1430: n6 = rank - m*(n-1) + m-1 - m*n;
1431: n15 = rank - m*(n-1) + m-1;
1432: n24 = rank - m*(n-1) + m-1 + m*n;
1433: }
1435: if ((xe==M*dof) && (zs==0)) { /* Assume an edge, not corner */
1436: n2 = size - (m*n-rank) - (m-1) - m;
1437: n5 = size - (m*n-rank) - (m-1);
1438: n8 = size - (m*n-rank) - (m-1) + m;
1439: }
1441: if ((xe==M*dof) && (ze==P)) { /* Assume an edge, not corner */
1442: n20 = m*n - (size - rank) - (m-1) - m;
1443: n23 = m*n - (size - rank) - (m-1);
1444: n26 = m*n - (size - rank) - (m-1) + m;
1445: }
1447: if ((xe==M*dof) && (ys==0)) { /* Assume an edge, not corner */
1448: n2 = rank + m*(n-1) - (m-1) - m*n;
1449: n11 = rank + m*(n-1) - (m-1);
1450: n20 = rank + m*(n-1) - (m-1) + m*n;
1451: }
1453: if ((xe==M*dof) && (ye==N)) { /* Assume an edge, not corner */
1454: n8 = rank - m*n +1 - m*n;
1455: n17 = rank - m*n +1;
1456: n26 = rank - m*n +1 + m*n;
1457: }
1459: if ((ys==0) && (zs==0)) { /* Assume an edge, not corner */
1460: n0 = size - m + rank -1;
1461: n1 = size - m + rank;
1462: n2 = size - m + rank +1;
1463: }
1465: if ((ys==0) && (ze==P)) { /* Assume an edge, not corner */
1466: n18 = m*n - (size - rank) + m*(n-1) -1;
1467: n19 = m*n - (size - rank) + m*(n-1);
1468: n20 = m*n - (size - rank) + m*(n-1) +1;
1469: }
1471: if ((ye==N) && (zs==0)) { /* Assume an edge, not corner */
1472: n6 = size - (m*n-rank) - m * (n-1) -1;
1473: n7 = size - (m*n-rank) - m * (n-1);
1474: n8 = size - (m*n-rank) - m * (n-1) +1;
1475: }
1477: if ((ye==N) && (ze==P)) { /* Assume an edge, not corner */
1478: n24 = rank - (size-m) -1;
1479: n25 = rank - (size-m);
1480: n26 = rank - (size-m) +1;
1481: }
1483: /* Check for Corners */
1484: if ((xs==0) && (ys==0) && (zs==0)) { n0 = size -1;}
1485: if ((xs==0) && (ys==0) && (ze==P)) { n18 = m*n-1;}
1486: if ((xs==0) && (ye==N) && (zs==0)) { n6 = (size-1)-m*(n-1);}
1487: if ((xs==0) && (ye==N) && (ze==P)) { n24 = m-1;}
1488: if ((xe==M*dof) && (ys==0) && (zs==0)) { n2 = size-m;}
1489: if ((xe==M*dof) && (ys==0) && (ze==P)) { n20 = m*n-m;}
1490: if ((xe==M*dof) && (ye==N) && (zs==0)) { n8 = size-m*n;}
1491: if ((xe==M*dof) && (ye==N) && (ze==P)) { n26 = 0;}
1493: /* Check for when not X,Y, and Z Periodic */
1495: /* If not X periodic */
1496: if (!DAXPeriodic(wrap)){
1497: if (xs==0) {n0 = n3 = n6 = n9 = n12 = n15 = n18 = n21 = n24 = -2;}
1498: if (xe==M*dof) {n2 = n5 = n8 = n11 = n14 = n17 = n20 = n23 = n26 = -2;}
1499: }
1501: /* If not Y periodic */
1502: if (!DAYPeriodic(wrap)){
1503: if (ys==0) {n0 = n1 = n2 = n9 = n10 = n11 = n18 = n19 = n20 = -2;}
1504: if (ye==N) {n6 = n7 = n8 = n15 = n16 = n17 = n24 = n25 = n26 = -2;}
1505: }
1507: /* If not Z periodic */
1508: if (!DAZPeriodic(wrap)){
1509: if (zs==0) {n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = -2;}
1510: if (ze==P) {n18 = n19 = n20 = n21 = n22 = n23 = n24 = n25 = n26 = -2;}
1511: }
1513: nn = 0;
1515: /* Bottom Level */
1516: for (k=0; k<s_z; k++) {
1517: for (i=1; i<=s_y; i++) {
1518: if (n0 >= 0) { /* left below */
1519: x_t = flx[n0 % m]*dof;
1520: y_t = fly[(n0 % (m*n))/m];
1521: z_t = flz[n0 / (m*n)];
1522: s_t = bases[n0] + x_t*y_t*z_t - (s_y-i)*x_t -s_x - (s_z-k-1)*x_t*y_t;
1523: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1524: }
1525: if (n1 >= 0) { /* directly below */
1526: x_t = x;
1527: y_t = fly[(n1 % (m*n))/m];
1528: z_t = flz[n1 / (m*n)];
1529: s_t = bases[n1] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1530: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1531: }
1532: if (n2 >= 0) { /* right below */
1533: x_t = flx[n2 % m]*dof;
1534: y_t = fly[(n2 % (m*n))/m];
1535: z_t = flz[n2 / (m*n)];
1536: s_t = bases[n2] + x_t*y_t*z_t - (s_y+1-i)*x_t - (s_z-k-1)*x_t*y_t;
1537: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1538: }
1539: }
1541: for (i=0; i<y; i++) {
1542: if (n3 >= 0) { /* directly left */
1543: x_t = flx[n3 % m]*dof;
1544: y_t = y;
1545: z_t = flz[n3 / (m*n)];
1546: s_t = bases[n3] + (i+1)*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1547: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1548: }
1550: if (n4 >= 0) { /* middle */
1551: x_t = x;
1552: y_t = y;
1553: z_t = flz[n4 / (m*n)];
1554: s_t = bases[n4] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1555: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1556: }
1558: if (n5 >= 0) { /* directly right */
1559: x_t = flx[n5 % m]*dof;
1560: y_t = y;
1561: z_t = flz[n5 / (m*n)];
1562: s_t = bases[n5] + i*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1563: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1564: }
1565: }
1567: for (i=1; i<=s_y; i++) {
1568: if (n6 >= 0) { /* left above */
1569: x_t = flx[n6 % m]*dof;
1570: y_t = fly[(n6 % (m*n))/m];
1571: z_t = flz[n6 / (m*n)];
1572: s_t = bases[n6] + i*x_t - s_x + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1573: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1574: }
1575: if (n7 >= 0) { /* directly above */
1576: x_t = x;
1577: y_t = fly[(n7 % (m*n))/m];
1578: z_t = flz[n7 / (m*n)];
1579: s_t = bases[n7] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1580: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1581: }
1582: if (n8 >= 0) { /* right above */
1583: x_t = flx[n8 % m]*dof;
1584: y_t = fly[(n8 % (m*n))/m];
1585: z_t = flz[n8 / (m*n)];
1586: s_t = bases[n8] + (i-1)*x_t + x_t*y_t*z_t - (s_z-k)*x_t*y_t;
1587: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1588: }
1589: }
1590: }
1592: /* Middle Level */
1593: for (k=0; k<z; k++) {
1594: for (i=1; i<=s_y; i++) {
1595: if (n9 >= 0) { /* left below */
1596: x_t = flx[n9 % m]*dof;
1597: y_t = fly[(n9 % (m*n))/m];
1598: /* z_t = z; */
1599: s_t = bases[n9] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1600: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1601: }
1602: if (n10 >= 0) { /* directly below */
1603: x_t = x;
1604: y_t = fly[(n10 % (m*n))/m];
1605: /* z_t = z; */
1606: s_t = bases[n10] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1607: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1608: }
1609: if (n11 >= 0) { /* right below */
1610: x_t = flx[n11 % m]*dof;
1611: y_t = fly[(n11 % (m*n))/m];
1612: /* z_t = z; */
1613: s_t = bases[n11] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1614: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1615: }
1616: }
1618: for (i=0; i<y; i++) {
1619: if (n12 >= 0) { /* directly left */
1620: x_t = flx[n12 % m]*dof;
1621: y_t = y;
1622: /* z_t = z; */
1623: s_t = bases[n12] + (i+1)*x_t - s_x + k*x_t*y_t;
1624: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1625: }
1627: /* Interior */
1628: s_t = bases[rank] + i*x + k*x*y;
1629: for (j=0; j<x; j++) { idx[nn++] = s_t++;}
1631: if (n14 >= 0) { /* directly right */
1632: x_t = flx[n14 % m]*dof;
1633: y_t = y;
1634: /* z_t = z; */
1635: s_t = bases[n14] + i*x_t + k*x_t*y_t;
1636: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1637: }
1638: }
1640: for (i=1; i<=s_y; i++) {
1641: if (n15 >= 0) { /* left above */
1642: x_t = flx[n15 % m]*dof;
1643: y_t = fly[(n15 % (m*n))/m];
1644: /* z_t = z; */
1645: s_t = bases[n15] + i*x_t - s_x + k*x_t*y_t;
1646: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1647: }
1648: if (n16 >= 0) { /* directly above */
1649: x_t = x;
1650: y_t = fly[(n16 % (m*n))/m];
1651: /* z_t = z; */
1652: s_t = bases[n16] + (i-1)*x_t + k*x_t*y_t;
1653: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1654: }
1655: if (n17 >= 0) { /* right above */
1656: x_t = flx[n17 % m]*dof;
1657: y_t = fly[(n17 % (m*n))/m];
1658: /* z_t = z; */
1659: s_t = bases[n17] + (i-1)*x_t + k*x_t*y_t;
1660: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1661: }
1662: }
1663: }
1664:
1665: /* Upper Level */
1666: for (k=0; k<s_z; k++) {
1667: for (i=1; i<=s_y; i++) {
1668: if (n18 >= 0) { /* left below */
1669: x_t = flx[n18 % m]*dof;
1670: y_t = fly[(n18 % (m*n))/m];
1671: /* z_t = flz[n18 / (m*n)]; */
1672: s_t = bases[n18] - (s_y-i)*x_t -s_x + (k+1)*x_t*y_t;
1673: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1674: }
1675: if (n19 >= 0) { /* directly below */
1676: x_t = x;
1677: y_t = fly[(n19 % (m*n))/m];
1678: /* z_t = flz[n19 / (m*n)]; */
1679: s_t = bases[n19] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1680: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1681: }
1682: if (n20 >= 0) { /* right belodof */
1683: x_t = flx[n20 % m]*dof;
1684: y_t = fly[(n20 % (m*n))/m];
1685: /* z_t = flz[n20 / (m*n)]; */
1686: s_t = bases[n20] - (s_y+1-i)*x_t + (k+1)*x_t*y_t;
1687: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1688: }
1689: }
1691: for (i=0; i<y; i++) {
1692: if (n21 >= 0) { /* directly left */
1693: x_t = flx[n21 % m]*dof;
1694: y_t = y;
1695: /* z_t = flz[n21 / (m*n)]; */
1696: s_t = bases[n21] + (i+1)*x_t - s_x + k*x_t*y_t;
1697: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1698: }
1700: if (n22 >= 0) { /* middle */
1701: x_t = x;
1702: y_t = y;
1703: /* z_t = flz[n22 / (m*n)]; */
1704: s_t = bases[n22] + i*x_t + k*x_t*y_t;
1705: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1706: }
1708: if (n23 >= 0) { /* directly right */
1709: x_t = flx[n23 % m]*dof;
1710: y_t = y;
1711: /* z_t = flz[n23 / (m*n)]; */
1712: s_t = bases[n23] + i*x_t + k*x_t*y_t;
1713: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1714: }
1715: }
1717: for (i=1; i<=s_y; i++) {
1718: if (n24 >= 0) { /* left above */
1719: x_t = flx[n24 % m]*dof;
1720: y_t = fly[(n24 % (m*n))/m];
1721: /* z_t = flz[n24 / (m*n)]; */
1722: s_t = bases[n24] + i*x_t - s_x + k*x_t*y_t;
1723: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1724: }
1725: if (n25 >= 0) { /* directly above */
1726: x_t = x;
1727: y_t = fly[(n25 % (m*n))/m];
1728: /* z_t = flz[n25 / (m*n)]; */
1729: s_t = bases[n25] + (i-1)*x_t + k*x_t*y_t;
1730: for (j=0; j<x_t; j++) { idx[nn++] = s_t++;}
1731: }
1732: if (n26 >= 0) { /* right above */
1733: x_t = flx[n26 % m]*dof;
1734: y_t = fly[(n26 % (m*n))/m];
1735: /* z_t = flz[n26 / (m*n)]; */
1736: s_t = bases[n26] + (i-1)*x_t + k*x_t*y_t;
1737: for (j=0; j<s_x; j++) { idx[nn++] = s_t++;}
1738: }
1739: }
1740: }
1741: PetscFree(bases);
1742: da->gtol = gtol;
1743: da->ltog = ltog;
1744: da->idx = idx;
1745: da->Nl = nn;
1746: da->base = base;
1747: da->ops->view = DAView_3d;
1748: da->wrap = wrap;
1749: *inra = da;
1751: /*
1752: Set the local to global ordering in the global vector, this allows use
1753: of VecSetValuesLocal().
1754: */
1755: ISLocalToGlobalMappingCreateNC(comm,nn,idx,&da->ltogmap);
1756: ISLocalToGlobalMappingBlock(da->ltogmap,da->w,&da->ltogmapb);
1757: PetscLogObjectParent(da,da->ltogmap);
1759: da->ltol = PETSC_NULL;
1760: da->ao = PETSC_NULL;
1762: da->lx = flx;
1763: da->ly = fly;
1764: da->lz = flz;
1766: DAView_Private(da);
1767: return(0);
1768: }
1772: /*@C
1773: DACreate - Creates an object that will manage the communication of regular array data that is distributed across some processors
1774: in 1, 2 or 3 dimensions
1776: Collective on MPI_Comm
1778: See the manual pages for the routines for each dimension.
1780: Level: beginner
1782:
1783: .keywords: distributed array, create, three-dimensional
1785: .seealso: DACreate1d(), DACreate2d(), DACreate3d(), DAGetOwnershipRanges()
1787: @*/
1788: PetscErrorCode DACreate(MPI_Comm comm,PetscInt dim,DAPeriodicType wrap,DAStencilType stencil_type,PetscInt M,
1789: PetscInt N,PetscInt P,PetscInt m,PetscInt n,PetscInt p,PetscInt dof,PetscInt s,const PetscInt lx[],const PetscInt ly[],const PetscInt lz[],DA *inra)
1790: {
1793: if (dim == 3) {
1794: DACreate3d(comm,wrap,stencil_type,M,N,P,m,n,p,dof,s,lx,ly,lz,inra);
1795: } else if (dim == 2) {
1796: DACreate2d(comm,wrap,stencil_type,M,N,m,n,dof,s,lx,ly,inra);
1797: } else if (dim == 1) {
1798: DACreate1d(comm,wrap,M,dof,s,lx,inra);
1799: }
1800: return(0);
1801: }