Actual source code: samgmgpetsc.c
1: #define PETSCKSP_DLL
3: #include global.h
4: #include petscfunc.h
5: #include petscksp.h
6: #include petscmg.h
8: static char help[] = "Does PETSc multigrid cycling using hierarchy build by SAMG\n\n";
10: PetscErrorCode samgmgpetsc(const int numnodes, double* Asky, int* ia,
11: int* ja, double* rhs, double* u_approx,
12: const OPTIONS *options)
13: {
14: /*..Petsc variables..*/
15: Vec x, b; /* approx solution and RHS */
16: Mat A; /* linear system matrix */
17: KSP ksp; /* linear solver context */
18: PC pc; /* preconditioner context */
19: PCType pctype; /* preconditioning technique */
20: KSP ksp; /* KSP context */
22: int its; /* Error messages and number of iterations */
23: /*..Other variables for the PETSc interface..*/
24: int *nnz_per_row; /* integer vector to hold the number of nonzeros */
25: /* of each row. This vector will be used to */
26: /* allocate memory for the matrix, and to store */
27: /* elements in the matrix */
28: int *cols; /* cols is a vector of collumn indices used in */
29: /* assembling the PETSc rhs vector */
30: PetscScalar *sol_array;/* sol_array used to pass the PETSc solution */
31: /* back to the calling program */
32: /*..Variables used to customize the convergence criterium to */
33: /* ||res|| / ||b|| < tol */
34: double bnrm2;
35: CONVHIST *convhist;
36: /*..Context for the SAMG preconditioner..*/
37: SamgShellPC *samg_ctx;
38: /*..Variables to extract SAMG hierarchy..*/
39: int k, levels, numnonzero;
40: double normdiff;
41: GridCtx grid[MAX_LEVELS];
42: char pathfilename[80], basefilename[80];
43: /*..Variables for intermediate levels..*/
44: KSP ksp_pre, ksp_post;
45: PC pc_pre, pc_post;
46: Mat FineLevelMatrix;
47: int petsc_level, size;
48: /*..Variables for coarse grid solve..*/
49: KSP coarsegridksp;
50: PC coarsegridpc;
51: KSP coarsegridksp;
52: int coarsegrid_n;
53: double coarsegrid_rnorm;
54: /*..Variables that determine behaviour of the code..*/
55: static PetscErrorCode debug = *(options->DEBUG);
56: /*..Other variables..*/
57: int I;
58: PetscTruth flg, issamg, issamg_print;
59: /*..Variables for CPU timings..*/
60: PetscLogDouble v1,v2,t_setup, t_solve;
62: /*..Executable statements..*/
63: PetscInitialize( (int*) 0, (char ***) 0,(char *) 0, help);
65: /*..Get start time of linear system setup..*/
66: PetscGetTime(&v1);
68: PetscMalloc(numnodes * sizeof(int),&nnz_per_row);
70: /*..The numbero f nonzeros entries in row I can be calculated as
71: ia[I+1] - 1 - ia[I] + 1 = ia[I+1] - ia[I] ..*/
72: for (I=0;I<numnodes;I++)
73: nnz_per_row[I] = ia[I+1] - ia[I];
75: /*..Allocate (create) SeqAIJ matrix for use within PETSc..*/
76: MatCreate(PETSC_COMM_WORLD,&A);
77: MatSetSizes(A,numnodes,numnodes,numnodes,numnodes);
78: MatSetType(A,MATSEQAIJ);
79: MatSeqAIJSetPreallocation(A,0,nnz_per_row);
81: /*..Assemble matrix for use within PETSc..*/
82: for (I=0;I<numnodes;I++){
83: MatSetValues(A,
84: 1, /* number of rows */
85: &I, /* pointer to global row number */
86: nnz_per_row[I], /* number of collums = number of nonzero ... */
87: /* entries in row I */
88: &(ja[ ia[I] ]),
89: /* vector global column indices */
90: (PetscScalar *) &(Asky[ ia[I] ]),
91: /* vector of coefficients */
92: INSERT_VALUES);
93: }
95: MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
96: MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
97: if (debug)
98: MatView(A,PETSC_VIEWER_STDOUT_SELF);
100: /*..Create solution and rhs vector. Note that we form vector from
101: scratch and then duplicate as needed..*/
102: VecCreate(PETSC_COMM_WORLD,&x);
103: VecSetSizes(x,PETSC_DECIDE,numnodes);
104: VecSetType(x,VECSEQ);
105: VecDuplicate(x,&b);
107: PetscMalloc(numnodes * sizeof(int),&cols);
108: for (I=0;I<numnodes;I++)
109: cols[I] = I;
111: /*..Assemble the right-hand side vector for use within PETSc..*/
112: VecSetValues(b,numnodes,cols,(PetscScalar*)rhs,INSERT_VALUES);
113:
114: VecAssemblyBegin(b);
115: VecAssemblyEnd(b);
116: if (debug){
117: printf("[PETSc]:The right-hand side \n");
118: VecView(b,PETSC_VIEWER_STDOUT_SELF);
119: printf("\n");
120: }
121: VecNorm(b,NORM_2,&bnrm2);
123: /*..Assemble the start solution vector for use within PETSc..*/
124: VecSetValues(x,numnodes,cols,(PetscScalar*)u_approx,INSERT_VALUES);
125:
126: VecAssemblyBegin(x);
127: VecAssemblyEnd(x);
129: VecNorm(b,NORM_2,&bnrm2);
130: if (debug)
131: printf("[PETSc]:The right-hand side norm = %e \n",bnrm2);
133: /*..Create linear solver context..*/
134: KSPCreate(MPI_COMM_WORLD,&ksp);
136: /*..Set operators. Here the matrix that defines the linear system
137: also serves as the preconditioning matrix..*/
138: KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);
140: /*..Extract pc type from context..*/
141: KSPGetPC(ksp,&pc);
143: /*..Customize tolerances..*/
144: KSPSetTolerances(ksp,1e-12,1e-14,PETSC_DEFAULT,
145: PETSC_DEFAULT);
147: /*..Create user defined context for the shell preconditioner..*/
148: SamgShellPCCreate(&samg_ctx);
150: /*..Do the setup for the SAMG precondioner..*/
151: SamgShellPCSetUp(samg_ctx,A);
153: /*..Give user defined preconditioner a name..*/
154: PCShellSetName(pc,"SAMG (Scalar mode)");
155:
157: /*..Parse SAMG hierarchy to PETSc variables..*/
158: levels = samg_ctx->LEVELS;
159: numnonzero = ia[numnodes];
160: SamgGetGrid(levels, numnodes, numnonzero, grid, PETSC_NULL);
162: /*..Print coarser grid and interpolation operators to file..*/
163: PetscOptionsHasName(PETSC_NULL,"-samg_print",&issamg_print);
164:
165: if (issamg_print){
166: for (k=2;k<=levels;k++){
167: sprintf(pathfilename,"./");
168: sprintf(basefilename,"Pcoarsemat.%02u",k);
169: PrintMatrix(grid[k].A, pathfilename, basefilename);
170: }
171: for (k=1;k<=levels-1;k++){
172: sprintf(basefilename,"Pinterpol.%02u%02",k, k-1);
173: PrintMatrix(grid[k].Interp, pathfilename, basefilename);
174: }
175: }
176:
177: /*..Perform check on parsing..*/
178: PetscOptionsHasName(PETSC_NULL,"-samg_check",&issamg_print);
179:
180: if (issamg_print)
181: SamgCheckGalerkin(levels, A, grid, PETSC_NULL);
182:
183: /*..Set KSP solver type..*/
184: KSPSetType(ksp,KSPRICHARDSON);
185: KSPMonitorSet(ksp,KSPMonitorDefault,PETSC_NULL, PETSC_NULL);
186:
187: /*..Set MG preconditioner..*/
188: PCSetType(pc,PCMG);
189: PCMGSetLevels(pc,levels, PETSC_NULL);
190: PCMGSetType(pc, PC_MG_MULTIPLICATIVE);
191: PCMGSetCycles(pc, 1);
192: PCMGSetNumberSmoothUp(pc,1);
193: PCMGSetNumberSmoothDown(pc,1);
195: /*....Set smoother, work vectors and residual calculation on each
196: level....*/
197: for (k=1;k<=levels;k++){
198: petsc_level = levels - k;
199: /*....Get pre-smoothing KSP context....*/
200: PCMGGetSmootherDown(pc,petsc_level,&grid[k].ksp_pre);
201:
202: PCMGGetSmootherUp(pc,petsc_level,&grid[k].ksp_post);
203:
204: if (k==1)
205: FineLevelMatrix = A;
206: else
207: FineLevelMatrix = grid[k].A;
208: MatGetSize(FineLevelMatrix, &size, &size);
209: VecCreate(MPI_COMM_WORLD,&grid[k].x);
210: VecSetSizes(grid[k].x,PETSC_DECIDE,size);
211: VecSetType(grid[k].x,VECSEQ);
212: VecDuplicate(grid[k].x,&grid[k].b);
213: VecDuplicate(grid[k].x,&grid[k].r);
215: /*....set ksp_pre context....*/
216: KSPSetOperators(grid[k].ksp_pre, FineLevelMatrix,
217: FineLevelMatrix, DIFFERENT_NONZERO_PATTERN);
218:
219: KSPGetPC(ksp_pre_pre,&pc_pre);
220: KSPSetType(grid[k].ksp_pre, KSPRICHARDSON);
221: KSPSetTolerances(grid[k].ksp_pre, 1e-12, 1e-50, 1e7,1);
222:
223: PCSetType(pc_pre, PCSOR);
224: PCSORSetSymmetric(pc_pre,SOR_FORWARD_SWEEP);
226: /*....set ksp_post context....*/
227: KSPSetOperators(grid[k].ksp_post, FineLevelMatrix,
228: FineLevelMatrix, DIFFERENT_NONZERO_PATTERN);
229:
230: KSPGetPC(grid[k].ksp_post,&pc_post);
231: KSPSetInitialGuessNonzero(grid[k].ksp_post, PETSC_TRUE);
232: KSPSetType(grid[k].ksp_post, KSPRICHARDSON);
233: KSPSetTolerances(grid[k].ksp_post, 1e-12, 1e-50, 1e7,1);
234:
235: PCSetType(pc_post, PCSOR);
236: PCSORSetSymmetric(pc_post,SOR_BACKWARD_SWEEP);
238: PCMGSetX(pc,petsc_level,grid[k].x);
239: PCMGSetRhs(pc,petsc_level,grid[k].b);
240: PCMGSetR(pc,petsc_level,grid[k].r);
241: PCMGSetResidual(pc,petsc_level,PCMGDefaultResidual,FineLevelMatrix);
242:
243: }
245: /*....Create interpolation between the levels....*/
246: for (k=1;k<=levels-1;k++){
247: petsc_level = levels - k;
248: PCMGSetInterpolation(pc,petsc_level,grid[k].Interp);
249: PCMGSetRestriction(pc,petsc_level,grid[k].Interp);
250: }
252: /*....Set coarse grid solver....*/
253: PCMGGetCoarseSolve(pc,&coarsegridksp);
254: KSPSetFromOptions(coarsegridksp);
255: KSPSetOperators(coarsegridksp, grid[levels].A, grid[levels].A,
256: DIFFERENT_NONZERO_PATTERN);
257: KSPGetPC(coarsegridksp,&coarsegridpc);
258: KSPSetType(coarsegridksp, KSPPREONLY);
259: PCSetType(coarsegridpc, PCLU);
261: /*..Allow above criterea to be overwritten..*/
262: KSPSetFromOptions(ksp);
264: /*..Indicate that we are going to use a non-zero initial solution..*/
265: KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
267: /*..Get end time of linear system setup..*/
268: PetscGetTime(&v2);
269: t_setup = v2 - v1;
271: /*..Get start time of linear solve..*/
272: PetscGetTime(&v1);
274: /*..Solve linear system..*/
275: KSPSolve(ksp,b,x);
276: KSPGetIterationNumber(ksp,&its);
277:
278: /*..Print number of iterations..*/
279: PetscPrintf(PETSC_COMM_WORLD,"\n** Number of iterations done = %d \n",
280: its);
282: /*..Get end time of linear solve..*/
283: PetscGetTime(&v2);
284: t_solve = v2 - v1;
286: printf("\n[PETSc]:Time spend in setup = %e \n",t_setup);
287: printf("[PETSc]:Time spend in solve = %e \n",t_solve);
288: printf("[PETSc]:Total time = %e \n\n", t_setup + t_solve);
289:
290: /*..Copy PETSc solution back..*/
291: ierr= VecGetArray(x, &sol_array);
292: for (I=0;I<numnodes;I++){
293: u_approx[I] = sol_array[I];
294: }
295: VecRestoreArray(x,&sol_array);
297: if (debug){
298: printf("[PETSc]:The solution \n");
299: VecView(x,PETSC_VIEWER_STDOUT_SELF);
300: printf("\n");
301: }
303: /*..Free work space..*/
304: SamgShellPCDestroy(samg_ctx);
305: VecDestroy(x);
306: VecDestroy(b);
307: MatDestroy(A);
308: KSPDestroy(ksp);
309: for (k=2;k<=levels;k++){
310: MatDestroy(grid[k].A);
311: }
312: for (k=1;k<=levels-1;k++){
313: MatDestroy(grid[k].Interp);
314: }
315: for (k=1;k<=levels-1;k++){
316: VecDestroy(grid[k].b);
317: VecDestroy(grid[k].x);
318: VecDestroy(grid[k].r);
319: }
321: PetscFinalize();
322:
323: return 0;
324: }