Actual source code: openmp.c
1: #define PETSCKSP_DLL
3: #include private/pcimpl.h
4: #include petscksp.h
6: typedef struct {
7: MatStructure flag; /* pc->flag */
8: PetscInt setupcalled; /* pc->setupcalled */
9: PetscInt n;
10: MPI_Comm comm; /* local world used by this preconditioner */
11: KSP ksp; /* actual solver used across local world */
12: Mat mat; /* matrix in local world */
13: Mat gmat; /* matrix known only to process 0 in the local world */
14: Vec x,y,xdummy,ydummy;
15: VecScatter scatter;
16: PetscTruth nonzero_guess;
17: } PC_OpenMP;
22: /*
23: Would like to have this simply call PCView() on the inner PC. The problem is
24: that the outter comm does not live on the inside so cannot do this. Instead
25: handle the special case when the viewer is stdout, construct a new one just
26: for this call.
27: */
29: static PetscErrorCode PCView_OpenMP_MP(MPI_Comm comm,void *ctx)
30: {
31: PC_OpenMP *red = (PC_OpenMP*)ctx;
33: PetscViewer viewer;
36: PetscViewerASCIIGetStdout(comm,&viewer);
37: PetscViewerASCIIPushTab(viewer); /* this is bogus in general */
38: KSPView(red->ksp,viewer);
39: PetscViewerASCIIPopTab(viewer);
40: return(0);
41: }
45: static PetscErrorCode PCView_OpenMP(PC pc,PetscViewer viewer)
46: {
47: PC_OpenMP *red = (PC_OpenMP*)pc->data;
48: PetscMPIInt size;
50: PetscTruth iascii;
55: MPI_Comm_size(red->comm,&size);
56: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
57: if (iascii) {
58: PetscViewerASCIIPrintf(viewer," Size of solver nodes %d\n",size);
59: PetscViewerASCIIPrintf(viewer," Parallel sub-solver given next\n",size);
60: /* should only make the next call if the viewer is associated with stdout */
61: PetscOpenMPRun(red->comm,PCView_OpenMP_MP,red);
62: }
63: return(0);
64: }
66: #include private/matimpl.h
67: #include private/vecimpl.h
68: #include ../src/mat/impls/aij/mpi/mpiaij.h
69: #include ../src/mat/impls/aij/seq/aij.h
76: static PetscErrorCode PCApply_OpenMP_1(PC pc,Vec x,Vec y)
77: {
78: PC_OpenMP *red = (PC_OpenMP*)pc->data;
82: KSPSetInitialGuessNonzero(red->ksp,pc->nonzero_guess);
83: KSPSolve(red->ksp,x,y);
84: return(0);
85: }
89: static PetscErrorCode PCSetUp_OpenMP_MP(MPI_Comm comm,void *ctx)
90: {
91: PC_OpenMP *red = (PC_OpenMP*)ctx;
93: PetscInt m;
94: MatReuse scal;
95: PetscMPIInt rank;
98: red->comm = comm;
99: MPI_Bcast(&red->setupcalled,1,MPIU_INT,0,comm);
100: MPI_Bcast(&red->flag,1,MPIU_INT,0,comm);
101: if (!red->setupcalled) {
102: /* setup vector communication */
103: MPI_Bcast(&red->n,1,MPIU_INT,0,comm);
104: VecCreateMPI(comm,PETSC_DECIDE,red->n,&red->x);
105: VecCreateMPI(comm,PETSC_DECIDE,red->n,&red->y);
106: VecScatterCreateToZero(red->x,&red->scatter,&red->xdummy);
107: VecDuplicate(red->xdummy,&red->ydummy);
108: MPI_Comm_rank(comm,&rank);
109: if (!rank) {
110: VecDestroy(red->xdummy);
111: VecDestroy(red->ydummy);
112: }
113: scal = MAT_INITIAL_MATRIX;
114: } else {
115: if (red->flag == DIFFERENT_NONZERO_PATTERN) {
116: MatDestroy(red->mat);
117: scal = MAT_INITIAL_MATRIX;
118: CHKMEMQ;
119: } else {
120: scal = MAT_REUSE_MATRIX;
121: }
122: }
124: /* copy matrix out onto processes */
125: VecGetLocalSize(red->x,&m);
126: MatDistribute_MPIAIJ(comm,red->gmat,m,scal,&red->mat);
127: if (!red->setupcalled) {
128: /* create the solver */
129: KSPCreate(comm,&red->ksp);
130: /* would like to set proper tablevel for KSP, but do not have direct access to parent pc */
131: KSPSetOptionsPrefix(red->ksp,"openmp_"); /* should actually append with global pc prefix */
132: KSPSetOperators(red->ksp,red->mat,red->mat,red->flag);
133: KSPSetFromOptions(red->ksp);
134: } else {
135: KSPSetOperators(red->ksp,red->mat,red->mat,red->flag);
136: }
137: return(0);
138: }
142: static PetscErrorCode PCSetUp_OpenMP(PC pc)
143: {
144: PC_OpenMP *red = (PC_OpenMP*)pc->data;
146: PetscMPIInt size;
149: red->gmat = pc->mat;
150: red->flag = pc->flag;
151: red->setupcalled = pc->setupcalled;
153: MPI_Comm_size(red->comm,&size);
154: if (size == 1) { /* special case where copy of matrix is not needed */
155: if (!red->setupcalled) {
156: /* create the solver */
157: KSPCreate(((PetscObject)pc)->comm,&red->ksp);
158: PetscObjectIncrementTabLevel((PetscObject)red->ksp,(PetscObject)pc,1);
159: KSPSetOptionsPrefix(red->ksp,"openmp_"); /* should actually append with global pc prefix */
160: KSPSetOperators(red->ksp,red->gmat,red->gmat,red->flag);
161: KSPSetFromOptions(red->ksp);
162: } else {
163: KSPSetOperators(red->ksp,red->gmat,red->gmat,red->flag);
164: }
165: pc->ops->apply = PCApply_OpenMP_1;
166: return(0);
167: } else {
168: MatGetSize(pc->mat,&red->n,PETSC_IGNORE);
169: PetscOpenMPRun(red->comm,PCSetUp_OpenMP_MP,red);
170: }
171: return(0);
172: }
176: static PetscErrorCode PCApply_OpenMP_MP(MPI_Comm comm,void *ctx)
177: {
178: PC_OpenMP *red = (PC_OpenMP*)ctx;
182: VecScatterBegin(red->scatter,red->xdummy,red->x,INSERT_VALUES,SCATTER_REVERSE);
183: VecScatterEnd(red->scatter,red->xdummy,red->x,INSERT_VALUES,SCATTER_REVERSE);
184: MPI_Bcast(&red->nonzero_guess,1,MPIU_INT,0,red->comm);
185: if (red->nonzero_guess) {
186: VecScatterBegin(red->scatter,red->ydummy,red->y,INSERT_VALUES,SCATTER_REVERSE);
187: VecScatterEnd(red->scatter,red->ydummy,red->y,INSERT_VALUES,SCATTER_REVERSE);
188: }
189: KSPSetInitialGuessNonzero(red->ksp,red->nonzero_guess);
191: KSPSolve(red->ksp,red->x,red->y);
193: VecScatterBegin(red->scatter,red->y,red->ydummy,INSERT_VALUES,SCATTER_FORWARD);
194: VecScatterEnd(red->scatter,red->y,red->ydummy,INSERT_VALUES,SCATTER_FORWARD);
195: return(0);
196: }
200: static PetscErrorCode PCApply_OpenMP(PC pc,Vec x,Vec y)
201: {
202: PC_OpenMP *red = (PC_OpenMP*)pc->data;
206: red->xdummy = x;
207: red->ydummy = y;
208: red->nonzero_guess = pc->nonzero_guess;
209: PetscOpenMPRun(red->comm,PCApply_OpenMP_MP,red);
210: return(0);
211: }
215: static PetscErrorCode PCDestroy_OpenMP_MP(MPI_Comm comm,void *ctx)
216: {
217: PC_OpenMP *red = (PC_OpenMP*)ctx;
218: PetscMPIInt rank;
222: if (red->scatter) {VecScatterDestroy(red->scatter);}
223: if (red->x) {VecDestroy(red->x);}
224: if (red->y) {VecDestroy(red->y);}
225: if (red->ksp) {KSPDestroy(red->ksp);}
226: if (red->mat) {MatDestroy(red->mat);}
227: MPI_Comm_rank(comm,&rank);
228: if (rank) {
229: if (red->xdummy) {VecDestroy(red->xdummy);}
230: if (red->ydummy) {VecDestroy(red->ydummy);}
231: }
232: return(0);
233: }
237: static PetscErrorCode PCDestroy_OpenMP(PC pc)
238: {
239: PC_OpenMP *red = (PC_OpenMP*)pc->data;
243: PetscOpenMPRun(red->comm,PCDestroy_OpenMP_MP,red);
244: PetscOpenMPFree(red->comm,red);
245: return(0);
246: }
250: static PetscErrorCode PCSetFromOptions_OpenMP(PC pc)
251: {
253: return(0);
254: }
257: /* -------------------------------------------------------------------------------------*/
258: /*MC
259: PCOPENMP - Runs a preconditioner for a single process matrix across several MPI processes
261: $ This will usually be run with -pc_type openmp -ksp_type preonly
262: $ solver options are set with -openmp_ksp_... and -openmp_pc_... for example
263: $ -openmp_ksp_type cg would use cg as the Krylov method or -openmp_ksp_monitor or
264: $ -openmp_pc_type hypre -openmp_pc_hypre_type boomeramg
266: Always run with -ksp_view (or -snes_view) to see what solver is actually being used.
268: Currently the solver options INSIDE the OpenMP preconditioner can ONLY be set via the
269: options database.
271: Level: intermediate
273: See PetscOpenMPMerge() and PetscOpenMPSpawn() for two ways to start up MPI for use with this preconditioner
275: .seealso: PCCreate(), PCSetType(), PCType (for list of available types)
277: M*/
283: PetscErrorCode PCCreate_OpenMP(PC pc)
284: {
286: PC_OpenMP *red;
287: PetscMPIInt size;
290: MPI_Comm_size(((PetscObject)pc)->comm,&size);
291: if (size > 1) SETERRQ(PETSC_ERR_ARG_SIZ,"OpenMP preconditioner only works for sequential solves");
292: /* caste the struct length to a PetscInt for easier MPI calls */
294: PetscOpenMPNew(PETSC_COMM_LOCAL_WORLD,(PetscInt)sizeof(PC_OpenMP),(void**)&red);
295: red->comm = PETSC_COMM_LOCAL_WORLD;
296: pc->data = (void*) red;
298: pc->ops->apply = PCApply_OpenMP;
299: pc->ops->destroy = PCDestroy_OpenMP;
300: pc->ops->setfromoptions = PCSetFromOptions_OpenMP;
301: pc->ops->setup = PCSetUp_OpenMP;
302: pc->ops->view = PCView_OpenMP;
303: return(0);
304: }