Actual source code: meshpcice.c

  1: #include <petscmesh_formats.hh>   /*I      "petscmesh.h"   I*/

  5: /*@C
  6:   PCICERenumberBoundary - Change global element names into offsets

  8:   Collective on Mesh

 10:   Input Parameters:
 11: . mesh - the mesh

 13:   Level: advanced

 15:   .seealso: MeshCreate()
 16: @*/
 17: PetscErrorCode  PCICERenumberBoundary(Mesh mesh)
 18: {
 19:   ALE::Obj<PETSC_MESH_TYPE> m;
 20:   PetscErrorCode      ierr;

 23:   MeshGetMesh(mesh, m);
 24:   try {
 25:     ALE::PCICE::fuseBoundary(m);
 26:   } catch(ALE::Exception e) {
 27:     SETERRQ(100, e.msg().c_str());
 28:   }
 29:   return(0);
 30: }

 34: /*@C
 35:   BCSectionGetArray - Returns the array underlying the BCSection.

 37:   Not Collective

 39:   Input Parameters:
 40: + mesh - The Mesh object
 41: - name - The section name

 43:   Output Parameters:
 44: + numElements - The number of mesh element with values
 45: . fiberDim - The number of values per element
 46: - array - The array

 48:   Level: intermediate

 50: .keywords: mesh, elements
 51: .seealso: MeshCreate()
 52: @*/
 53: PetscErrorCode BCSectionGetArray(Mesh mesh, const char name[], PetscInt *numElements, PetscInt *fiberDim, PetscInt *array[])
 54: {
 55:   ALE::Obj<PETSC_MESH_TYPE> m;
 56:   PetscErrorCode      ierr;

 59:   MeshGetMesh(mesh, m);
 60:   const ALE::Obj<PETSC_MESH_TYPE::int_section_type>& section = m->getIntSection(std::string(name));
 61:   if (!section->size()) {
 62:     *numElements = 0;
 63:     *fiberDim    = 0;
 64:     *array       = NULL;
 65:     return(0);
 66:   }
 67:   const PETSC_MESH_TYPE::int_section_type::chart_type& chart = section->getChart();
 68:   int fiberDimMin = section->getFiberDimension(*chart.begin());
 69:   int numElem     = 0;

 71:   for(PETSC_MESH_TYPE::int_section_type::chart_type::const_iterator c_iter = chart.begin(); c_iter != chart.end(); ++c_iter) {
 72:     const int fiberDim = section->getFiberDimension(*c_iter);

 74:     if (fiberDim < fiberDimMin) fiberDimMin = fiberDim;
 75:   }
 76:   for(PETSC_MESH_TYPE::int_section_type::chart_type::const_iterator c_iter = chart.begin(); c_iter != chart.end(); ++c_iter) {
 77:     const int fiberDim = section->getFiberDimension(*c_iter);

 79:     numElem += fiberDim/fiberDimMin;
 80:   }
 81:   *numElements = numElem;
 82:   *fiberDim    = fiberDimMin;
 83:   *array       = (PetscInt *) section->restrictSpace();
 84:   return(0);
 85: }

 89: /*@C
 90:   BCSectionRealCreate - Creates a BCSection.

 92:   Not Collective

 94:   Input Parameters:
 95: + mesh - The Mesh object
 96: . name - The section name
 97: - fiberDim - The number of values per element

 99:   Level: intermediate

101: .keywords: mesh, elements
102: .seealso: MeshCreate()
103: @*/
104: PetscErrorCode BCSectionRealCreate(Mesh mesh, const char name[], PetscInt fiberDim)
105: {
106:   ALE::Obj<PETSC_MESH_TYPE> m;
107:   PetscErrorCode      ierr;

110:   MeshGetMesh(mesh, m);
111:   const ALE::Obj<PETSC_MESH_TYPE::real_section_type>&  section = m->getRealSection(std::string(name));
112:   const ALE::Obj<PETSC_MESH_TYPE::int_section_type>&   ibc     = m->getIntSection("IBC");
113:   const PETSC_MESH_TYPE::int_section_type::chart_type& chart   = ibc->getChart();

115:   for(PETSC_MESH_TYPE::int_section_type::chart_type::const_iterator p_iter = chart.begin(); p_iter != chart.end(); ++p_iter) {
116:     section->setFiberDimension(*p_iter, ibc->getFiberDimension(*p_iter));
117:   }
118:   m->allocate(section);
119:   return(0);
120: }

124: /*@C
125:   BCSectionRealGetArray - Returns the array underlying the BCSection.

127:   Not Collective

129:   Input Parameters:
130: + mesh - The Mesh object
131: - name - The section name

133:   Output Parameters:
134: + numElements - The number of mesh element with values
135: . fiberDim - The number of values per element
136: - array - The array

138:   Level: intermediate

140: .keywords: mesh, elements
141: .seealso: MeshCreate()
142: @*/
143: PetscErrorCode BCSectionRealGetArray(Mesh mesh, const char name[], PetscInt *numElements, PetscInt *fiberDim, PetscReal *array[])
144: {
145:   ALE::Obj<PETSC_MESH_TYPE> m;
146:   PetscErrorCode      ierr;

149:   MeshGetMesh(mesh, m);
150:   const ALE::Obj<PETSC_MESH_TYPE::real_section_type>& section = m->getRealSection(std::string(name));
151:   if (!section->size()) {
152:     *numElements = 0;
153:     *fiberDim    = 0;
154:     *array       = NULL;
155:     return(0);
156:   }
157:   const PETSC_MESH_TYPE::real_section_type::chart_type& chart = section->getChart();
158:   int fiberDimMin = section->getFiberDimension(*chart.begin());
159:   int numElem     = 0;

161:   for(PETSC_MESH_TYPE::real_section_type::chart_type::const_iterator c_iter = chart.begin(); c_iter != chart.end(); ++c_iter) {
162:     const int fiberDim = section->getFiberDimension(*c_iter);

164:     if (fiberDim < fiberDimMin) fiberDimMin = fiberDim;
165:   }
166:   for(PETSC_MESH_TYPE::real_section_type::chart_type::const_iterator c_iter = chart.begin(); c_iter != chart.end(); ++c_iter) {
167:     const int fiberDim = section->getFiberDimension(*c_iter);

169:     numElem += fiberDim/fiberDimMin;
170:   }
171:   *numElements = numElem;
172:   *fiberDim    = fiberDimMin;
173:   *array       = (PetscReal *) section->restrictSpace();
174:   return(0);
175: }

179: PetscErrorCode BCFUNCGetArray(Mesh mesh, PetscInt *numElements, PetscInt *fiberDim, PetscScalar *array[])
180: {
181:   ALE::Obj<PETSC_MESH_TYPE> m;
182:   PetscErrorCode      ierr;

185:   MeshGetMesh(mesh, m);
186: #if 0
187:   PETSC_MESH_TYPE::bc_values_type& bcValues = m->getBCValues();
188:   *numElements = bcValues.size();
189:   *fiberDim    = 4;
190:   *array       = new PetscScalar[(*numElements)*(*fiberDim)];
191:   for(int bcf = 1; bcf <= (int) bcValues.size(); ++bcf) {
192:     (*array)[(bcf-1)*4+0] = bcValues[bcf].rho;
193:     (*array)[(bcf-1)*4+1] = bcValues[bcf].u;
194:     (*array)[(bcf-1)*4+2] = bcValues[bcf].v;
195:     (*array)[(bcf-1)*4+3] = bcValues[bcf].p;
196:   }
197: #else
198:   *numElements = 0;
199:   *fiberDim    = 0;
200:   *array       = NULL;
201: #endif
202:   return(0);
203: }

205: namespace ALE {
206:   namespace PCICE {
207:     //
208:     // Builder methods
209:     //
210:     void Builder::readConnectivity(MPI_Comm comm, const std::string& filename, int& corners, const bool useZeroBase, int& numElements, int *vertices[]) {
211:       PetscViewer    viewer;
212:       FILE          *f;
213:       PetscInt       numCells, cellCount = 0;
214:       PetscInt      *verts;
215:       char           buf[2048];
216:       PetscInt       c;
217:       PetscInt       commRank;

220:       MPI_Comm_rank(comm, &commRank);

222:       if (commRank != 0) return;
223:       PetscViewerCreate(PETSC_COMM_SELF, &viewer);
224:       PetscViewerSetType(viewer, PETSC_VIEWER_ASCII);
225:       PetscViewerFileSetMode(viewer, FILE_MODE_READ);
226:       PetscViewerFileSetName(viewer, filename.c_str());
227:       if (ierr) {
228:         ostringstream txt;
229:         txt << "Could not open PCICE connectivity file: " << filename;
230:         throw ALE::Exception(txt.str().c_str());
231:       }
232:       PetscViewerASCIIGetPointer(viewer, &f);
233:       if (fgets(buf, 2048, f) == NULL) {
234:         throw ALE::Exception("Invalid connectivity file: Missing number of elements");
235:       }
236:       const char *sizes = strtok(buf, " ");
237:       numCells = atoi(sizes);
238:       sizes = strtok(NULL, " ");
239:       if (sizes != NULL) {
240:         corners = atoi(sizes);
241:         std::cout << "Reset corners to " << corners << std::endl;
242:       }
243:       PetscMalloc(numCells*corners * sizeof(PetscInt), &verts);
244:       while(fgets(buf, 2048, f) != NULL) {
245:         const char *v = strtok(buf, " ");
246: 
247:         /* Ignore cell number */
248:         v = strtok(NULL, " ");
249:         for(c = 0; c < corners; c++) {
250:           int vertex = atoi(v);
251: 
252:           if (!useZeroBase) vertex -= 1;
253:           verts[cellCount*corners+c] = vertex;
254:           v = strtok(NULL, " ");
255:         }
256:         cellCount++;
257:       }
258:       PetscViewerDestroy(viewer);
259:       numElements = numCells;
260:       *vertices = verts;
261:     };
262:     void Builder::readCoordinates(MPI_Comm comm, const std::string& filename, const int dim, int& numVertices, double *coordinates[]) {
263:       PetscViewer    viewer;
264:       FILE          *f;
265:       PetscInt       numVerts, vertexCount = 0;
266:       PetscScalar   *coords;
267:       char           buf[2048];
268:       PetscInt       c;
269:       PetscInt       commRank;

272:       MPI_Comm_rank(comm, &commRank);

274:       if (commRank != 0) return;
275:       PetscViewerCreate(PETSC_COMM_SELF, &viewer);
276:       PetscViewerSetType(viewer, PETSC_VIEWER_ASCII);
277:       PetscViewerFileSetMode(viewer, FILE_MODE_READ);
278:       PetscViewerFileSetName(viewer, filename.c_str());
279:       if (ierr) {
280:         ostringstream txt;
281:         txt << "Could not open PCICE coordinate file: " << filename;
282:         throw ALE::Exception(txt.str().c_str());
283:       }
284:       PetscViewerASCIIGetPointer(viewer, &f);
285:       numVerts = atoi(fgets(buf, 2048, f));
286:       PetscMalloc(numVerts*dim * sizeof(PetscScalar), &coords);
287:       while(fgets(buf, 2048, f) != NULL) {
288:         const char *x = strtok(buf, " ");
289: 
290:         /* Ignore vertex number */
291:         x = strtok(NULL, " ");
292:         for(c = 0; c < dim; c++) {
293:           coords[vertexCount*dim+c] = atof(x);
294:           x = strtok(NULL, " ");
295:         }
296:         vertexCount++;
297:       }
298:       PetscViewerDestroy(viewer);
299:       numVertices = numVerts;
300:       *coordinates = coords;
301:     };
302:     Obj<PETSC_MESH_TYPE> Builder::readMesh(MPI_Comm comm, const int dim, const std::string& basename, const bool useZeroBase = true, const bool interpolate = true, const int debug = 0) {
303:       return readMesh(comm, dim, basename+".nodes", basename+".lcon", useZeroBase, interpolate, debug);
304:     };
305: #ifdef PETSC_OPT_SIEVE
306:     Obj<PETSC_MESH_TYPE> Builder::readMesh(MPI_Comm comm, const int dim, const std::string& coordFilename, const std::string& adjFilename, const bool useZeroBase = true, const bool interpolate = true, const int debug = 0) {
307:       Obj<Mesh>          mesh  = new Mesh(comm, dim, debug);
308:       Obj<sieve_type>    sieve = new sieve_type(comm, debug);
309:       const Obj<ALE::Mesh>             m = new ALE::Mesh(comm, dim, debug);
310:       const Obj<ALE::Mesh::sieve_type> s = new ALE::Mesh::sieve_type(comm, debug);
311:       int    *cells            = NULL;
312:       double *coordinates      = NULL;
313:       int     numCells = 0, numVertices = 0, numCorners = dim+1;

316:       ALE::PCICE::Builder::readConnectivity(comm, adjFilename, numCorners, useZeroBase, numCells, &cells);
317:       ALE::PCICE::Builder::readCoordinates(comm, coordFilename, dim, numVertices, &coordinates);
318:       ALE::SieveBuilder<ALE::Mesh>::buildTopology(s, dim, numCells, cells, numVertices, interpolate, numCorners, -1, m->getArrowSection("orientation"));
319:       m->setSieve(s);
320:       m->stratify();
321:       mesh->setSieve(sieve);
322:       std::map<Mesh::point_type,Mesh::point_type> renumbering;
323:       ALE::ISieveConverter::convertSieve(*s, *sieve, renumbering, false);
324:       mesh->stratify();
325:       ALE::ISieveConverter::convertOrientation(*s, *sieve, renumbering, m->getArrowSection("orientation").ptr());
326:       ALE::SieveBuilder<PETSC_MESH_TYPE>::buildCoordinates(mesh, dim, coordinates);
327:       if (cells) {PetscFree(cells);}
328:       if (coordinates) {PetscFree(coordinates);}
329:       return mesh;
330:     };
331:     void Builder::readBoundary(const Obj<Mesh>& mesh, const std::string& bcFilename) {
332:       throw ALE::Exception("Not implemented for optimized sieves");
333:     };
334:     void Builder::outputVerticesLocal(const Obj<Mesh>& mesh, int *numVertices, int *dim, double *coordinates[], const bool columnMajor) {
335:       const Obj<Mesh::real_section_type>& coordSec = mesh->getRealSection("coordinates");
336:       if (!coordSec->size()) {
337:         *numVertices = 0;
338:         *dim         = 0;
339:         *coordinates = NULL;
340:         return;
341:       }
342:       const Obj<Mesh::label_sequence>& vertices   = mesh->depthStratum(0);
343:       const Obj<Mesh::numbering_type>& vNumbering = mesh->getFactory()->getLocalNumbering(mesh, 0);
344:       int            size     = vertices->size();
345:       int            embedDim = coordSec->getFiberDimension(*vertices->begin());
346:       double        *coords;

349:       PetscMalloc(vertices->size()*embedDim * sizeof(double), &coords);
350:       for(Mesh::label_sequence::iterator v_iter = vertices->begin(); v_iter != vertices->end(); ++v_iter) {
351:         const Mesh::real_section_type::value_type *array = coordSec->restrictPoint(*v_iter);
352:         const int                                  row   = vNumbering->getIndex(*v_iter);

354:         if (columnMajor) {
355:           for(int d = 0; d < embedDim; d++) {
356:             coords[d*size + row] = array[d];
357:           }
358:         } else {
359:           for(int d = 0; d < embedDim; d++) {
360:             coords[row*embedDim + d] = array[d];
361:           }
362:         }
363:       }
364:       *numVertices = size;
365:       *dim         = embedDim;
366:       *coordinates = coords;
367:     };
368:     void Builder::outputElementsLocal(const Obj<Mesh>& mesh, int *numElements, int *numCorners, int *vertices[], const bool columnMajor) {
369:       if (!mesh->heightStratum(0)->size()) {
370:         *numElements = 0;
371:         *numCorners  = 0;
372:         *vertices    = NULL;
373:         return;
374:       }
375:       const Obj<Mesh::sieve_type>&     sieve      = mesh->getSieve();
376:       const Obj<Mesh::label_sequence>& elements   = mesh->heightStratum(0);
377:       const Obj<Mesh::numbering_type>& eNumbering = mesh->getFactory()->getLocalNumbering(mesh, mesh->depth());
378:       const Obj<Mesh::numbering_type>& vNumbering = mesh->getFactory()->getLocalNumbering(mesh, 0);
379:       int            size         = elements->size();
380:       //int            corners      = sieve->nCone(*elements->begin(), topology->depth())->size();
381:       int            corners      = sieve->getConeSize(*elements->begin());
382:       int           *v;

385:       PetscMalloc(size*corners * sizeof(int), &v);
386:       for(Mesh::label_sequence::iterator e_iter = elements->begin(); e_iter != elements->end(); ++e_iter) {
387:         const Obj<Mesh::sieve_type::coneSequence>      cone  = sieve->cone(*e_iter);
388:         Mesh::sieve_type::coneSequence::const_iterator begin = cone->begin();
389:         Mesh::sieve_type::coneSequence::const_iterator end   = cone->end();

391:         const int row = eNumbering->getIndex(*e_iter);
392:         int       c   = -1;
393:         if (columnMajor) {
394:           for(Mesh::sieve_type::coneSequence::iterator c_iter = begin; c_iter != end; ++c_iter) {
395:             v[(++c)*size + row] = vNumbering->getIndex(*c_iter)+1;
396:           }
397:         } else {
398:           for(Mesh::sieve_type::coneSequence::iterator c_iter = begin; c_iter != end; ++c_iter) {
399:             v[row*corners + ++c] = vNumbering->getIndex(*c_iter)+1;
400:           }
401:         }
402:       }
403:       *numElements = size;
404:       *numCorners  = corners;
405:       *vertices    = v;
406:     };
407:     PetscErrorCode Viewer::writeVertices(const ALE::Obj<Mesh>& mesh, PetscViewer viewer) {
408:       throw ALE::Exception("Not implemented for optimized sieves");
409:     };
410:     PetscErrorCode Viewer::writeElements(const ALE::Obj<Mesh>& mesh, PetscViewer viewer) {
411:       throw ALE::Exception("Not implemented for optimized sieves");
412:     };
413:     PetscErrorCode Viewer::writeVerticesLocal(const Obj<Mesh>& mesh, PetscViewer viewer) {
414:       throw ALE::Exception("Not implemented for optimized sieves");
415:     };
416:     PetscErrorCode Viewer::writeRestart(const Obj<Mesh>& mesh, PetscViewer viewer) {
417:       throw ALE::Exception("Not implemented for optimized sieves");
418:     };
419:     void fuseBoundary(const ALE::Obj<PETSC_MESH_TYPE>& mesh) {
420:       throw ALE::Exception("Not implemented for optimized sieves");
421:     };
422: #else
423:     Obj<PETSC_MESH_TYPE> Builder::readMesh(MPI_Comm comm, const int dim, const std::string& coordFilename, const std::string& adjFilename, const bool useZeroBase = true, const bool interpolate = true, const int debug = 0) {
424:       Obj<Mesh>          mesh     = new Mesh(comm, dim, debug);
425:       Obj<sieve_type>    sieve    = new sieve_type(comm, debug);
426:       int    *cells = NULL;
427:       double *coordinates = NULL;
428:       int     numCells = 0, numVertices = 0, numCorners = dim+1;

431:       ALE::PCICE::Builder::readConnectivity(comm, adjFilename, numCorners, useZeroBase, numCells, &cells);
432:       ALE::PCICE::Builder::readCoordinates(comm, coordFilename, dim, numVertices, &coordinates);
433:       ALE::SieveBuilder<PETSC_MESH_TYPE>::buildTopology(sieve, dim, numCells, cells, numVertices, interpolate, numCorners, -1, mesh->getArrowSection("orientation"));
434:       mesh->setSieve(sieve);
435:       mesh->stratify();
436:       ALE::SieveBuilder<PETSC_MESH_TYPE>::buildCoordinates(mesh, dim, coordinates);
437:       if (cells) {PetscFree(cells);}
438:       if (coordinates) {PetscFree(coordinates);}
439:       return mesh;
440:     };
441:     // Creates boundary sections:
442:     //   IBC[NBFS,2]:     ALL
443:     //     BL[NBFS,1]:
444:     //     BNVEC[NBFS,2]:
445:     //   BCFUNC[NBCF,NV]: ALL
446:     //   IBNDFS[NBN,2]:   STILL NEED 4-5
447:     //     BNNV[NBN,2]
448:     void Builder::readBoundary(const Obj<Mesh>& mesh, const std::string& bcFilename) {
449:       PetscViewer    viewer;
450:       FILE          *f;
451:       char           buf[2048];

454:       const Obj<Mesh::int_section_type>&  ibc    = mesh->getIntSection("IBC");
455:       const Obj<Mesh::int_section_type>&  ibndfs = mesh->getIntSection("IBNDFS");
456:       const Obj<Mesh::int_section_type>&  ibcnum = mesh->getIntSection("IBCNUM");
457:       const Obj<Mesh::int_section_type>&  ibfcon = mesh->getIntSection("IBFCON");
458:       const Obj<Mesh::real_section_type>& bl     = mesh->getRealSection("BL");
459:       const Obj<Mesh::real_section_type>& bnvec  = mesh->getRealSection("BNVEC");
460:       const Obj<Mesh::real_section_type>& bnnv   = mesh->getRealSection("BNNV");
461:       if (mesh->commRank() != 0) {
462: #if 0
463:         mesh->distributeBCValues();
464: #endif
465:         return;
466:       }
467:       PetscViewerCreate(PETSC_COMM_SELF, &viewer);
468:       PetscViewerSetType(viewer, PETSC_VIEWER_ASCII);
469:       PetscViewerFileSetMode(viewer, FILE_MODE_READ);
470:       PetscViewerFileSetName(viewer, bcFilename.c_str());
471:       PetscViewerASCIIGetPointer(viewer, &f);
472:       // Create IBC section
473:       int  numBdFaces = atoi(strtok(fgets(buf, 2048, f), " "));
474:       int *tmpIBC     = new int[numBdFaces*4];
475:       std::map<int,std::set<int> > elem2Idx;
476:       std::map<int,int> bfReorder;
477:       for(int bf = 0; bf < numBdFaces; bf++) {
478:         const char *x = strtok(fgets(buf, 2048, f), " ");

480:         // Ignore boundary face number
481:         x = strtok(NULL, " ");
482:         tmpIBC[bf*4+0] = atoi(x);
483:         x = strtok(NULL, " ");
484:         tmpIBC[bf*4+1] = atoi(x);
485:         x = strtok(NULL, " ");
486:         tmpIBC[bf*4+2] = atoi(x);
487:         x = strtok(NULL, " ");
488:         tmpIBC[bf*4+3] = atoi(x);
489:         const int elem = tmpIBC[bf*4+0]-1;

491:         ibc->addFiberDimension(elem, 4);
492:         ibcnum->addFiberDimension(elem, 1);
493:         ibfcon->addFiberDimension(elem, 2);
494:         bl->addFiberDimension(elem, 1);
495:         bnvec->addFiberDimension(elem, 2);
496:         elem2Idx[elem].insert(bf);
497:       }
498:       mesh->allocate(ibc);
499:       mesh->allocate(ibcnum);
500:       mesh->allocate(ibfcon);
501:       mesh->allocate(bl);
502:       mesh->allocate(bnvec);
503:       const Mesh::int_section_type::chart_type& chart = ibc->getChart();
504:       int num = 1;

506:       for(Mesh::int_section_type::chart_type::const_iterator p_iter = chart.begin(); p_iter != chart.end(); ++p_iter) {
507:         const int elem = *p_iter;
508:         int bfNum[2];
509:         int k = 0;

511:         for(std::set<int>::const_iterator i_iter = elem2Idx[elem].begin(); i_iter != elem2Idx[elem].end(); ++i_iter) {
512:           bfReorder[(*i_iter)+1] = num;
513:           bfNum[k++] = num;
514:           num++;
515:         }
516:         ibcnum->updatePoint(elem, bfNum);
517:       }
518:       for(int bf = 0; bf < numBdFaces; bf++) {
519:         const int elem = tmpIBC[bf*4]-1;

521:         if (elem2Idx[elem].size() > 1) {
522:           if (*elem2Idx[elem].begin() == bf) {
523:             int values[8];
524:             int k = 0;

526:             for(std::set<int>::const_iterator i_iter = elem2Idx[elem].begin(); i_iter != elem2Idx[elem].end(); ++i_iter) {
527:               for(int v = 0; v < 4; ++v) {
528:                 values[k*4+v] = tmpIBC[*i_iter*4+v];
529:               }
530:               k++;
531:             }
532:             ibc->updatePoint(elem, values);
533:           }
534:         } else {
535:           ibc->updatePoint(elem, &tmpIBC[bf*4]);
536:         }
537:       }
538:       delete [] tmpIBC;
539:       // Create BCFUNC section
540:       int numBcFunc = atoi(strtok(fgets(buf, 2048, f), " "));
541:       if (numBcFunc != 0) {throw ALE::Exception("Cannot handle BCFUNCS after rewrite");}
542:       for(int bc = 0; bc < numBcFunc; bc++) {
543: #if 0
544:         const char *x = strtok(fgets(buf, 2048, f), " ");
545:         Mesh::bc_value_type value;

547:         // Ignore function number
548:         x = strtok(NULL, " ");
549:         value.rho = atof(x);
550:         x = strtok(NULL, " ");
551:         value.u   = atof(x);
552:         x = strtok(NULL, " ");
553:         value.v   = atof(x);
554:         x = strtok(NULL, " ");
555:         value.p   = atof(x);
556:         mesh->setBCValue(bc+1, value);
557: #endif
558:       }
559: #if 0
560:       mesh->distributeBCValues();
561: #endif
562:       // Create IBNDFS section
563:       int       numBdVertices = atoi(strtok(fgets(buf, 2048, f), " "));
564:       const int numElements   = mesh->heightStratum(0)->size();
565:       int      *tmpIBNDFS     = new int[numBdVertices*3];

567:       for(int bv = 0; bv < numBdVertices; bv++) {
568:         const char *x = strtok(fgets(buf, 2048, f), " ");

570:         // Ignore boundary node number
571:         x = strtok(NULL, " ");
572:         tmpIBNDFS[bv*3+0] = atoi(x);
573:         x = strtok(NULL, " ");
574:         tmpIBNDFS[bv*3+1] = atoi(x);
575:         x = strtok(NULL, " ");
576:         tmpIBNDFS[bv*3+2] = atoi(x);
577:         ibndfs->setFiberDimension(tmpIBNDFS[bv*3+0]-1+numElements, 6);
578:       }
579:       mesh->allocate(ibndfs);
580:       for(int bv = 0; bv < numBdVertices; bv++) {
581:         int values[5];

583:         values[0] = tmpIBNDFS[bv*3+0];
584:         // Covert to new boundary face numbers
585:         values[1] = bfReorder[tmpIBNDFS[bv*3+1]];
586:         values[2] = bfReorder[tmpIBNDFS[bv*3+2]];
587:         values[3] = 0;
588:         values[4] = 0;
589:         ibndfs->updatePoint(values[0]-1+numElements, values);
590:       }
591:       PetscViewerDestroy(viewer);
592:       // Create BNNV[NBN,2]
593:       const int dim = mesh->getDimension();

595:       for(int bv = 0; bv < numBdVertices; bv++) {
596:         bnnv->setFiberDimension(tmpIBNDFS[bv*3+0]-1+numElements, dim);
597:       }
598:       mesh->allocate(bnnv);
599:       delete [] tmpIBNDFS;
600:     };
601:     void Builder::outputVerticesLocal(const Obj<Mesh>& mesh, int *numVertices, int *dim, double *coordinates[], const bool columnMajor) {
602:       const Obj<Mesh::real_section_type>& coordSec = mesh->getRealSection("coordinates");
603:       if (!coordSec->size()) {
604:         *numVertices = 0;
605:         *dim         = 0;
606:         *coordinates = NULL;
607:         return;
608:       }
609:       const Obj<Mesh::label_sequence>& vertices   = mesh->depthStratum(0);
610:       const Obj<Mesh::numbering_type>& vNumbering = mesh->getFactory()->getLocalNumbering(mesh, 0);
611:       int            size     = vertices->size();
612:       int            embedDim = coordSec->getFiberDimension(*vertices->begin());
613:       double        *coords;

616:       PetscMalloc(vertices->size()*embedDim * sizeof(double), &coords);
617:       for(Mesh::label_sequence::iterator v_iter = vertices->begin(); v_iter != vertices->end(); ++v_iter) {
618:         const Mesh::real_section_type::value_type *array = coordSec->restrictPoint(*v_iter);
619:         const int                                  row   = vNumbering->getIndex(*v_iter);

621:         if (columnMajor) {
622:           for(int d = 0; d < embedDim; d++) {
623:             coords[d*size + row] = array[d];
624:           }
625:         } else {
626:           for(int d = 0; d < embedDim; d++) {
627:             coords[row*embedDim + d] = array[d];
628:           }
629:         }
630:       }
631:       *numVertices = size;
632:       *dim         = embedDim;
633:       *coordinates = coords;
634:     };
635:     void Builder::outputElementsLocal(const Obj<Mesh>& mesh, int *numElements, int *numCorners, int *vertices[], const bool columnMajor) {
636:       if (!mesh->heightStratum(0)->size()) {
637:         *numElements = 0;
638:         *numCorners  = 0;
639:         *vertices    = NULL;
640:         return;
641:       }
642:       const Obj<Mesh::sieve_type>&     sieve      = mesh->getSieve();
643:       const Obj<Mesh::label_sequence>& elements   = mesh->heightStratum(0);
644:       const Obj<Mesh::numbering_type>& eNumbering = mesh->getFactory()->getLocalNumbering(mesh, mesh->depth());
645:       const Obj<Mesh::numbering_type>& vNumbering = mesh->getFactory()->getLocalNumbering(mesh, 0);
646:       int            size         = elements->size();
647:       //int            corners      = sieve->nCone(*elements->begin(), topology->depth())->size();
648:       int            corners      = sieve->getConeSize(*elements->begin());
649:       int           *v;

652:       PetscMalloc(elements->size()*corners * sizeof(int), &v);
653:       for(Mesh::label_sequence::iterator e_iter = elements->begin(); e_iter != elements->end(); ++e_iter) {
654:         const Obj<Mesh::sieve_type::traits::coneSequence> cone  = sieve->cone(*e_iter);
655:         Mesh::sieve_type::traits::coneSequence::iterator  begin = cone->begin();
656:         Mesh::sieve_type::traits::coneSequence::iterator  end   = cone->end();

658:         const int row = eNumbering->getIndex(*e_iter);
659:         int       c   = -1;
660:         if (columnMajor) {
661:           for(Mesh::sieve_type::traits::coneSequence::iterator c_iter = begin; c_iter != end; ++c_iter) {
662:             v[(++c)*size + row] = vNumbering->getIndex(*c_iter)+1;
663:           }
664:         } else {
665:           for(Mesh::sieve_type::traits::coneSequence::iterator c_iter = begin; c_iter != end; ++c_iter) {
666:             v[row*corners + ++c] = vNumbering->getIndex(*c_iter)+1;
667:           }
668:         }
669:       }
670:       *numElements = size;
671:       *numCorners  = corners;
672:       *vertices    = v;
673:     };
676:     PetscErrorCode Viewer::writeVertices(const ALE::Obj<Mesh>& mesh, PetscViewer viewer) {
677:       ALE::Obj<Mesh::real_section_type> coordinates = mesh->getRealSection("coordinates");
678: #if 0
679:       Mesh::field_type::patch_type patch;
680:       const double  *array = coordinates->restrict(patch);
681:       int            numVertices;

685:       //FIX:
686:       if (vertexBundle->getGlobalOffsets()) {
687:         numVertices = vertexBundle->getGlobalOffsets()[mesh->commSize()];
688:       } else {
689:         numVertices = mesh->getTopology()->depthStratum(0)->size();
690:       }
691:       PetscViewerASCIIPrintf(viewer, "%D\n", numVertices);
692:       if (mesh->commRank() == 0) {
693:         int numLocalVertices = mesh->getTopology()->depthStratum(0)->size();
694:         int embedDim = coordinates->getFiberDimension(patch, *mesh->getTopology()->depthStratum(0)->begin());
695:         int vertexCount = 1;

697:         for(int v = 0; v < numLocalVertices; v++) {
698:           PetscViewerASCIIPrintf(viewer, "%7D   ", vertexCount++);
699:           for(int d = 0; d < embedDim; d++) {
700:             if (d > 0) {
701:               PetscViewerASCIIPrintf(viewer, " ");
702:             }
703:             PetscViewerASCIIPrintf(viewer, "% 12.5E", array[v*embedDim+d]);
704:           }
705:           PetscViewerASCIIPrintf(viewer, "\n");
706:         }
707:         for(int p = 1; p < mesh->commSize(); p++) {
708:           double    *remoteCoords;
709:           MPI_Status status;

711:           MPI_Recv(&numLocalVertices, 1, MPI_INT, p, 1, mesh->comm(), &status);
712:           PetscMalloc(numLocalVertices*embedDim * sizeof(double), &remoteCoords);
713:           MPI_Recv(remoteCoords, numLocalVertices*embedDim, MPI_DOUBLE, p, 1, mesh->comm(), &status);
714:           for(int v = 0; v < numLocalVertices; v++) {
715:             PetscViewerASCIIPrintf(viewer,"%7D   ", vertexCount++);
716:             for(int d = 0; d < embedDim; d++) {
717:               if (d > 0) {
718:                 PetscViewerASCIIPrintf(viewer, " ");
719:               }
720:               PetscViewerASCIIPrintf(viewer, "% 12.5E", remoteCoords[v*embedDim+d]);
721:             }
722:             PetscViewerASCIIPrintf(viewer, "\n");
723:           }
724:         }
725:       } else {
726:         ALE::Obj<Mesh::bundle_type>                           globalOrder = coordinates->getGlobalOrder();
727:         ALE::Obj<Mesh::bundle_type::order_type::coneSequence> cone        = globalOrder->getPatch(patch);
728:         const int *offsets = coordinates->getGlobalOffsets();
729:         int        embedDim = coordinates->getFiberDimension(patch, *mesh->getTopology()->depthStratum(0)->begin());
730:         int        numLocalVertices = (offsets[mesh->commRank()+1] - offsets[mesh->commRank()])/embedDim;
731:         double    *localCoords;
732:         int        k = 0;

734:         PetscMalloc(numLocalVertices*embedDim * sizeof(double), &localCoords);
735:         for(Mesh::bundle_type::order_type::coneSequence::iterator p_iter = cone->begin(); p_iter != cone->end(); ++p_iter) {
736:           int dim = globalOrder->getFiberDimension(patch, *p_iter);

738:           if (dim > 0) {
739:             int offset = coordinates->getFiberOffset(patch, *p_iter);

741:             for(int i = offset; i < offset+dim; ++i) {
742:               localCoords[k++] = array[i];
743:             }
744:           }
745:         }
746:         if (k != numLocalVertices*embedDim) {
747:           SETERRQ2(PETSC_ERR_PLIB, "Invalid number of coordinates to send %d should be %d", k, numLocalVertices*embedDim);
748:         }
749:         MPI_Send(&numLocalVertices, 1, MPI_INT, 0, 1, mesh->comm());
750:         MPI_Send(localCoords, numLocalVertices*embedDim, MPI_DOUBLE, 0, 1, mesh->comm());
751:         PetscFree(localCoords);
752:       }
753: #endif
754:       return(0);
755:     };
758:     PetscErrorCode Viewer::writeElements(const ALE::Obj<Mesh>& mesh, PetscViewer viewer) {
759: #if 0
760:       ALE::Obj<Mesh::sieve_type::traits::heightSequence> elements = topology->heightStratum(0);
761:       ALE::Obj<Mesh::bundle_type> elementBundle = mesh->getBundle(topology->depth());
762:       ALE::Obj<Mesh::bundle_type> vertexBundle = mesh->getBundle(0);
763:       ALE::Obj<Mesh::bundle_type> globalVertex = vertexBundle->getGlobalOrder();
764:       ALE::Obj<Mesh::bundle_type> globalElement = elementBundle->getGlobalOrder();
765:       Mesh::bundle_type::patch_type patch;
766:       std::string    orderName("element");
767:       int            dim  = mesh->getDimension();
768:       int            corners = topology->nCone(*elements->begin(), topology->depth())->size();
769:       int            numElements;

773:       if (corners != dim+1) {
774:         SETERRQ(PETSC_ERR_SUP, "PCICE only supports simplicies");
775:       }
776:       if (!globalVertex) {
777:         globalVertex = vertexBundle;
778:       }
779:       if (elementBundle->getGlobalOffsets()) {
780:         numElements = elementBundle->getGlobalOffsets()[mesh->commSize()];
781:       } else {
782:         numElements = mesh->getTopology()->heightStratum(0)->size();
783:       }
784:       if (mesh->commRank() == 0) {
785:         int elementCount = 1;

787:         PetscViewerASCIIPrintf(viewer, "%d\n", numElements);
788:         for(Mesh::sieve_type::traits::heightSequence::iterator e_itor = elements->begin(); e_itor != elements->end(); ++e_itor) {
789:           ALE::Obj<Mesh::bundle_type::order_type::coneSequence> cone = vertexBundle->getPatch(orderName, *e_itor);

791:           PetscViewerASCIIPrintf(viewer, "%7d", elementCount++);
792:           for(Mesh::bundle_type::order_type::coneSequence::iterator c_itor = cone->begin(); c_itor != cone->end(); ++c_itor) {
793:             PetscViewerASCIIPrintf(viewer, " %7d", globalVertex->getIndex(patch, *c_itor).prefix);
794:           }
795:           PetscViewerASCIIPrintf(viewer, "\n");
796:         }
797:         for(int p = 1; p < mesh->commSize(); p++) {
798:           int        numLocalElements;
799:           int       *remoteVertices;
800:           MPI_Status status;

802:           MPI_Recv(&numLocalElements, 1, MPI_INT, p, 1, mesh->comm(), &status);
803:           PetscMalloc(numLocalElements*corners * sizeof(int), &remoteVertices);
804:           MPI_Recv(remoteVertices, numLocalElements*corners, MPI_INT, p, 1, mesh->comm(), &status);
805:           for(int e = 0; e < numLocalElements; e++) {
806:             PetscViewerASCIIPrintf(viewer, "%7d", elementCount++);
807:             for(int c = 0; c < corners; c++) {
808:               PetscViewerASCIIPrintf(viewer, " %7d", remoteVertices[e*corners+c]);
809:             }
810:             PetscViewerASCIIPrintf(viewer, "\n");
811:           }
812:           PetscFree(remoteVertices);
813:         }
814:       } else {
815:         const int *offsets = elementBundle->getGlobalOffsets();
816:         int        numLocalElements = offsets[mesh->commRank()+1] - offsets[mesh->commRank()];
817:         int       *localVertices;
818:         int        k = 0;

820:         PetscMalloc(numLocalElements*corners * sizeof(int), &localVertices);
821:         for(Mesh::sieve_type::traits::heightSequence::iterator e_itor = elements->begin(); e_itor != elements->end(); ++e_itor) {
822:           ALE::Obj<Mesh::bundle_type::order_type::coneSequence> cone = vertexBundle->getPatch(orderName, *e_itor);

824:           if (globalElement->getFiberDimension(patch, *e_itor) > 0) {
825:             for(Mesh::bundle_type::order_type::coneSequence::iterator c_itor = cone->begin(); c_itor != cone->end(); ++c_itor) {
826:               localVertices[k++] = globalVertex->getIndex(patch, *c_itor).prefix;
827:             }
828:           }
829:         }
830:         if (k != numLocalElements*corners) {
831:           SETERRQ2(PETSC_ERR_PLIB, "Invalid number of vertices to send %d should be %d", k, numLocalElements*corners);
832:         }
833:         MPI_Send(&numLocalElements, 1, MPI_INT, 0, 1, mesh->comm());
834:         MPI_Send(localVertices, numLocalElements*corners, MPI_INT, 0, 1, mesh->comm());
835:         PetscFree(localVertices);
836:       }
837: #endif
838:       return(0);
839:     };
842:     PetscErrorCode Viewer::writeVerticesLocal(const Obj<Mesh>& mesh, PetscViewer viewer) {
843:       Obj<Mesh::real_section_type>     coordinates = mesh->getRealSection("coordinates");
844:       const Obj<Mesh::label_sequence>& vertices    = mesh->depthStratum(0);
845:       const Obj<Mesh::numbering_type>& vNumbering  = mesh->getFactory()->getLocalNumbering(mesh, 0);
846:       int            embedDim = coordinates->getFiberDimension(*vertices->begin());

850:       PetscViewerASCIIPrintf(viewer, "%D\n", vertices->size());
851:       for(Mesh::label_sequence::iterator v_iter = vertices->begin(); v_iter != vertices->end(); ++v_iter) {
852:         const Mesh::real_section_type::value_type *array = coordinates->restrictPoint(*v_iter);

854:         PetscViewerASCIIPrintf(viewer, "%7D   ", vNumbering->getIndex(*v_iter)+1);
855:         for(int d = 0; d < embedDim; d++) {
856:           if (d > 0) {
857:             PetscViewerASCIIPrintf(viewer, " ");
858:           }
859:           PetscViewerASCIIPrintf(viewer, "% 12.5E", array[d]);
860:         }
861:         PetscViewerASCIIPrintf(viewer, "\n");
862:       }
863:       return(0);
864:     };
867:     PetscErrorCode Viewer::writeRestart(const Obj<Mesh>& mesh, PetscViewer viewer) {
868:       const Obj<Mesh::real_section_type>&   velocity    = mesh->getRealSection("VELN");
869:       const Obj<Mesh::real_section_type>&   pressure    = mesh->getRealSection("PN");
870:       const Obj<Mesh::real_section_type>&   temperature = mesh->getRealSection("TN");
871:       const Obj<Mesh::numbering_type>& cNumbering  = mesh->getFactory()->getNumbering(mesh, mesh->depth());
872:       const Obj<Mesh::numbering_type>& vNumbering  = mesh->getFactory()->getNumbering(mesh, 0);
873:       const int                        numCells    = cNumbering->getGlobalSize();

877:       int          blen[2];
878:       MPI_Aint     indices[2];
879:       MPI_Datatype oldtypes[2], newtype;
880:       blen[0] = 1; indices[0] = 0;           oldtypes[0] = MPI_INT;
881:       blen[1] = 4; indices[1] = sizeof(int); oldtypes[1] = MPI_DOUBLE;
882:       MPI_Type_struct(2, blen, indices, oldtypes, &newtype);
883:       MPI_Type_commit(&newtype);

885:       if (mesh->commRank() == 0) {
886:         const Obj<Mesh::label_sequence>& vertices = mesh->depthStratum(0);

888:         for(Mesh::label_sequence::iterator v_iter = vertices->begin(); v_iter != vertices->end(); ++v_iter) {
889:           if (vNumbering->isLocal(*v_iter)) {
890:             const Mesh::real_section_type::value_type *veln = velocity->restrictPoint(*v_iter);
891:             const Mesh::real_section_type::value_type *pn   = pressure->restrictPoint(*v_iter);
892:             const Mesh::real_section_type::value_type *tn   = temperature->restrictPoint(*v_iter);

894:             PetscViewerASCIIPrintf(viewer, "%6d% 16.8E% 16.8E% 16.8E% 16.8E\n", *v_iter-numCells+1, veln[0], veln[1], pn[0], tn[0]);
895:           }
896:         }
897:         for(int p = 1; p < mesh->commSize(); p++) {
898:           RestartType *remoteValues;
899:           int          numLocalElements;
900:           MPI_Status   status;

902:           MPI_Recv(&numLocalElements, 1, MPI_INT, p, 1, mesh->comm(), &status);
903:           PetscMalloc(numLocalElements * sizeof(RestartType), &remoteValues);
904:           MPI_Recv(remoteValues, numLocalElements, newtype, p, 1, mesh->comm(), &status);
905:           for(int e = 0; e < numLocalElements; e++) {
906:             PetscViewerASCIIPrintf(viewer, "%6d% 16.8E% 16.8E% 16.8E% 16.8E\n", remoteValues[e].vertex-numCells+1, remoteValues[e].veln_x, remoteValues[e].veln_y, remoteValues[e].pn, remoteValues[e].tn);
907:           }
908:         }
909:       } else {
910:         const Obj<Mesh::label_sequence>& vertices = mesh->depthStratum(0);
911:         RestartType *localValues;
912:         int numLocalElements = vNumbering->getLocalSize();
913:         int k = 0;

915:         PetscMalloc(numLocalElements * sizeof(RestartType), &localValues);
916:         for(Mesh::label_sequence::iterator v_iter = vertices->begin(); v_iter != vertices->end(); ++v_iter) {
917:           if (vNumbering->isLocal(*v_iter)) {
918:             const Mesh::real_section_type::value_type *veln = velocity->restrictPoint(*v_iter);
919:             const Mesh::real_section_type::value_type *pn   = pressure->restrictPoint(*v_iter);
920:             const Mesh::real_section_type::value_type *tn   = temperature->restrictPoint(*v_iter);

922:             localValues[k].vertex = *v_iter;
923:             localValues[k].veln_x = veln[0];
924:             localValues[k].veln_y = veln[1];
925:             localValues[k].pn     = pn[0];
926:             localValues[k].tn     = tn[0];
927:             k++;
928:           }
929:         }
930:         if (k != numLocalElements) {
931:           SETERRQ2(PETSC_ERR_PLIB, "Invalid number of values to send for field, %d should be %d", k, numLocalElements);
932:         }
933:         MPI_Send(&numLocalElements, 1, MPI_INT, 0, 1, mesh->comm());
934:         MPI_Send(localValues, numLocalElements, newtype, 0, 1, mesh->comm());
935:         PetscFree(localValues);
936:       }
937:       MPI_Type_free(&newtype);
938:       return(0);
939:     };

941:     //   This class reconstructs the local pieces of the boundary that distributed PCICE needs.
942:     // The boundary along with the boundary conditions is encoded in a collection of sections
943:     // over the PCICE mesh.  These sections contain a description of the boundary topology
944:     // using elements' global names.  This is unacceptable for PCICE, since it interprets
945:     // elements of the connectivity data arrays as local offsets into (some other of) these arrays.
946:     //   This subroutine performs the renumbering based on the local numbering on the distributed
947:     // mesh.  Essentially, we replace each global element id by its local number.
948:     //
949:     // Note: Any vertex or element number from PCICE is 1-based, but in Sieve we are 0-based. Thus
950:     //       we add and subtract 1 during conversion. Also, Sieve vertices are numbered after cells.
951:     void fuseBoundary(const ALE::Obj<PETSC_MESH_TYPE>& mesh) {
952:       // Extract PCICE boundary sections
953:       ALE::Obj<PETSC_MESH_TYPE::int_section_type> IBCsec    = mesh->getIntSection("IBC");
954:       ALE::Obj<PETSC_MESH_TYPE::int_section_type> IBNDFSsec = mesh->getIntSection("IBNDFS");
955:       ALE::Obj<PETSC_MESH_TYPE::int_section_type> IBCNUMsec = mesh->getIntSection("IBCNUM");

957:       // Look at the sections, if debugging
958:       if (mesh->debug()) {
959:         IBCsec->view("IBCsec", mesh->comm());
960:         IBNDFSsec->view("IBNDFSsec", mesh->comm());
961:       }

963:       // Extract the local numberings
964:       ALE::Obj<PETSC_MESH_TYPE::numbering_type> vertexNum  = mesh->getFactory()->getLocalNumbering(mesh, 0);
965:       ALE::Obj<PETSC_MESH_TYPE::numbering_type> elementNum = mesh->getFactory()->getLocalNumbering(mesh, mesh->depth());
966:       const int numElements = mesh->getFactory()->getNumbering(mesh, mesh->depth())->getGlobalSize();
967:       std::map<int,int> bfMap;
968:       // Declare points to the extracted numbering data
969:       const PETSC_MESH_TYPE::numbering_type::value_type *vNum, *eNum;

971:       // Create map from serial bdFace numbers to local bdFace numbers
972:       {
973:         const PETSC_MESH_TYPE::int_section_type::chart_type           chart = IBCNUMsec->getChart();
974:         PETSC_MESH_TYPE::int_section_type::chart_type::const_iterator begin = chart.begin();
975:         PETSC_MESH_TYPE::int_section_type::chart_type::const_iterator end   = chart.end();
976:         int num = 0;

978:         for(PETSC_MESH_TYPE::int_section_type::chart_type::const_iterator p_iter = begin; p_iter != end; ++p_iter) {
979:           const int  fiberDim  = IBCNUMsec->getFiberDimension(*p_iter);
980:           const int *globalNum = IBCNUMsec->restrictPoint(*p_iter);

982:           for(int n = 0; n < fiberDim; ++n) {
983:             bfMap[globalNum[n]] = ++num;
984:           }
985:         }
986:       }
987:       // Renumber vertex section IBC
988:       {
989:         const PETSC_MESH_TYPE::int_section_type::chart_type           IBCchart = IBCsec->getChart();
990:         PETSC_MESH_TYPE::int_section_type::chart_type::const_iterator begin    = IBCchart.begin();
991:         PETSC_MESH_TYPE::int_section_type::chart_type::const_iterator end      = IBCchart.end();
992:         for(PETSC_MESH_TYPE::int_section_type::chart_type::const_iterator p_iter = begin; p_iter != end; ++p_iter) {
993:           PETSC_MESH_TYPE::point_type p = *p_iter;
994:           const PETSC_MESH_TYPE::int_section_type::value_type *ibc_in = IBCsec->restrictPoint(p);
995:           int fiberDimension = IBCsec->getFiberDimension(p);
996:           PETSC_MESH_TYPE::int_section_type::value_type ibc_out[8];
997:           // k controls the update of edge connectivity for each containing element;
998:           // if fiberDimension is 4, only one boundary face is connected to the element, and that edge's data
999:           // are contained in entries 0 - 3 of the section over the element p;
1000:           // if fiberDimension is 8, two boundary faces are connected to the element, so the second edge's data
1001:           // are contained in entries 4 - 7
1002:           for(int k = 0; k < fiberDimension/4; k++) {
1003:             // Extract IBC entry 1 (entry kk*4) for edge kk connected to element p.
1004:             // This is the entry that needs renumbering for renumbering (2,3 & 4 are invariant under distribution),
1005:             // see IBC's description.
1006:             // Here we assume that elementNum's domain contains all boundary elements found in IBC,
1007:             // so we can restrict to the extracted entry.
1008:             eNum = elementNum->restrictPoint((PETSC_MESH_TYPE::numbering_type::point_type) ibc_in[k*4]-1);
1009:             ibc_out[k*4+0] = eNum[0]+1;
1010:             // Copy the other entries right over
1011:             ibc_out[k*4+1] = ibc_in[k*4+1];
1012:             ibc_out[k*4+2] = ibc_in[k*4+2];
1013:             ibc_out[k*4+3] = ibc_in[k*4+3];
1014:           }
1015:           // Update IBC
1016:           IBCsec->updatePoint(p, ibc_out);
1017:         }
1018:       }
1019:       {
1020:         // Renumber vertex section IBNDFS
1021:         const PETSC_MESH_TYPE::int_section_type::chart_type           IBNDFSchart = IBNDFSsec->getChart();
1022:         PETSC_MESH_TYPE::int_section_type::chart_type::const_iterator begin       = IBNDFSchart.begin();
1023:         PETSC_MESH_TYPE::int_section_type::chart_type::const_iterator end         = IBNDFSchart.end();
1024:         for(PETSC_MESH_TYPE::int_section_type::chart_type::const_iterator p_iter = begin; p_iter != end; ++p_iter) {
1025:           PETSC_MESH_TYPE::point_type p = *p_iter;
1026:           const PETSC_MESH_TYPE::int_section_type::value_type *ibndfs_in = IBNDFSsec->restrictPoint(p);
1027:           // Here we assume the fiber dimension is 5
1028:           PETSC_MESH_TYPE::int_section_type::value_type ibndfs_out[5];
1029:           // Renumber entries 1,2 & 3 (4 & 5 are invariant under distribution), see IBNDFS's description
1030:           // Here we assume that vertexNum's domain contains all boundary verticies found in IBNDFS, so we can restrict to the first extracted entry
1031:           vNum= vertexNum->restrictPoint((PETSC_MESH_TYPE::numbering_type::point_type)ibndfs_in[0]-1+numElements);
1032:           ibndfs_out[0] = vNum[0]+1;
1033:           // Map serial bdFace numbers to local bdFace numbers
1034:           ibndfs_out[1] = bfMap[ibndfs_in[1]];
1035:           ibndfs_out[2] = bfMap[ibndfs_in[2]];
1036:           // Copy the other entries right over
1037:           ibndfs_out[3] = ibndfs_in[3];
1038:           ibndfs_out[4] = ibndfs_in[4];
1039:           // Update IBNDFS
1040:           IBNDFSsec->updatePoint(p,ibndfs_out);
1041:         }
1042:       }
1043:       if (mesh->debug()) {
1044:         IBCsec->view("Renumbered IBCsec", mesh->comm());
1045:         IBNDFSsec->view("Renumbered IBNDFSsec", mesh->comm());
1046:       }
1047:       // It's not clear whether IBFCON needs to be renumbered (what does it mean that its entries are not "GLOBAL NODE NUMBER(s)" -- see IBFCON's descriptions
1048:     };
1049: #endif
1050:   };
1051: };