Actual source code: plexcreate.c

      1: #define PETSCDM_DLL
      2: #include <petsc/private/dmpleximpl.h>
      3: #include <petsc/private/hashseti.h>
      4: #include <petscsf.h>
      5: #include <petscdmplextransform.h>
      6: #include <petscdmlabelephemeral.h>
      7: #include <petsc/private/kernels/blockmatmult.h>
      8: #include <petsc/private/kernels/blockinvert.h>

     10: #ifdef PETSC_HAVE_UNISTD_H
     11:   #include <unistd.h>
     12: #endif
     13: #include <errno.h>

     15: PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_CreateFromOptions, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;

     17: /* External function declarations here */
     18: static PetscErrorCode DMInitialize_Plex(DM dm);

     20: PETSC_EXTERN PetscErrorCode DMPlexCheckEGADS_Private(DM dm)
     21: {
     22:   PetscObject modelObj;

     24:   PetscFunctionBegin;
     25:   PetscCall(PetscObjectQuery((PetscObject)dm, "EGADS Model", &modelObj));
     26:   PetscCheck(modelObj, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Input DM must have attached EGADS Geometry Model");
     27:   PetscFunctionReturn(PETSC_SUCCESS);
     28: }

     30: static PetscErrorCode DMPlexCopyContext_Private(DM dmin, const char name[], DM dmout)
     31: {
     32:   PetscObject obj;

     34:   PetscFunctionBegin;
     35:   PetscCall(PetscObjectQuery((PetscObject)dmin, name, &obj));
     36:   if (obj) PetscCall(PetscObjectCompose((PetscObject)dmout, name, obj));
     37:   PetscFunctionReturn(PETSC_SUCCESS);
     38: }

     40: static PetscErrorCode DMPlexSwapContext_Private(DM dmA, const char name[], DM dmB)
     41: {
     42:   PetscObject objA, objB;

     44:   PetscFunctionBegin;
     45:   PetscCall(PetscObjectQuery((PetscObject)dmA, name, &objA));
     46:   PetscCall(PetscObjectQuery((PetscObject)dmB, name, &objB));
     47:   PetscCall(PetscObjectReference(objA));
     48:   PetscCall(PetscObjectReference(objB));
     49:   PetscCall(PetscObjectCompose((PetscObject)dmA, name, objB));
     50:   PetscCall(PetscObjectCompose((PetscObject)dmB, name, objA));
     51:   PetscCall(PetscObjectDereference(objA));
     52:   PetscCall(PetscObjectDereference(objB));
     53:   PetscFunctionReturn(PETSC_SUCCESS);
     54: }

     56: PetscErrorCode DMPlexCopyEGADSInfo_Internal(DM dmin, DM dmout)
     57: {
     58:   PetscFunctionBegin;
     59:   PetscCall(DMPlexCopyContext_Private(dmin, "EGADS Model", dmout));
     60:   PetscCall(DMPlexCopyContext_Private(dmin, "EGADS Context", dmout));
     61:   PetscCall(DMPlexCopyContext_Private(dmin, "EGADSlite Model", dmout));
     62:   PetscCall(DMPlexCopyContext_Private(dmin, "EGADSlite Context", dmout));
     63:   PetscFunctionReturn(PETSC_SUCCESS);
     64: }

     66: static PetscErrorCode DMPlexSwapEGADSInfo_Private(DM dmA, DM dmB)
     67: {
     68:   PetscFunctionBegin;
     69:   PetscCall(DMPlexSwapContext_Private(dmA, "EGADS Model", dmB));
     70:   PetscCall(DMPlexSwapContext_Private(dmA, "EGADS Context", dmB));
     71:   PetscCall(DMPlexSwapContext_Private(dmA, "EGADSlite Model", dmB));
     72:   PetscCall(DMPlexSwapContext_Private(dmA, "EGADSlite Context", dmB));
     73:   PetscFunctionReturn(PETSC_SUCCESS);
     74: }

     76: /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
     77: PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
     78: {
     79:   const PetscReal     *maxCell, *Lstart, *L;
     80:   VecType              vecType;
     81:   MatType              matType;
     82:   PetscBool            dist, useCeed, balance_partition;
     83:   DMReorderDefaultFlag reorder;

     85:   PetscFunctionBegin;
     86:   if (dmin == dmout) PetscFunctionReturn(PETSC_SUCCESS);
     87:   PetscCall(DMGetVecType(dmin, &vecType));
     88:   PetscCall(DMSetVecType(dmout, vecType));
     89:   PetscCall(DMGetMatType(dmin, &matType));
     90:   PetscCall(DMSetMatType(dmout, matType));
     91:   if (copyPeriodicity) {
     92:     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
     93:     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
     94:     PetscCall(DMLocalizeCoordinates(dmout));
     95:   }
     96:   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
     97:   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
     98:   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
     99:   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
    100:   PetscCall(DMPlexGetUseCeed(dmin, &useCeed));
    101:   PetscCall(DMPlexSetUseCeed(dmout, useCeed));
    102:   PetscCall(DMPlexGetPartitionBalance(dmin, &balance_partition));
    103:   PetscCall(DMPlexSetPartitionBalance(dmout, balance_partition));
    104:   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
    105:   ((DM_Plex *)dmout->data)->printSetValues  = ((DM_Plex *)dmin->data)->printSetValues;
    106:   ((DM_Plex *)dmout->data)->printFEM        = ((DM_Plex *)dmin->data)->printFEM;
    107:   ((DM_Plex *)dmout->data)->printFVM        = ((DM_Plex *)dmin->data)->printFVM;
    108:   ((DM_Plex *)dmout->data)->printL2         = ((DM_Plex *)dmin->data)->printL2;
    109:   ((DM_Plex *)dmout->data)->printLocate     = ((DM_Plex *)dmin->data)->printLocate;
    110:   ((DM_Plex *)dmout->data)->printProject    = ((DM_Plex *)dmin->data)->printProject;
    111:   ((DM_Plex *)dmout->data)->printTol        = ((DM_Plex *)dmin->data)->printTol;
    112:   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
    113:   PetscCall(DMPlexCopyEGADSInfo_Internal(dmin, dmout));
    114:   PetscFunctionReturn(PETSC_SUCCESS);
    115: }

    117: /* Replace dm with the contents of ndm, and then destroy ndm
    118:    - Share the DM_Plex structure
    119:    - Share the coordinates
    120:    - Share the SF
    121: */
    122: PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
    123: {
    124:   PetscSF          sf;
    125:   DM               dmNew = *ndm, coordDM, coarseDM;
    126:   Vec              coords;
    127:   PetscPointFn    *coordFunc;
    128:   const PetscReal *maxCell, *Lstart, *L;
    129:   PetscInt         dim, cdim;
    130:   PetscBool        use_natural;

    132:   PetscFunctionBegin;
    133:   if (dm == dmNew) {
    134:     PetscCall(DMDestroy(ndm));
    135:     PetscFunctionReturn(PETSC_SUCCESS);
    136:   }
    137:   dm->setupcalled = dmNew->setupcalled;
    138:   if (!dm->hdr.name) {
    139:     const char *name;

    141:     PetscCall(PetscObjectGetName((PetscObject)*ndm, &name));
    142:     PetscCall(PetscObjectSetName((PetscObject)dm, name));
    143:   }
    144:   {
    145:     PetscInt ndim;

    147:     // If topological dimensions are the same, we retain the old coordinate map,
    148:     //   otherwise we overwrite with the new one
    149:     PetscCall(DMGetDimension(dm, &dim));
    150:     PetscCall(DMGetDimension(dmNew, &ndim));
    151:     PetscCall(DMPlexGetCoordinateMap(dm, &coordFunc));
    152:     if (dim == ndim) PetscCall(DMPlexSetCoordinateMap(dmNew, coordFunc));
    153:   }
    154:   PetscCall(DMGetDimension(dmNew, &dim));
    155:   PetscCall(DMSetDimension(dm, dim));
    156:   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
    157:   PetscCall(DMSetCoordinateDim(dm, cdim));
    158:   PetscCall(DMGetPointSF(dmNew, &sf));
    159:   PetscCall(DMSetPointSF(dm, sf));
    160:   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
    161:   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
    162:   PetscCall(DMSetCoordinateDM(dm, coordDM));
    163:   PetscCall(DMSetCoordinatesLocal(dm, coords));
    164:   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
    165:   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
    166:   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
    167:   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
    168:   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
    169:   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
    170:   dm->coordinates[0].field = dmNew->coordinates[0].field;
    171:   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
    172:   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
    173:   PetscCall(DMGetNaturalSF(dmNew, &sf));
    174:   PetscCall(DMSetNaturalSF(dm, sf));
    175:   PetscCall(DMGetUseNatural(dmNew, &use_natural));
    176:   PetscCall(DMSetUseNatural(dm, use_natural));
    177:   PetscCall(DMDestroy_Plex(dm));
    178:   PetscCall(DMInitialize_Plex(dm));
    179:   dm->data = dmNew->data;
    180:   ((DM_Plex *)dmNew->data)->refct++;
    181:   {
    182:     PetscInt       num_face_sfs;
    183:     const PetscSF *sfs;
    184:     PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &sfs));
    185:     PetscCall(DMPlexSetIsoperiodicFaceSF(dm, num_face_sfs, (PetscSF *)sfs)); // for the compose function effect on dm
    186:   }
    187:   PetscCall(DMDestroyLabelLinkList_Internal(dm));
    188:   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
    189:   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
    190:   PetscCall(DMSetCoarseDM(dm, coarseDM));
    191:   PetscCall(DMPlexCopyEGADSInfo_Internal(dmNew, dm));
    192:   PetscCall(DMDestroy(ndm));
    193:   PetscFunctionReturn(PETSC_SUCCESS);
    194: }

    196: /* Swap dm with the contents of dmNew
    197:    - Swap the DM_Plex structure
    198:    - Swap the coordinates
    199:    - Swap the point PetscSF
    200: */
    201: static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
    202: {
    203:   DM          coordDMA, coordDMB;
    204:   Vec         coordsA, coordsB;
    205:   PetscSF     sfA, sfB;
    206:   DMField     fieldTmp;
    207:   void       *tmp;
    208:   DMLabelLink listTmp;
    209:   DMLabel     depthTmp;
    210:   PetscInt    tmpI;

    212:   PetscFunctionBegin;
    213:   if (dmA == dmB) PetscFunctionReturn(PETSC_SUCCESS);
    214:   PetscCall(DMGetPointSF(dmA, &sfA));
    215:   PetscCall(DMGetPointSF(dmB, &sfB));
    216:   PetscCall(PetscObjectReference((PetscObject)sfA));
    217:   PetscCall(DMSetPointSF(dmA, sfB));
    218:   PetscCall(DMSetPointSF(dmB, sfA));
    219:   PetscCall(PetscObjectDereference((PetscObject)sfA));

    221:   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
    222:   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
    223:   PetscCall(PetscObjectReference((PetscObject)coordDMA));
    224:   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
    225:   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
    226:   PetscCall(PetscObjectDereference((PetscObject)coordDMA));

    228:   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
    229:   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
    230:   PetscCall(PetscObjectReference((PetscObject)coordsA));
    231:   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
    232:   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
    233:   PetscCall(PetscObjectDereference((PetscObject)coordsA));

    235:   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
    236:   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
    237:   PetscCall(PetscObjectReference((PetscObject)coordDMA));
    238:   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
    239:   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
    240:   PetscCall(PetscObjectDereference((PetscObject)coordDMA));

    242:   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
    243:   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
    244:   PetscCall(PetscObjectReference((PetscObject)coordsA));
    245:   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
    246:   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
    247:   PetscCall(PetscObjectDereference((PetscObject)coordsA));

    249:   PetscCall(DMPlexSwapEGADSInfo_Private(dmA, dmB));

    251:   fieldTmp                  = dmA->coordinates[0].field;
    252:   dmA->coordinates[0].field = dmB->coordinates[0].field;
    253:   dmB->coordinates[0].field = fieldTmp;
    254:   fieldTmp                  = dmA->coordinates[1].field;
    255:   dmA->coordinates[1].field = dmB->coordinates[1].field;
    256:   dmB->coordinates[1].field = fieldTmp;
    257:   tmp                       = dmA->data;
    258:   dmA->data                 = dmB->data;
    259:   dmB->data                 = tmp;
    260:   listTmp                   = dmA->labels;
    261:   dmA->labels               = dmB->labels;
    262:   dmB->labels               = listTmp;
    263:   depthTmp                  = dmA->depthLabel;
    264:   dmA->depthLabel           = dmB->depthLabel;
    265:   dmB->depthLabel           = depthTmp;
    266:   depthTmp                  = dmA->celltypeLabel;
    267:   dmA->celltypeLabel        = dmB->celltypeLabel;
    268:   dmB->celltypeLabel        = depthTmp;
    269:   tmpI                      = dmA->levelup;
    270:   dmA->levelup              = dmB->levelup;
    271:   dmB->levelup              = tmpI;
    272:   PetscFunctionReturn(PETSC_SUCCESS);
    273: }

    275: PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
    276: {
    277:   DM idm;

    279:   PetscFunctionBegin;
    280:   PetscCall(DMPlexInterpolate(dm, &idm));
    281:   PetscCall(DMPlexCopyCoordinates(dm, idm));
    282:   PetscCall(DMPlexReplace_Internal(dm, &idm));
    283:   PetscFunctionReturn(PETSC_SUCCESS);
    284: }

    286: /*@C
    287:   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates

    289:   Collective

    291:   Input Parameters:
    292: + dm        - The `DMPLEX`
    293: . degree    - The degree of the finite element or `PETSC_DECIDE`
    294: . localized - Flag to create a localized (DG) coordinate space
    295: - project   - Flag to project current coordinates into the space

    297:   Level: advanced

    299: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `PetscPointFn`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`, `DMPlexSetCoordinateMap()`
    300: @*/
    301: PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscBool localized, PetscBool project)
    302: {
    303:   PetscFE  fe = NULL;
    304:   DM       cdm;
    305:   PetscInt dim, cdim, dE, qorder, height;

    307:   PetscFunctionBegin;
    308:   PetscCall(DMGetDimension(dm, &dim));
    309:   cdim = dim;
    310:   PetscCall(DMGetCoordinateDim(dm, &dE));
    311:   qorder = degree;
    312:   PetscCall(DMGetCoordinateDM(dm, &cdm));
    313:   PetscObjectOptionsBegin((PetscObject)cdm);
    314:   PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
    315:   PetscCall(PetscOptionsBoundedInt("-dm_plex_coordinate_dim", "Set the coordinate dimension", "DMPlexCreateCoordinateSpace", cdim, &cdim, NULL, dim));
    316:   PetscOptionsEnd();
    317:   PetscCall(DMPlexGetVTKCellHeight(dm, &height));
    318:   if (cdim > dim) {
    319:     DM           cdm;
    320:     PetscSection cs, csNew;
    321:     Vec          coordinates, coordinatesNew;
    322:     VecType      vectype;
    323:     IS           idx;
    324:     PetscInt    *indices;
    325:     PetscInt     bs, n;

    327:     // Recreate coordinate section
    328:     {
    329:       const char *fieldName = NULL, *compName = NULL;
    330:       PetscInt    Nc, pStart, pEnd;

    332:       PetscCall(DMGetCoordinateDM(dm, &cdm));
    333:       PetscCall(DMGetLocalSection(cdm, &cs));
    334:       PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)cs), &csNew));
    335:       PetscCall(PetscSectionSetNumFields(csNew, 1));
    336:       PetscCall(PetscSectionGetFieldName(cs, 0, &fieldName));
    337:       PetscCall(PetscSectionSetFieldName(csNew, 0, fieldName));
    338:       PetscCall(PetscSectionGetFieldComponents(cs, 0, &Nc));
    339:       PetscCall(PetscSectionSetFieldComponents(csNew, 0, cdim));
    340:       for (PetscInt c = 0; c < Nc; ++c) {
    341:         PetscCall(PetscSectionGetComponentName(cs, 0, c, &compName));
    342:         PetscCall(PetscSectionSetComponentName(csNew, 0, c, compName));
    343:       }
    344:       PetscCall(PetscSectionGetChart(cs, &pStart, &pEnd));
    345:       PetscCall(PetscSectionSetChart(csNew, pStart, pEnd));
    346:       for (PetscInt p = pStart; p < pEnd; ++p) {
    347:         PetscInt dof;

    349:         PetscCall(PetscSectionGetDof(cs, p, &dof));
    350:         if (dof) {
    351:           PetscCall(PetscSectionSetDof(csNew, p, cdim));
    352:           PetscCall(PetscSectionSetFieldDof(csNew, p, 0, cdim));
    353:         }
    354:       }
    355:       PetscCall(PetscSectionSetUp(csNew));
    356:     }
    357:     PetscCall(DMSetLocalSection(cdm, csNew));
    358:     PetscCall(PetscSectionDestroy(&csNew));
    359:     // Reset coordinate dimension for coordinate DM
    360:     PetscCall(DMSetCoordinateDim(cdm, cdim));
    361:     PetscCall(DMSetCoordinateField(cdm, NULL));
    362:     // Inject coordinates into higher dimension
    363:     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
    364:     PetscCall(VecGetBlockSize(coordinates, &bs));
    365:     PetscCheck(bs == dim, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "We can only inject simple coordinates into a higher dimension");
    366:     PetscCall(VecCreate(PetscObjectComm((PetscObject)coordinates), &coordinatesNew));
    367:     PetscCall(VecGetType(coordinates, &vectype));
    368:     PetscCall(VecSetType(coordinatesNew, vectype));
    369:     PetscCall(VecGetLocalSize(coordinates, &n));
    370:     PetscCheck(!(n % bs), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "We can only inject simple coordinates into a higher dimension");
    371:     n /= bs;
    372:     PetscCall(VecSetSizes(coordinatesNew, n * cdim, PETSC_DETERMINE));
    373:     PetscCall(VecSetUp(coordinatesNew));
    374:     PetscCall(PetscMalloc1(n * bs, &indices));
    375:     for (PetscInt i = 0; i < n; ++i)
    376:       for (PetscInt b = 0; b < bs; ++b) indices[i * bs + b] = i * cdim + b;
    377:     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, n * bs, indices, PETSC_OWN_POINTER, &idx));
    378:     PetscCall(VecISCopy(coordinatesNew, idx, SCATTER_FORWARD, coordinates));
    379:     PetscCall(ISDestroy(&idx));
    380:     PetscCall(DMSetCoordinatesLocal(dm, coordinatesNew));
    381:     PetscCall(VecDestroy(&coordinatesNew));
    382:     PetscCall(DMSetCoordinateDim(dm, cdim));
    383:     {
    384:       PetscInt gn;

    386:       PetscCall(DMGetCoordinates(dm, &coordinatesNew));
    387:       PetscCall(VecGetLocalSize(coordinatesNew, &gn));
    388:       PetscCheck(gn == n * cdim, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Global coordinate size %" PetscInt_FMT " != %" PetscInt_FMT "local coordinate size", gn, n * cdim);
    389:     }
    390:     dE      = cdim;
    391:     project = PETSC_FALSE;
    392:   }
    393:   if (degree >= 0) {
    394:     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
    395:     PetscInt       cStart, cEnd, gct;

    397:     PetscCall(DMPlexGetHeightStratum(dm, height, &cStart, &cEnd));
    398:     if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
    399:     gct = (PetscInt)ct;
    400:     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &gct, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
    401:     ct = (DMPolytopeType)gct;
    402:     // Work around current bug in PetscDualSpaceSetUp_Lagrange()
    403:     //   Can be seen in plex_tutorials-ex10_1
    404:     if (ct != DM_POLYTOPE_SEG_PRISM_TENSOR && ct != DM_POLYTOPE_TRI_PRISM_TENSOR && ct != DM_POLYTOPE_QUAD_PRISM_TENSOR) {
    405:       PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
    406:       if (localized) {
    407:         PetscFE dgfe = NULL;

    409:         PetscCall(PetscFECreateBrokenElement(fe, &dgfe));
    410:         PetscCall(PetscFEDestroy(&fe));
    411:         fe = dgfe;
    412:       }
    413:     }
    414:   }
    415:   PetscCall(DMSetCoordinateDisc(dm, fe, localized, project));
    416:   PetscCall(PetscFEDestroy(&fe));
    417:   PetscFunctionReturn(PETSC_SUCCESS);
    418: }

    420: /*@
    421:   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.

    423:   Collective

    425:   Input Parameters:
    426: + comm            - The communicator for the `DM` object
    427: . dim             - The spatial dimension
    428: . simplex         - Flag for simplicial cells, otherwise they are tensor product cells
    429: . interpolate     - Flag to create intermediate mesh pieces (edges, faces)
    430: - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell

    432:   Output Parameter:
    433: . newdm - The `DM` object

    435:   Level: beginner

    437: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
    438: @*/
    439: PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
    440: {
    441:   DM          dm;
    442:   PetscMPIInt rank;

    444:   PetscFunctionBegin;
    445:   PetscCall(DMCreate(comm, &dm));
    446:   PetscCall(DMSetType(dm, DMPLEX));
    447:   PetscCall(DMSetDimension(dm, dim));
    448:   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
    449:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
    450:   switch (dim) {
    451:   case 2:
    452:     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
    453:     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
    454:     break;
    455:   case 3:
    456:     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
    457:     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
    458:     break;
    459:   default:
    460:     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
    461:   }
    462:   if (rank) {
    463:     PetscInt numPoints[2] = {0, 0};
    464:     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
    465:   } else {
    466:     switch (dim) {
    467:     case 2:
    468:       if (simplex) {
    469:         PetscInt    numPoints[2]        = {4, 2};
    470:         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
    471:         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
    472:         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
    473:         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};

    475:         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    476:       } else {
    477:         PetscInt    numPoints[2]        = {6, 2};
    478:         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
    479:         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
    480:         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
    481:         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};

    483:         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    484:       }
    485:       break;
    486:     case 3:
    487:       if (simplex) {
    488:         PetscInt    numPoints[2]        = {5, 2};
    489:         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
    490:         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
    491:         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
    492:         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};

    494:         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    495:       } else {
    496:         PetscInt    numPoints[2]         = {12, 2};
    497:         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
    498:         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
    499:         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
    500:         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};

    502:         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    503:       }
    504:       break;
    505:     default:
    506:       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
    507:     }
    508:   }
    509:   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
    510:   *newdm = dm;
    511:   if (refinementLimit > 0.0) {
    512:     DM          rdm;
    513:     const char *name;

    515:     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
    516:     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
    517:     PetscCall(DMRefine(*newdm, comm, &rdm));
    518:     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
    519:     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
    520:     PetscCall(DMDestroy(newdm));
    521:     *newdm = rdm;
    522:   }
    523:   if (interpolate) {
    524:     DM idm;

    526:     PetscCall(DMPlexInterpolate(*newdm, &idm));
    527:     PetscCall(DMDestroy(newdm));
    528:     *newdm = idm;
    529:   }
    530:   PetscFunctionReturn(PETSC_SUCCESS);
    531: }

    533: static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
    534: {
    535:   const PetscInt numVertices    = 2;
    536:   PetscInt       markerRight    = 1;
    537:   PetscInt       markerLeft     = 1;
    538:   PetscBool      markerSeparate = PETSC_FALSE;
    539:   Vec            coordinates;
    540:   PetscSection   coordSection;
    541:   PetscScalar   *coords;
    542:   PetscInt       coordSize;
    543:   PetscMPIInt    rank;
    544:   PetscInt       cdim = 1, v;

    546:   PetscFunctionBegin;
    547:   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
    548:   if (markerSeparate) {
    549:     markerRight = 2;
    550:     markerLeft  = 1;
    551:   }
    552:   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
    553:   if (rank == 0) {
    554:     PetscCall(DMPlexSetChart(dm, 0, numVertices));
    555:     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
    556:     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
    557:     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
    558:   }
    559:   PetscCall(DMPlexSymmetrize(dm));
    560:   PetscCall(DMPlexStratify(dm));
    561:   /* Build coordinates */
    562:   PetscCall(DMSetCoordinateDim(dm, cdim));
    563:   PetscCall(DMGetCoordinateSection(dm, &coordSection));
    564:   PetscCall(PetscSectionSetNumFields(coordSection, 1));
    565:   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
    566:   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
    567:   for (v = 0; v < numVertices; ++v) {
    568:     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
    569:     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
    570:   }
    571:   PetscCall(PetscSectionSetUp(coordSection));
    572:   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
    573:   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
    574:   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    575:   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
    576:   PetscCall(VecSetBlockSize(coordinates, cdim));
    577:   PetscCall(VecSetType(coordinates, VECSTANDARD));
    578:   PetscCall(VecGetArray(coordinates, &coords));
    579:   coords[0] = lower[0];
    580:   coords[1] = upper[0];
    581:   PetscCall(VecRestoreArray(coordinates, &coords));
    582:   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
    583:   PetscCall(VecDestroy(&coordinates));
    584:   PetscFunctionReturn(PETSC_SUCCESS);
    585: }

    587: static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
    588: {
    589:   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
    590:   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
    591:   PetscInt       markerTop      = 1;
    592:   PetscInt       markerBottom   = 1;
    593:   PetscInt       markerRight    = 1;
    594:   PetscInt       markerLeft     = 1;
    595:   PetscBool      markerSeparate = PETSC_FALSE;
    596:   Vec            coordinates;
    597:   PetscSection   coordSection;
    598:   PetscScalar   *coords;
    599:   PetscInt       coordSize;
    600:   PetscMPIInt    rank;
    601:   PetscInt       v, vx, vy;

    603:   PetscFunctionBegin;
    604:   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
    605:   if (markerSeparate) {
    606:     markerTop    = 3;
    607:     markerBottom = 1;
    608:     markerRight  = 2;
    609:     markerLeft   = 4;
    610:   }
    611:   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
    612:   if (rank == 0) {
    613:     PetscInt e, ex, ey;

    615:     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
    616:     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
    617:     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
    618:     for (vx = 0; vx <= edges[0]; vx++) {
    619:       for (ey = 0; ey < edges[1]; ey++) {
    620:         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
    621:         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
    622:         PetscInt cone[2];

    624:         cone[0] = vertex;
    625:         cone[1] = vertex + edges[0] + 1;
    626:         PetscCall(DMPlexSetCone(dm, edge, cone));
    627:         if (vx == edges[0]) {
    628:           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
    629:           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
    630:           if (ey == edges[1] - 1) {
    631:             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
    632:             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
    633:           }
    634:         } else if (vx == 0) {
    635:           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
    636:           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
    637:           if (ey == edges[1] - 1) {
    638:             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
    639:             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
    640:           }
    641:         }
    642:       }
    643:     }
    644:     for (vy = 0; vy <= edges[1]; vy++) {
    645:       for (ex = 0; ex < edges[0]; ex++) {
    646:         PetscInt edge   = vy * edges[0] + ex;
    647:         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
    648:         PetscInt cone[2];

    650:         cone[0] = vertex;
    651:         cone[1] = vertex + 1;
    652:         PetscCall(DMPlexSetCone(dm, edge, cone));
    653:         if (vy == edges[1]) {
    654:           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
    655:           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
    656:           if (ex == edges[0] - 1) {
    657:             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
    658:             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
    659:           }
    660:         } else if (vy == 0) {
    661:           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
    662:           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
    663:           if (ex == edges[0] - 1) {
    664:             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
    665:             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
    666:           }
    667:         }
    668:       }
    669:     }
    670:   }
    671:   PetscCall(DMPlexSymmetrize(dm));
    672:   PetscCall(DMPlexStratify(dm));
    673:   /* Build coordinates */
    674:   PetscCall(DMSetCoordinateDim(dm, 2));
    675:   PetscCall(DMGetCoordinateSection(dm, &coordSection));
    676:   PetscCall(PetscSectionSetNumFields(coordSection, 1));
    677:   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
    678:   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
    679:   for (v = numEdges; v < numEdges + numVertices; ++v) {
    680:     PetscCall(PetscSectionSetDof(coordSection, v, 2));
    681:     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
    682:   }
    683:   PetscCall(PetscSectionSetUp(coordSection));
    684:   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
    685:   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
    686:   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    687:   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
    688:   PetscCall(VecSetBlockSize(coordinates, 2));
    689:   PetscCall(VecSetType(coordinates, VECSTANDARD));
    690:   PetscCall(VecGetArray(coordinates, &coords));
    691:   for (vy = 0; vy <= edges[1]; ++vy) {
    692:     for (vx = 0; vx <= edges[0]; ++vx) {
    693:       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
    694:       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
    695:     }
    696:   }
    697:   PetscCall(VecRestoreArray(coordinates, &coords));
    698:   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
    699:   PetscCall(VecDestroy(&coordinates));
    700:   PetscFunctionReturn(PETSC_SUCCESS);
    701: }

    703: static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
    704: {
    705:   PetscInt     vertices[3], numVertices;
    706:   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
    707:   PetscInt     markerTop      = 1;
    708:   PetscInt     markerBottom   = 1;
    709:   PetscInt     markerFront    = 1;
    710:   PetscInt     markerBack     = 1;
    711:   PetscInt     markerRight    = 1;
    712:   PetscInt     markerLeft     = 1;
    713:   PetscBool    markerSeparate = PETSC_FALSE;
    714:   Vec          coordinates;
    715:   PetscSection coordSection;
    716:   PetscScalar *coords;
    717:   PetscInt     coordSize;
    718:   PetscMPIInt  rank;
    719:   PetscInt     v, vx, vy, vz;
    720:   PetscInt     voffset, iface = 0, cone[4];

    722:   PetscFunctionBegin;
    723:   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
    724:   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
    725:   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
    726:   if (markerSeparate) {
    727:     markerBottom = 1;
    728:     markerTop    = 2;
    729:     markerFront  = 3;
    730:     markerBack   = 4;
    731:     markerRight  = 5;
    732:     markerLeft   = 6;
    733:   }
    734:   vertices[0] = faces[0] + 1;
    735:   vertices[1] = faces[1] + 1;
    736:   vertices[2] = faces[2] + 1;
    737:   numVertices = vertices[0] * vertices[1] * vertices[2];
    738:   if (rank == 0) {
    739:     PetscInt f;

    741:     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
    742:     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
    743:     PetscCall(DMSetUp(dm)); /* Allocate space for cones */

    745:     /* Side 0 (Top) */
    746:     for (vy = 0; vy < faces[1]; vy++) {
    747:       for (vx = 0; vx < faces[0]; vx++) {
    748:         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
    749:         cone[0] = voffset;
    750:         cone[1] = voffset + 1;
    751:         cone[2] = voffset + vertices[0] + 1;
    752:         cone[3] = voffset + vertices[0];
    753:         PetscCall(DMPlexSetCone(dm, iface, cone));
    754:         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
    755:         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
    756:         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
    757:         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
    758:         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
    759:         iface++;
    760:       }
    761:     }

    763:     /* Side 1 (Bottom) */
    764:     for (vy = 0; vy < faces[1]; vy++) {
    765:       for (vx = 0; vx < faces[0]; vx++) {
    766:         voffset = numFaces + vy * (faces[0] + 1) + vx;
    767:         cone[0] = voffset + 1;
    768:         cone[1] = voffset;
    769:         cone[2] = voffset + vertices[0];
    770:         cone[3] = voffset + vertices[0] + 1;
    771:         PetscCall(DMPlexSetCone(dm, iface, cone));
    772:         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
    773:         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
    774:         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
    775:         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
    776:         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
    777:         iface++;
    778:       }
    779:     }

    781:     /* Side 2 (Front) */
    782:     for (vz = 0; vz < faces[2]; vz++) {
    783:       for (vx = 0; vx < faces[0]; vx++) {
    784:         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
    785:         cone[0] = voffset;
    786:         cone[1] = voffset + 1;
    787:         cone[2] = voffset + vertices[0] * vertices[1] + 1;
    788:         cone[3] = voffset + vertices[0] * vertices[1];
    789:         PetscCall(DMPlexSetCone(dm, iface, cone));
    790:         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
    791:         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
    792:         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
    793:         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
    794:         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
    795:         iface++;
    796:       }
    797:     }

    799:     /* Side 3 (Back) */
    800:     for (vz = 0; vz < faces[2]; vz++) {
    801:       for (vx = 0; vx < faces[0]; vx++) {
    802:         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
    803:         cone[0] = voffset + vertices[0] * vertices[1];
    804:         cone[1] = voffset + vertices[0] * vertices[1] + 1;
    805:         cone[2] = voffset + 1;
    806:         cone[3] = voffset;
    807:         PetscCall(DMPlexSetCone(dm, iface, cone));
    808:         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
    809:         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
    810:         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
    811:         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
    812:         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
    813:         iface++;
    814:       }
    815:     }

    817:     /* Side 4 (Left) */
    818:     for (vz = 0; vz < faces[2]; vz++) {
    819:       for (vy = 0; vy < faces[1]; vy++) {
    820:         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
    821:         cone[0] = voffset;
    822:         cone[1] = voffset + vertices[0] * vertices[1];
    823:         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
    824:         cone[3] = voffset + vertices[0];
    825:         PetscCall(DMPlexSetCone(dm, iface, cone));
    826:         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
    827:         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
    828:         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
    829:         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
    830:         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
    831:         iface++;
    832:       }
    833:     }

    835:     /* Side 5 (Right) */
    836:     for (vz = 0; vz < faces[2]; vz++) {
    837:       for (vy = 0; vy < faces[1]; vy++) {
    838:         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
    839:         cone[0] = voffset + vertices[0] * vertices[1];
    840:         cone[1] = voffset;
    841:         cone[2] = voffset + vertices[0];
    842:         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
    843:         PetscCall(DMPlexSetCone(dm, iface, cone));
    844:         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
    845:         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
    846:         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
    847:         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
    848:         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
    849:         iface++;
    850:       }
    851:     }
    852:   }
    853:   PetscCall(DMPlexSymmetrize(dm));
    854:   PetscCall(DMPlexStratify(dm));
    855:   /* Build coordinates */
    856:   PetscCall(DMSetCoordinateDim(dm, 3));
    857:   PetscCall(DMGetCoordinateSection(dm, &coordSection));
    858:   PetscCall(PetscSectionSetNumFields(coordSection, 1));
    859:   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
    860:   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
    861:   for (v = numFaces; v < numFaces + numVertices; ++v) {
    862:     PetscCall(PetscSectionSetDof(coordSection, v, 3));
    863:     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
    864:   }
    865:   PetscCall(PetscSectionSetUp(coordSection));
    866:   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
    867:   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
    868:   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    869:   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
    870:   PetscCall(VecSetBlockSize(coordinates, 3));
    871:   PetscCall(VecSetType(coordinates, VECSTANDARD));
    872:   PetscCall(VecGetArray(coordinates, &coords));
    873:   for (vz = 0; vz <= faces[2]; ++vz) {
    874:     for (vy = 0; vy <= faces[1]; ++vy) {
    875:       for (vx = 0; vx <= faces[0]; ++vx) {
    876:         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
    877:         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
    878:         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
    879:       }
    880:     }
    881:   }
    882:   PetscCall(VecRestoreArray(coordinates, &coords));
    883:   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
    884:   PetscCall(VecDestroy(&coordinates));
    885:   PetscFunctionReturn(PETSC_SUCCESS);
    886: }

    888: static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
    889: {
    890:   PetscFunctionBegin;
    892:   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
    893:   PetscCall(DMSetDimension(dm, dim - 1));
    894:   PetscCall(DMSetCoordinateDim(dm, dim));
    895:   switch (dim) {
    896:   case 1:
    897:     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
    898:     break;
    899:   case 2:
    900:     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
    901:     break;
    902:   case 3:
    903:     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
    904:     break;
    905:   default:
    906:     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
    907:   }
    908:   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
    909:   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
    910:   PetscFunctionReturn(PETSC_SUCCESS);
    911: }

    913: /*@C
    914:   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).

    916:   Collective

    918:   Input Parameters:
    919: + comm        - The communicator for the `DM` object
    920: . dim         - The spatial dimension of the box, so the resulting mesh is has dimension `dim`-1
    921: . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
    922: . lower       - The lower left corner, or `NULL` for (0, 0, 0)
    923: . upper       - The upper right corner, or `NULL` for (1, 1, 1)
    924: - interpolate - Flag to create intermediate mesh pieces (edges, faces)

    926:   Output Parameter:
    927: . dm - The `DM` object

    929:   Level: beginner

    931: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
    932: @*/
    933: PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
    934: {
    935:   PetscInt  fac[3] = {1, 1, 1};
    936:   PetscReal low[3] = {0, 0, 0};
    937:   PetscReal upp[3] = {1, 1, 1};

    939:   PetscFunctionBegin;
    940:   PetscCall(DMCreate(comm, dm));
    941:   PetscCall(DMSetType(*dm, DMPLEX));
    942:   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
    943:   PetscFunctionReturn(PETSC_SUCCESS);
    944: }

    946: static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
    947: {
    948:   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
    949:   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
    950:   PetscScalar *vertexCoords;
    951:   PetscReal    L, maxCell;
    952:   PetscBool    markerSeparate = PETSC_FALSE;
    953:   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
    954:   PetscInt     markerRight = 1, faceMarkerRight = 2;
    955:   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
    956:   PetscMPIInt  rank;

    958:   PetscFunctionBegin;
    959:   PetscAssertPointer(dm, 1);

    961:   PetscCall(DMSetDimension(dm, 1));
    962:   PetscCall(DMCreateLabel(dm, "marker"));
    963:   PetscCall(DMCreateLabel(dm, "Face Sets"));

    965:   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
    966:   if (rank == 0) numCells = segments;
    967:   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);

    969:   numPoints[0] = numVerts;
    970:   numPoints[1] = numCells;
    971:   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
    972:   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
    973:   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
    974:   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
    975:   for (i = 0; i < numCells; ++i) {
    976:     cones[2 * i]     = numCells + i % numVerts;
    977:     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
    978:   }
    979:   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
    980:   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    981:   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));

    983:   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
    984:   if (markerSeparate) {
    985:     markerLeft  = faceMarkerLeft;
    986:     markerRight = faceMarkerRight;
    987:   }
    988:   if (!wrap && rank == 0) {
    989:     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
    990:     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
    991:     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
    992:     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
    993:     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
    994:   }
    995:   if (wrap) {
    996:     L       = upper - lower;
    997:     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
    998:     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
    999:   }
    1000:   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
    1001:   PetscFunctionReturn(PETSC_SUCCESS);
    1002: }

    1004: // Creates "Face Sets" label based on the standard box labeling conventions
    1005: static PetscErrorCode DMPlexSetBoxLabel_Internal(DM dm, const DMBoundaryType periodicity[])
    1006: {
    1007:   DM              cdm;
    1008:   PetscSection    csection;
    1009:   Vec             coordinates;
    1010:   DMLabel         label;
    1011:   IS              faces_is;
    1012:   PetscInt        dim, num_face = 0;
    1013:   const PetscInt *faces;
    1014:   PetscInt        faceMarkerBottom, faceMarkerTop, faceMarkerFront, faceMarkerBack, faceMarkerRight, faceMarkerLeft;

    1016:   PetscFunctionBeginUser;
    1017:   PetscCall(DMGetDimension(dm, &dim));
    1018:   PetscCheck((dim == 2) || (dim == 3), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "DMPlex box labeling only supports 2D and 3D meshes, received DM of dimension %" PetscInt_FMT, dim);
    1019:   // Get Face Sets label
    1020:   PetscCall(DMGetLabel(dm, "Face Sets", &label));
    1021:   if (label) {
    1022:     PetscCall(DMLabelReset(label));
    1023:   } else {
    1024:     PetscCall(DMCreateLabel(dm, "Face Sets"));
    1025:     PetscCall(DMGetLabel(dm, "Face Sets", &label));
    1026:   }
    1027:   PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
    1028:   PetscCall(DMGetStratumIS(dm, "Face Sets", 1, &faces_is));

    1030:   switch (dim) {
    1031:   case 2:
    1032:     faceMarkerTop    = 3;
    1033:     faceMarkerBottom = 1;
    1034:     faceMarkerRight  = 2;
    1035:     faceMarkerLeft   = 4;
    1036:     break;
    1037:   case 3:
    1038:     faceMarkerBottom = 1;
    1039:     faceMarkerTop    = 2;
    1040:     faceMarkerFront  = 3;
    1041:     faceMarkerBack   = 4;
    1042:     faceMarkerRight  = 5;
    1043:     faceMarkerLeft   = 6;
    1044:     break;
    1045:   default:
    1046:     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
    1047:   }

    1049:   if (faces_is) PetscCall(ISGetLocalSize(faces_is, &num_face));
    1050:   if (faces_is) PetscCall(ISGetIndices(faces_is, &faces));
    1051:   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
    1052:   PetscCall(DMGetCoordinateDM(dm, &cdm));
    1053:   PetscCall(DMGetLocalSection(cdm, &csection));
    1054:   for (PetscInt f = 0; f < num_face; ++f) {
    1055:     PetscScalar *coords = NULL;
    1056:     PetscInt     face = faces[f], flip = 1, label_value = -1, coords_size;

    1058:     { // Determine if orientation of face is flipped
    1059:       PetscInt        num_cells_support, num_faces, start = -1;
    1060:       const PetscInt *orients, *cell_faces, *cells;

    1062:       PetscCall(DMPlexGetSupport(dm, face, &cells));
    1063:       PetscCall(DMPlexGetSupportSize(dm, face, &num_cells_support));
    1064:       PetscCheck(num_cells_support == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Expected one cell in support of exterior face, but got %" PetscInt_FMT " cells", num_cells_support);
    1065:       PetscCall(DMPlexGetCone(dm, cells[0], &cell_faces));
    1066:       PetscCall(DMPlexGetConeSize(dm, cells[0], &num_faces));
    1067:       for (PetscInt i = 0; i < num_faces; i++) {
    1068:         if (cell_faces[i] == face) start = i;
    1069:       }
    1070:       PetscCheck(start >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Could not find face %" PetscInt_FMT " in cone of its support", face);
    1071:       PetscCall(DMPlexGetConeOrientation(dm, cells[0], &orients));
    1072:       if (orients[start] < 0) flip = -1;
    1073:     }

    1075:     // Cannot use DMPlexComputeCellGeometryFVM() for high-order geometry, so must calculate normal vectors manually
    1076:     // Use the vertices (depth 0) of coordinate DM to calculate normal vector
    1077:     PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
    1078:     switch (dim) {
    1079:     case 2: {
    1080:       PetscScalar vec[2];

    1082:       for (PetscInt d = 0; d < dim; ++d) vec[d] = flip * (PetscRealPart(coords[1 * dim + d]) - PetscRealPart(coords[0 * dim + d]));
    1083:       PetscScalar normal[] = {vec[1], -vec[0]};
    1084:       if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[1])) {
    1085:         label_value = PetscRealPart(normal[0]) > 0 ? faceMarkerRight : faceMarkerLeft;
    1086:       } else {
    1087:         label_value = PetscRealPart(normal[1]) > 0 ? faceMarkerTop : faceMarkerBottom;
    1088:       }
    1089:     } break;
    1090:     case 3: {
    1091:       PetscScalar vec1[3], vec2[3], normal[3];

    1093:       for (PetscInt d = 0; d < dim; ++d) {
    1094:         vec1[d] = PetscRealPart(coords[1 * dim + d]) - PetscRealPart(coords[0 * dim + d]);
    1095:         vec2[d] = PetscRealPart(coords[2 * dim + d]) - PetscRealPart(coords[1 * dim + d]);
    1096:       }

    1098:       // Calculate normal vector via cross-product
    1099:       normal[0] = flip * ((vec1[1] * vec2[2]) - (vec1[2] * vec2[1]));
    1100:       normal[1] = flip * ((vec1[2] * vec2[0]) - (vec1[0] * vec2[2]));
    1101:       normal[2] = flip * ((vec1[0] * vec2[1]) - (vec1[1] * vec2[0]));

    1103:       if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[1])) {
    1104:         if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[2])) {
    1105:           label_value = PetscRealPart(normal[0]) > 0 ? faceMarkerRight : faceMarkerLeft;
    1106:         } else {
    1107:           label_value = PetscRealPart(normal[2]) > 0 ? faceMarkerTop : faceMarkerBottom;
    1108:         }
    1109:       } else {
    1110:         if (PetscAbsScalar(normal[1]) > PetscAbsScalar(normal[2])) {
    1111:           label_value = PetscRealPart(normal[1]) > 0 ? faceMarkerBack : faceMarkerFront;
    1112:         } else {
    1113:           label_value = PetscRealPart(normal[2]) > 0 ? faceMarkerTop : faceMarkerBottom;
    1114:         }
    1115:       }
    1116:     } break;
    1117:     }

    1119:     PetscInt previous_label_value; // always 1 due to DMPlexMarkBoundaryFaces call above
    1120:     PetscCall(DMGetLabelValue(dm, "Face Sets", face, &previous_label_value));
    1121:     PetscCall(DMClearLabelValue(dm, "Face Sets", face, previous_label_value));
    1122:     PetscCall(DMSetLabelValue(dm, "Face Sets", face, label_value));
    1123:     PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
    1124:   }
    1125:   if (faces_is) PetscCall(ISRestoreIndices(faces_is, &faces));
    1126:   PetscCall(ISDestroy(&faces_is));

    1128:   // Create Isoperiodic SF from newly-created face labels
    1129:   PetscSF     periodicsfs[3];
    1130:   PetscInt    periodic_sf_index  = 0;
    1131:   PetscScalar transform[3][4][4] = {{{0.}}};
    1132:   for (PetscInt d = 0; d < dim; d++) {
    1133:     IS              donor_is, periodic_is;
    1134:     const PetscInt *donor_faces = NULL, *periodic_faces = NULL;
    1135:     PetscInt        num_donor = 0, num_periodic = 0;
    1136:     PetscSF         centroidsf;
    1137:     PetscReal       donor_to_periodic_distance;
    1138:     const PetscInt  face_pairings[2][3][2] = {
    1139:       // 2D face pairings, {donor, periodic}
    1140:       {{4, 2}, {1, 3}},
    1141:       // 3D face pairings
    1142:       {{5, 6}, {3, 4}, {1, 2}}
    1143:     };

    1145:     if (periodicity[d] != DM_BOUNDARY_PERIODIC) continue;
    1146:     {
    1147:       // Compute centroidsf, which is the mapping from donor faces to periodic faces
    1148:       // Matches the centroid of the faces together, ignoring the periodic direction component (which should not match between donor and periodic face)
    1149:       PetscInt     coords_size, centroid_comps = dim - 1;
    1150:       PetscScalar *coords = NULL;
    1151:       PetscReal   *donor_centroids, *periodic_centroids;
    1152:       PetscReal    loc_periodic[2] = {PETSC_MIN_REAL, PETSC_MIN_REAL}, loc_periodic_global[2]; // Location of donor (0) and periodic (1) faces in periodic direction

    1154:       PetscCall(DMGetStratumIS(dm, "Face Sets", face_pairings[dim - 2][d][0], &donor_is));
    1155:       PetscCall(DMGetStratumIS(dm, "Face Sets", face_pairings[dim - 2][d][1], &periodic_is));
    1156:       if (donor_is) {
    1157:         PetscCall(ISGetLocalSize(donor_is, &num_donor));
    1158:         PetscCall(ISGetIndices(donor_is, &donor_faces));
    1159:       }
    1160:       if (periodic_is) {
    1161:         PetscCall(ISGetLocalSize(periodic_is, &num_periodic));
    1162:         PetscCall(ISGetIndices(periodic_is, &periodic_faces));
    1163:       }
    1164:       PetscCall(PetscCalloc2(num_donor * centroid_comps, &donor_centroids, num_periodic * centroid_comps, &periodic_centroids));
    1165:       for (PetscInt f = 0; f < num_donor; f++) {
    1166:         PetscInt face = donor_faces[f], num_coords;
    1167:         PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
    1168:         num_coords = coords_size / dim;
    1169:         for (PetscInt c = 0; c < num_coords; c++) {
    1170:           PetscInt comp_index = 0;
    1171:           loc_periodic[0]     = PetscRealPart(coords[c * dim + d]);
    1172:           for (PetscInt i = 0; i < dim; i++) {
    1173:             if (i == d) continue; // Periodic direction not used for centroid calculation
    1174:             donor_centroids[f * centroid_comps + comp_index] += PetscRealPart(coords[c * dim + i]) / num_coords;
    1175:             comp_index++;
    1176:           }
    1177:         }
    1178:         PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
    1179:       }

    1181:       for (PetscInt f = 0; f < num_periodic; f++) {
    1182:         PetscInt face = periodic_faces[f], num_coords;
    1183:         PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
    1184:         num_coords = coords_size / dim;
    1185:         for (PetscInt c = 0; c < num_coords; c++) {
    1186:           PetscInt comp_index = 0;
    1187:           loc_periodic[1]     = PetscRealPart(coords[c * dim + d]);
    1188:           for (PetscInt i = 0; i < dim; i++) {
    1189:             if (i == d) continue; // Periodic direction not used for centroid calculation
    1190:             periodic_centroids[f * centroid_comps + comp_index] += PetscRealPart(coords[c * dim + i]) / num_coords;
    1191:             comp_index++;
    1192:           }
    1193:         }
    1194:         PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
    1195:       }
    1196:       PetscCallMPI(MPIU_Allreduce(loc_periodic, loc_periodic_global, 2, MPIU_REAL, MPIU_MAX, PetscObjectComm((PetscObject)dm)));
    1197:       donor_to_periodic_distance = loc_periodic_global[1] - loc_periodic_global[0];

    1199:       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)dm), &centroidsf));
    1200:       PetscCall(PetscSFSetGraphFromCoordinates(centroidsf, num_donor, num_periodic, centroid_comps, 1e-10, donor_centroids, periodic_centroids));
    1201:       PetscCall(PetscSFViewFromOptions(centroidsf, NULL, "-dm_plex_box_label_centroid_sf_view"));
    1202:       PetscCall(PetscFree2(donor_centroids, periodic_centroids));
    1203:     }

    1205:     { // Create Isoperiodic SF using centroidsSF
    1206:       PetscInt           pStart, pEnd;
    1207:       PetscInt          *leaf_faces;
    1208:       const PetscSFNode *firemote;
    1209:       PetscSFNode       *isoperiodic_leaves;

    1211:       PetscCall(PetscMalloc1(num_periodic, &leaf_faces));
    1212:       PetscCall(PetscSFBcastBegin(centroidsf, MPIU_INT, donor_faces, leaf_faces, MPI_REPLACE));
    1213:       PetscCall(PetscSFBcastEnd(centroidsf, MPIU_INT, donor_faces, leaf_faces, MPI_REPLACE));

    1215:       PetscCall(PetscMalloc1(num_periodic, &isoperiodic_leaves));
    1216:       PetscCall(PetscSFGetGraph(centroidsf, NULL, NULL, NULL, &firemote));
    1217:       for (PetscInt l = 0; l < num_periodic; ++l) {
    1218:         isoperiodic_leaves[l].index = leaf_faces[l];
    1219:         isoperiodic_leaves[l].rank  = firemote[l].rank;
    1220:       }

    1222:       PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
    1223:       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)dm), &periodicsfs[periodic_sf_index]));
    1224:       PetscCall(PetscSFSetGraph(periodicsfs[periodic_sf_index], pEnd - pStart, num_periodic, (PetscInt *)periodic_faces, PETSC_COPY_VALUES, isoperiodic_leaves, PETSC_OWN_POINTER));
    1225:       PetscCall(PetscSFViewFromOptions(periodicsfs[periodic_sf_index], NULL, "-dm_plex_box_label_periodic_sf_view"));
    1226:       PetscCall(PetscFree(leaf_faces));
    1227:     }

    1229:     transform[periodic_sf_index][0][0] = 1;
    1230:     transform[periodic_sf_index][1][1] = 1;
    1231:     transform[periodic_sf_index][2][2] = 1;
    1232:     transform[periodic_sf_index][3][3] = 1;
    1233:     transform[periodic_sf_index][d][3] = donor_to_periodic_distance;

    1235:     periodic_sf_index++;
    1236:     PetscCall(PetscSFDestroy(&centroidsf));
    1237:     if (donor_is) {
    1238:       PetscCall(ISRestoreIndices(donor_is, &donor_faces));
    1239:       PetscCall(ISDestroy(&donor_is));
    1240:     }
    1241:     if (periodic_is) {
    1242:       PetscCall(ISRestoreIndices(periodic_is, &periodic_faces));
    1243:       PetscCall(ISDestroy(&periodic_is));
    1244:     }
    1245:     PetscCall(DMClearLabelStratum(dm, "Face Sets", face_pairings[dim - 2][d][0]));
    1246:     PetscCall(DMClearLabelStratum(dm, "Face Sets", face_pairings[dim - 2][d][1]));
    1247:   }
    1248:   PetscCall(DMPlexSetIsoperiodicFaceSF(dm, periodic_sf_index, periodicsfs));
    1249:   PetscCall(DMPlexSetIsoperiodicFaceTransform(dm, periodic_sf_index, (const PetscScalar *)transform));
    1250:   for (PetscInt p = 0; p < periodic_sf_index; p++) PetscCall(PetscSFDestroy(&periodicsfs[p]));

    1252:   { // Update coordinate DM with new Face Sets label
    1253:     DM      cdm;
    1254:     DMLabel oldFaceSets, newFaceSets;
    1255:     PetscCall(DMGetCoordinateDM(dm, &cdm));
    1256:     PetscCall(DMGetLabel(cdm, "Face Sets", &oldFaceSets));
    1257:     if (oldFaceSets) PetscCall(DMRemoveLabelBySelf(cdm, &oldFaceSets, PETSC_FALSE));
    1258:     PetscCall(DMLabelDuplicate(label, &newFaceSets));
    1259:     PetscCall(DMAddLabel(cdm, newFaceSets));
    1260:     PetscCall(DMLabelDestroy(&newFaceSets));
    1261:   }
    1262:   PetscFunctionReturn(PETSC_SUCCESS);
    1263: }

    1265: static PetscErrorCode DMPlexCreateSquareMesh_Simplex_CrissCross(DM dm, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType bd[])
    1266: {
    1267:   PetscInt       markerTop = 1, faceMarkerTop = 3;
    1268:   PetscInt       markerBottom = 1, faceMarkerBottom = 1;
    1269:   PetscInt       markerRight = 1, faceMarkerRight = 2;
    1270:   PetscInt       markerLeft = 1, faceMarkerLeft = 4;
    1271:   PetscBool      markerSeparate = PETSC_FALSE;
    1272:   DMBoundaryType bdX = bd[0], bdY = bd[1];
    1273:   PetscMPIInt    rank;

    1275:   PetscFunctionBegin;
    1276:   PetscCheck(bdX == DM_BOUNDARY_NONE || bdX == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Not implemented for boundary type %s", DMBoundaryTypes[bdX]);
    1277:   PetscCheck(bdY == DM_BOUNDARY_NONE || bdY == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Not implemented for boundary type %s", DMBoundaryTypes[bdY]);
    1278:   PetscCall(DMSetDimension(dm, 2));
    1279:   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
    1280:   PetscCall(DMCreateLabel(dm, "marker"));
    1281:   PetscCall(DMCreateLabel(dm, "Face Sets"));
    1282:   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
    1283:   if (markerSeparate) {
    1284:     markerBottom = faceMarkerBottom;
    1285:     markerTop    = faceMarkerTop;
    1286:     markerRight  = faceMarkerRight;
    1287:     markerLeft   = faceMarkerLeft;
    1288:   }
    1289:   {
    1290:     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
    1291:     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
    1292:     const PetscInt numZEdges    = rank == 0 ? 4 * edges[0] * edges[1] : 0; /* Z-edges are the 4 internal edges per cell */
    1293:     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC ? edges[0] : edges[0] + 1) : 0;
    1294:     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC ? edges[1] : edges[1] + 1) : 0;
    1295:     const PetscInt numZVertices = rank == 0 ? edges[0] * edges[1] : 0;
    1296:     const PetscInt numCells     = 4 * numXEdges * numYEdges;
    1297:     const PetscInt numTotXEdges = numXEdges * numYVertices;
    1298:     const PetscInt numTotYEdges = numYEdges * numXVertices;
    1299:     const PetscInt numVertices  = numXVertices * numYVertices + numZVertices;
    1300:     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numZEdges;
    1301:     const PetscInt firstVertex  = numCells;
    1302:     const PetscInt firstXEdge   = numCells + numVertices;
    1303:     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
    1304:     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
    1305:     Vec            coordinates;
    1306:     PetscSection   coordSection;
    1307:     PetscScalar   *coords;
    1308:     PetscInt       coordSize;
    1309:     PetscInt       v, vx, vy;
    1310:     PetscInt       c, e, ex, ey;

    1312:     PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVertices));
    1313:     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 3));
    1314:     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
    1315:     PetscCall(DMSetUp(dm));

    1317:     /* Build cells and Z-edges */
    1318:     for (ey = 0; ey < numYEdges; ++ey) {
    1319:       for (ex = 0; ex < numXEdges; ++ex) {
    1320:         const PetscInt exp1 = (ex + 1) % numXVertices;
    1321:         const PetscInt eyp1 = (ey + 1) % numYVertices;
    1322:         const PetscInt ez   = firstZEdge + 4 * (ey * numXEdges + ex);
    1323:         const PetscInt vc   = firstVertex + numXVertices * numYVertices + ey * numXEdges + ex;
    1324:         const PetscInt v0   = firstVertex + ey * numXVertices + ex;
    1325:         const PetscInt v1   = firstVertex + ey * numXVertices + exp1;
    1326:         const PetscInt v2   = firstVertex + eyp1 * numXVertices + exp1;
    1327:         const PetscInt v3   = firstVertex + eyp1 * numXVertices + ex;
    1328:         const PetscInt e0   = firstXEdge + ey * numXEdges + ex;
    1329:         const PetscInt e1   = firstYEdge + exp1 * numYEdges + ey;
    1330:         const PetscInt e2   = firstXEdge + eyp1 * numXEdges + ex;
    1331:         const PetscInt e3   = firstYEdge + ex * numYEdges + ey;

    1333:         const PetscInt cones[] = {ez, e0, ez + 1, ez + 1, e1, ez + 2, ez + 2, e2, ez + 3, ez + 3, e3, ez};
    1334:         const PetscInt ornts[] = {-1, 0, 0, -1, 0, 0, -1, -1, 0, -1, -1, 0};
    1335:         const PetscInt verts[] = {v0, vc, v1, vc, v2, vc, v3, vc};

    1337:         for (c = 0; c < 4; c++) {
    1338:           PetscInt cell = 4 * (ey * numXEdges + ex) + c;
    1339:           PetscInt edge = ez + c;

    1341:           PetscCall(DMPlexSetCone(dm, cell, cones + 3 * c));
    1342:           PetscCall(DMPlexSetConeOrientation(dm, cell, ornts + 3 * c));
    1343:           PetscCall(DMPlexSetCone(dm, edge, verts + 2 * c));
    1344:         }
    1345:       }
    1346:     }

    1348:     /* Build Y edges*/
    1349:     for (vx = 0; vx < numXVertices; vx++) {
    1350:       for (ey = 0; ey < numYEdges; ey++) {
    1351:         const PetscInt edge   = firstYEdge + vx * numYEdges + ey;
    1352:         const PetscInt v0     = firstVertex + ey * numXVertices + vx;
    1353:         const PetscInt v1     = firstVertex + ((ey + 1) % numYVertices) * numXVertices + vx;
    1354:         const PetscInt cone[] = {v0, v1};

    1356:         PetscCall(DMPlexSetCone(dm, edge, cone));
    1357:         if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
    1358:           if (vx == numXVertices - 1) {
    1359:             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
    1360:             PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
    1361:             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
    1362:             if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
    1363:           } else if (vx == 0) {
    1364:             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
    1365:             PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
    1366:             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
    1367:             if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
    1368:           }
    1369:         }
    1370:       }
    1371:     }

    1373:     /* Build X edges*/
    1374:     for (vy = 0; vy < numYVertices; vy++) {
    1375:       for (ex = 0; ex < numXEdges; ex++) {
    1376:         const PetscInt edge   = firstXEdge + vy * numXEdges + ex;
    1377:         const PetscInt v0     = firstVertex + vy * numXVertices + ex;
    1378:         const PetscInt v1     = firstVertex + vy * numXVertices + (ex + 1) % numXVertices;
    1379:         const PetscInt cone[] = {v0, v1};

    1381:         PetscCall(DMPlexSetCone(dm, edge, cone));
    1382:         if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
    1383:           if (vy == numYVertices - 1) {
    1384:             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
    1385:             PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
    1386:             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
    1387:             if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
    1388:           } else if (vy == 0) {
    1389:             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
    1390:             PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
    1391:             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
    1392:             if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
    1393:           }
    1394:         }
    1395:       }
    1396:     }

    1398:     /* Compute support, stratify, and celltype label */
    1399:     PetscCall(DMPlexSymmetrize(dm));
    1400:     PetscCall(DMPlexStratify(dm));
    1401:     PetscCall(DMPlexComputeCellTypes(dm));

    1403:     /* Build coordinates */
    1404:     PetscCall(DMGetCoordinateSection(dm, &coordSection));
    1405:     PetscCall(PetscSectionSetNumFields(coordSection, 1));
    1406:     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
    1407:     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
    1408:     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
    1409:       PetscCall(PetscSectionSetDof(coordSection, v, 2));
    1410:       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
    1411:     }
    1412:     PetscCall(PetscSectionSetUp(coordSection));
    1413:     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
    1414:     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
    1415:     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    1416:     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
    1417:     PetscCall(VecSetBlockSize(coordinates, 2));
    1418:     PetscCall(VecSetType(coordinates, VECSTANDARD));
    1419:     PetscCall(VecGetArray(coordinates, &coords));
    1420:     for (vy = 0; vy < numYVertices; ++vy) {
    1421:       for (vx = 0; vx < numXVertices; ++vx) {
    1422:         coords[2 * (vy * numXVertices + vx) + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
    1423:         coords[2 * (vy * numXVertices + vx) + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
    1424:       }
    1425:     }
    1426:     for (ey = 0; ey < numYEdges; ++ey) {
    1427:       for (ex = 0; ex < numXEdges; ++ex) {
    1428:         const PetscInt c = ey * numXEdges + ex + numYVertices * numXVertices;

    1430:         coords[2 * c + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * (ex + 0.5);
    1431:         coords[2 * c + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * (ey + 0.5);
    1432:       }
    1433:     }
    1434:     PetscCall(VecRestoreArray(coordinates, &coords));
    1435:     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
    1436:     PetscCall(VecDestroy(&coordinates));

    1438:     /* handle periodic BC */
    1439:     if (bdX == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_PERIODIC) {
    1440:       PetscReal L[2]       = {-1., -1.};
    1441:       PetscReal maxCell[2] = {-1., -1.};

    1443:       for (PetscInt d = 0; d < 2; ++d) {
    1444:         if (bd[d] != DM_BOUNDARY_NONE) {
    1445:           L[d]       = upper[d] - lower[d];
    1446:           maxCell[d] = 1.1 * (L[d] / PetscMax(1, edges[d]));
    1447:         }
    1448:       }
    1449:       PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
    1450:     }
    1451:   }
    1452:   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
    1453:   PetscFunctionReturn(PETSC_SUCCESS);
    1454: }

    1456: static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
    1457: {
    1458:   DM        boundary, vol;
    1459:   DMLabel   bdlabel;
    1460:   PetscBool crisscross = PETSC_FALSE;

    1462:   PetscFunctionBegin;
    1463:   PetscAssertPointer(dm, 1);
    1464:   if (dim == 2) PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_box_crisscross", &crisscross, NULL));
    1465:   if (crisscross) {
    1466:     PetscCall(DMPlexCreateSquareMesh_Simplex_CrissCross(dm, faces, lower, upper, periodicity));
    1467:   } else {
    1468:     for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
    1469:     PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
    1470:     PetscCall(DMSetType(boundary, DMPLEX));
    1471:     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)boundary, ((PetscObject)dm)->prefix));
    1472:     PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
    1473:     PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
    1474:     PetscCall(DMGetLabel(vol, "marker", &bdlabel));
    1475:     if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
    1476:     PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
    1477:     PetscCall(DMPlexReplace_Internal(dm, &vol));
    1478:     PetscCall(DMDestroy(&boundary));
    1479:   }
    1480:   if (interpolate) {
    1481:     PetscCall(DMPlexInterpolateInPlace_Internal(dm));
    1482:     PetscCall(DMPlexSetBoxLabel_Internal(dm, periodicity));
    1483:   }
    1484:   PetscFunctionReturn(PETSC_SUCCESS);
    1485: }

    1487: static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
    1488: {
    1489:   DMLabel     cutLabel  = NULL;
    1490:   PetscInt    markerTop = 1, faceMarkerTop = 1;
    1491:   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
    1492:   PetscInt    markerFront = 1, faceMarkerFront = 1;
    1493:   PetscInt    markerBack = 1, faceMarkerBack = 1;
    1494:   PetscInt    markerRight = 1, faceMarkerRight = 1;
    1495:   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
    1496:   PetscInt    dim;
    1497:   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
    1498:   PetscMPIInt rank;

    1500:   PetscFunctionBegin;
    1501:   PetscCall(DMGetDimension(dm, &dim));
    1502:   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
    1503:   PetscCall(DMCreateLabel(dm, "marker"));
    1504:   PetscCall(DMCreateLabel(dm, "Face Sets"));
    1505:   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
    1506:   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
    1507:     if (cutMarker) {
    1508:       PetscCall(DMCreateLabel(dm, "periodic_cut"));
    1509:       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
    1510:     }
    1511:   }
    1512:   switch (dim) {
    1513:   case 2:
    1514:     faceMarkerTop    = 3;
    1515:     faceMarkerBottom = 1;
    1516:     faceMarkerRight  = 2;
    1517:     faceMarkerLeft   = 4;
    1518:     break;
    1519:   case 3:
    1520:     faceMarkerBottom = 1;
    1521:     faceMarkerTop    = 2;
    1522:     faceMarkerFront  = 3;
    1523:     faceMarkerBack   = 4;
    1524:     faceMarkerRight  = 5;
    1525:     faceMarkerLeft   = 6;
    1526:     break;
    1527:   default:
    1528:     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
    1529:   }
    1530:   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
    1531:   if (markerSeparate) {
    1532:     markerBottom = faceMarkerBottom;
    1533:     markerTop    = faceMarkerTop;
    1534:     markerFront  = faceMarkerFront;
    1535:     markerBack   = faceMarkerBack;
    1536:     markerRight  = faceMarkerRight;
    1537:     markerLeft   = faceMarkerLeft;
    1538:   }
    1539:   {
    1540:     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
    1541:     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
    1542:     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
    1543:     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
    1544:     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
    1545:     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
    1546:     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
    1547:     const PetscInt numXFaces    = numYEdges * numZEdges;
    1548:     const PetscInt numYFaces    = numXEdges * numZEdges;
    1549:     const PetscInt numZFaces    = numXEdges * numYEdges;
    1550:     const PetscInt numTotXFaces = numXVertices * numXFaces;
    1551:     const PetscInt numTotYFaces = numYVertices * numYFaces;
    1552:     const PetscInt numTotZFaces = numZVertices * numZFaces;
    1553:     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
    1554:     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
    1555:     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
    1556:     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
    1557:     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
    1558:     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
    1559:     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
    1560:     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
    1561:     const PetscInt firstYFace   = firstXFace + numTotXFaces;
    1562:     const PetscInt firstZFace   = firstYFace + numTotYFaces;
    1563:     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
    1564:     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
    1565:     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
    1566:     Vec            coordinates;
    1567:     PetscSection   coordSection;
    1568:     PetscScalar   *coords;
    1569:     PetscInt       coordSize;
    1570:     PetscInt       v, vx, vy, vz;
    1571:     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;

    1573:     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
    1574:     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
    1575:     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
    1576:     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
    1577:     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
    1578:     /* Build cells */
    1579:     for (fz = 0; fz < numZEdges; ++fz) {
    1580:       for (fy = 0; fy < numYEdges; ++fy) {
    1581:         for (fx = 0; fx < numXEdges; ++fx) {
    1582:           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
    1583:           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
    1584:           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
    1585:           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
    1586:           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
    1587:           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
    1588:           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
    1589:           /* B,  T,  F,  K,  R,  L */
    1590:           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
    1591:           PetscInt cone[6];

    1593:           /* no boundary twisting in 3D */
    1594:           cone[0] = faceB;
    1595:           cone[1] = faceT;
    1596:           cone[2] = faceF;
    1597:           cone[3] = faceK;
    1598:           cone[4] = faceR;
    1599:           cone[5] = faceL;
    1600:           PetscCall(DMPlexSetCone(dm, cell, cone));
    1601:           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
    1602:           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
    1603:           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
    1604:           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
    1605:         }
    1606:       }
    1607:     }
    1608:     /* Build x faces */
    1609:     for (fz = 0; fz < numZEdges; ++fz) {
    1610:       for (fy = 0; fy < numYEdges; ++fy) {
    1611:         for (fx = 0; fx < numXVertices; ++fx) {
    1612:           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
    1613:           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
    1614:           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
    1615:           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
    1616:           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
    1617:           PetscInt ornt[4] = {0, 0, -1, -1};
    1618:           PetscInt cone[4];

    1620:           if (dim == 3) {
    1621:             /* markers */
    1622:             if (bdX != DM_BOUNDARY_PERIODIC) {
    1623:               if (fx == numXVertices - 1) {
    1624:                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
    1625:                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
    1626:               } else if (fx == 0) {
    1627:                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
    1628:                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
    1629:               }
    1630:             }
    1631:           }
    1632:           cone[0] = edgeB;
    1633:           cone[1] = edgeR;
    1634:           cone[2] = edgeT;
    1635:           cone[3] = edgeL;
    1636:           PetscCall(DMPlexSetCone(dm, face, cone));
    1637:           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
    1638:         }
    1639:       }
    1640:     }
    1641:     /* Build y faces */
    1642:     for (fz = 0; fz < numZEdges; ++fz) {
    1643:       for (fx = 0; fx < numXEdges; ++fx) {
    1644:         for (fy = 0; fy < numYVertices; ++fy) {
    1645:           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
    1646:           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
    1647:           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
    1648:           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
    1649:           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
    1650:           PetscInt ornt[4] = {0, 0, -1, -1};
    1651:           PetscInt cone[4];

    1653:           if (dim == 3) {
    1654:             /* markers */
    1655:             if (bdY != DM_BOUNDARY_PERIODIC) {
    1656:               if (fy == numYVertices - 1) {
    1657:                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
    1658:                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
    1659:               } else if (fy == 0) {
    1660:                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
    1661:                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
    1662:               }
    1663:             }
    1664:           }
    1665:           cone[0] = edgeB;
    1666:           cone[1] = edgeR;
    1667:           cone[2] = edgeT;
    1668:           cone[3] = edgeL;
    1669:           PetscCall(DMPlexSetCone(dm, face, cone));
    1670:           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
    1671:         }
    1672:       }
    1673:     }
    1674:     /* Build z faces */
    1675:     for (fy = 0; fy < numYEdges; ++fy) {
    1676:       for (fx = 0; fx < numXEdges; ++fx) {
    1677:         for (fz = 0; fz < numZVertices; fz++) {
    1678:           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
    1679:           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
    1680:           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
    1681:           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
    1682:           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
    1683:           PetscInt ornt[4] = {0, 0, -1, -1};
    1684:           PetscInt cone[4];

    1686:           if (dim == 2) {
    1687:             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
    1688:               edgeR += numYEdges - 1 - 2 * fy;
    1689:               ornt[1] = -1;
    1690:             }
    1691:             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
    1692:               edgeT += numXEdges - 1 - 2 * fx;
    1693:               ornt[2] = 0;
    1694:             }
    1695:             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
    1696:             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
    1697:           } else {
    1698:             /* markers */
    1699:             if (bdZ != DM_BOUNDARY_PERIODIC) {
    1700:               if (fz == numZVertices - 1) {
    1701:                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
    1702:                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
    1703:               } else if (fz == 0) {
    1704:                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
    1705:                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
    1706:               }
    1707:             }
    1708:           }
    1709:           cone[0] = edgeB;
    1710:           cone[1] = edgeR;
    1711:           cone[2] = edgeT;
    1712:           cone[3] = edgeL;
    1713:           PetscCall(DMPlexSetCone(dm, face, cone));
    1714:           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
    1715:         }
    1716:       }
    1717:     }
    1718:     /* Build Z edges*/
    1719:     for (vy = 0; vy < numYVertices; vy++) {
    1720:       for (vx = 0; vx < numXVertices; vx++) {
    1721:         for (ez = 0; ez < numZEdges; ez++) {
    1722:           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
    1723:           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
    1724:           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
    1725:           PetscInt       cone[2];

    1727:           cone[0] = vertexB;
    1728:           cone[1] = vertexT;
    1729:           PetscCall(DMPlexSetCone(dm, edge, cone));
    1730:           if (dim == 3) {
    1731:             if (bdX != DM_BOUNDARY_PERIODIC) {
    1732:               if (vx == numXVertices - 1) {
    1733:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
    1734:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
    1735:                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
    1736:               } else if (vx == 0) {
    1737:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
    1738:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
    1739:                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
    1740:               }
    1741:             }
    1742:             if (bdY != DM_BOUNDARY_PERIODIC) {
    1743:               if (vy == numYVertices - 1) {
    1744:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
    1745:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
    1746:                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
    1747:               } else if (vy == 0) {
    1748:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
    1749:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
    1750:                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
    1751:               }
    1752:             }
    1753:           }
    1754:         }
    1755:       }
    1756:     }
    1757:     /* Build Y edges*/
    1758:     for (vz = 0; vz < numZVertices; vz++) {
    1759:       for (vx = 0; vx < numXVertices; vx++) {
    1760:         for (ey = 0; ey < numYEdges; ey++) {
    1761:           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
    1762:           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
    1763:           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
    1764:           const PetscInt vertexK = firstVertex + nextv;
    1765:           PetscInt       cone[2];

    1767:           cone[0] = vertexF;
    1768:           cone[1] = vertexK;
    1769:           PetscCall(DMPlexSetCone(dm, edge, cone));
    1770:           if (dim == 2) {
    1771:             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
    1772:               if (vx == numXVertices - 1) {
    1773:                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
    1774:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
    1775:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
    1776:                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
    1777:               } else if (vx == 0) {
    1778:                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
    1779:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
    1780:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
    1781:                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
    1782:               }
    1783:             } else {
    1784:               if (vx == 0 && cutLabel) {
    1785:                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
    1786:                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
    1787:                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
    1788:               }
    1789:             }
    1790:           } else {
    1791:             if (bdX != DM_BOUNDARY_PERIODIC) {
    1792:               if (vx == numXVertices - 1) {
    1793:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
    1794:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
    1795:                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
    1796:               } else if (vx == 0) {
    1797:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
    1798:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
    1799:                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
    1800:               }
    1801:             }
    1802:             if (bdZ != DM_BOUNDARY_PERIODIC) {
    1803:               if (vz == numZVertices - 1) {
    1804:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
    1805:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
    1806:                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
    1807:               } else if (vz == 0) {
    1808:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
    1809:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
    1810:                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
    1811:               }
    1812:             }
    1813:           }
    1814:         }
    1815:       }
    1816:     }
    1817:     /* Build X edges*/
    1818:     for (vz = 0; vz < numZVertices; vz++) {
    1819:       for (vy = 0; vy < numYVertices; vy++) {
    1820:         for (ex = 0; ex < numXEdges; ex++) {
    1821:           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
    1822:           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
    1823:           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
    1824:           const PetscInt vertexR = firstVertex + nextv;
    1825:           PetscInt       cone[2];

    1827:           cone[0] = vertexL;
    1828:           cone[1] = vertexR;
    1829:           PetscCall(DMPlexSetCone(dm, edge, cone));
    1830:           if (dim == 2) {
    1831:             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
    1832:               if (vy == numYVertices - 1) {
    1833:                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
    1834:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
    1835:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
    1836:                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
    1837:               } else if (vy == 0) {
    1838:                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
    1839:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
    1840:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
    1841:                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
    1842:               }
    1843:             } else {
    1844:               if (vy == 0 && cutLabel) {
    1845:                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
    1846:                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
    1847:                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
    1848:               }
    1849:             }
    1850:           } else {
    1851:             if (bdY != DM_BOUNDARY_PERIODIC) {
    1852:               if (vy == numYVertices - 1) {
    1853:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
    1854:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
    1855:                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
    1856:               } else if (vy == 0) {
    1857:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
    1858:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
    1859:                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
    1860:               }
    1861:             }
    1862:             if (bdZ != DM_BOUNDARY_PERIODIC) {
    1863:               if (vz == numZVertices - 1) {
    1864:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
    1865:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
    1866:                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
    1867:               } else if (vz == 0) {
    1868:                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
    1869:                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
    1870:                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
    1871:               }
    1872:             }
    1873:           }
    1874:         }
    1875:       }
    1876:     }
    1877:     PetscCall(DMPlexSymmetrize(dm));
    1878:     PetscCall(DMPlexStratify(dm));
    1879:     /* Build coordinates */
    1880:     PetscCall(DMGetCoordinateSection(dm, &coordSection));
    1881:     PetscCall(PetscSectionSetNumFields(coordSection, 1));
    1882:     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
    1883:     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
    1884:     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
    1885:       PetscCall(PetscSectionSetDof(coordSection, v, dim));
    1886:       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
    1887:     }
    1888:     PetscCall(PetscSectionSetUp(coordSection));
    1889:     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
    1890:     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
    1891:     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    1892:     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
    1893:     PetscCall(VecSetBlockSize(coordinates, dim));
    1894:     PetscCall(VecSetType(coordinates, VECSTANDARD));
    1895:     PetscCall(VecGetArray(coordinates, &coords));
    1896:     for (vz = 0; vz < numZVertices; ++vz) {
    1897:       for (vy = 0; vy < numYVertices; ++vy) {
    1898:         for (vx = 0; vx < numXVertices; ++vx) {
    1899:           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
    1900:           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
    1901:           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
    1902:         }
    1903:       }
    1904:     }
    1905:     PetscCall(VecRestoreArray(coordinates, &coords));
    1906:     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
    1907:     PetscCall(VecDestroy(&coordinates));
    1908:   }
    1909:   PetscFunctionReturn(PETSC_SUCCESS);
    1910: }

    1912: static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
    1913: {
    1914:   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
    1915:   PetscInt       fac[3] = {0, 0, 0}, d;

    1917:   PetscFunctionBegin;
    1918:   PetscAssertPointer(dm, 1);
    1920:   PetscCall(DMSetDimension(dm, dim));
    1921:   for (d = 0; d < dim; ++d) {
    1922:     fac[d] = faces[d];
    1923:     bdt[d] = periodicity[d];
    1924:   }
    1925:   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
    1926:   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
    1927:     PetscReal L[3]       = {-1., -1., 0.};
    1928:     PetscReal maxCell[3] = {-1., -1., 0.};

    1930:     for (d = 0; d < dim; ++d) {
    1931:       if (periodicity[d] != DM_BOUNDARY_NONE) {
    1932:         L[d]       = upper[d] - lower[d];
    1933:         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
    1934:       }
    1935:     }
    1936:     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
    1937:   }
    1938:   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
    1939:   PetscFunctionReturn(PETSC_SUCCESS);
    1940: }

    1942: static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, DMPlexShape shape, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
    1943: {
    1944:   PetscFunctionBegin;
    1945:   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
    1946:   if (shape == DM_SHAPE_ZBOX) PetscCall(DMPlexCreateBoxMesh_Tensor_SFC_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
    1947:   else if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
    1948:   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
    1949:   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
    1950:   if (!interpolate && dim > 1 && !simplex) {
    1951:     DM udm;

    1953:     PetscCall(DMPlexUninterpolate(dm, &udm));
    1954:     PetscCall(DMPlexCopyCoordinates(dm, udm));
    1955:     PetscCall(DMPlexReplace_Internal(dm, &udm));
    1956:   }
    1957:   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
    1958:   PetscFunctionReturn(PETSC_SUCCESS);
    1959: }

    1961: /*@
    1962:   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).

    1964:   Collective

    1966:   Input Parameters:
    1967: + comm               - The communicator for the `DM` object
    1968: . dim                - The spatial dimension
    1969: . simplex            - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
    1970: . faces              - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
    1971: . lower              - The lower left corner, or `NULL` for (0, 0, 0)
    1972: . upper              - The upper right corner, or `NULL` for (1, 1, 1)
    1973: . periodicity        - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
    1974: . interpolate        - Flag to create intermediate mesh pieces (edges, faces)
    1975: . localizationHeight - Flag to localize edges and faces in addition to cells; only significant for periodic meshes
    1976: - sparseLocalize     - Flag to localize coordinates only for cells near the periodic boundary; only significant for periodic meshes

    1978:   Output Parameter:
    1979: . dm - The `DM` object

    1981:   Level: beginner

    1983:   Note:
    1984:   To customize this mesh using options, use
    1985: .vb
    1986:   DMCreate(comm, &dm);
    1987:   DMSetType(dm, DMPLEX);
    1988:   DMSetFromOptions(dm);
    1989: .ve
    1990:   and use the options in `DMSetFromOptions()`.

    1992:   Here is the numbering returned for 2 faces in each direction for tensor cells\:
    1993: .vb
    1994:  10---17---11---18----12
    1995:   |         |         |
    1996:   |         |         |
    1997:  20    2   22    3    24
    1998:   |         |         |
    1999:   |         |         |
    2000:   7---15----8---16----9
    2001:   |         |         |
    2002:   |         |         |
    2003:  19    0   21    1   23
    2004:   |         |         |
    2005:   |         |         |
    2006:   4---13----5---14----6
    2007: .ve
    2008:   and for simplicial cells
    2009: .vb
    2010:  14----8---15----9----16
    2011:   |\     5  |\      7 |
    2012:   | \       | \       |
    2013:  13   2    14    3    15
    2014:   | 4   \   | 6   \   |
    2015:   |       \ |       \ |
    2016:  11----6---12----7----13
    2017:   |\        |\        |
    2018:   | \    1  | \     3 |
    2019:  10   0    11    1    12
    2020:   | 0   \   | 2   \   |
    2021:   |       \ |       \ |
    2022:   8----4----9----5----10
    2023: .ve

    2025: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
    2026: @*/
    2027: PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, PetscInt localizationHeight, PetscBool sparseLocalize, DM *dm)
    2028: {
    2029:   PetscInt       fac[3] = {1, 1, 1};
    2030:   PetscReal      low[3] = {0, 0, 0};
    2031:   PetscReal      upp[3] = {1, 1, 1};
    2032:   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};

    2034:   PetscFunctionBegin;
    2035:   PetscCall(DMCreate(comm, dm));
    2036:   PetscCall(DMSetType(*dm, DMPLEX));
    2037:   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, DM_SHAPE_BOX, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
    2038:   if (periodicity) {
    2039:     DM cdm;

    2041:     PetscCall(DMGetCoordinateDM(*dm, &cdm));
    2042:     PetscCall(DMPlexSetMaxProjectionHeight(cdm, localizationHeight));
    2043:     PetscCall(DMSetSparseLocalize(*dm, sparseLocalize));
    2044:     PetscCall(DMLocalizeCoordinates(*dm));
    2045:   }
    2046:   PetscFunctionReturn(PETSC_SUCCESS);
    2047: }

    2049: static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
    2050: {
    2051:   DM       bdm, vol;
    2052:   PetscInt i;

    2054:   PetscFunctionBegin;
    2055:   // TODO Now we can support periodicity
    2056:   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
    2057:   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
    2058:   PetscCall(DMSetType(bdm, DMPLEX));
    2059:   PetscCall(DMSetDimension(bdm, 2));
    2060:   PetscCall(PetscLogEventBegin(DMPLEX_Generate, bdm, 0, 0, 0));
    2061:   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
    2062:   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, NULL, NULL, NULL, &vol));
    2063:   PetscCall(PetscLogEventEnd(DMPLEX_Generate, bdm, 0, 0, 0));
    2064:   PetscCall(DMDestroy(&bdm));
    2065:   PetscCall(DMPlexReplace_Internal(dm, &vol));
    2066:   if (lower[2] != 0.0) {
    2067:     Vec          v;
    2068:     PetscScalar *x;
    2069:     PetscInt     cDim, n;

    2071:     PetscCall(DMGetCoordinatesLocal(dm, &v));
    2072:     PetscCall(VecGetBlockSize(v, &cDim));
    2073:     PetscCall(VecGetLocalSize(v, &n));
    2074:     PetscCall(VecGetArray(v, &x));
    2075:     x += cDim;
    2076:     for (i = 0; i < n; i += cDim) x[i] += lower[2];
    2077:     PetscCall(VecRestoreArray(v, &x));
    2078:     PetscCall(DMSetCoordinatesLocal(dm, v));
    2079:   }
    2080:   PetscFunctionReturn(PETSC_SUCCESS);
    2081: }

    2083: /*@
    2084:   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tessellating the (x,y) plane and extruding in the third direction using wedge cells.

    2086:   Collective

    2088:   Input Parameters:
    2089: + comm        - The communicator for the `DM` object
    2090: . faces       - Number of faces per dimension, or `NULL` for (1, 1, 1)
    2091: . lower       - The lower left corner, or `NULL` for (0, 0, 0)
    2092: . upper       - The upper right corner, or `NULL` for (1, 1, 1)
    2093: . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
    2094: . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
    2095: - interpolate - Flag to create intermediate mesh pieces (edges, faces)

    2097:   Output Parameter:
    2098: . dm - The `DM` object

    2100:   Level: beginner

    2102: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
    2103: @*/
    2104: PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
    2105: {
    2106:   PetscInt       fac[3] = {1, 1, 1};
    2107:   PetscReal      low[3] = {0, 0, 0};
    2108:   PetscReal      upp[3] = {1, 1, 1};
    2109:   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};

    2111:   PetscFunctionBegin;
    2112:   PetscCall(DMCreate(comm, dm));
    2113:   PetscCall(DMSetType(*dm, DMPLEX));
    2114:   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
    2115:   if (!interpolate) {
    2116:     DM udm;

    2118:     PetscCall(DMPlexUninterpolate(*dm, &udm));
    2119:     PetscCall(DMPlexReplace_Internal(*dm, &udm));
    2120:   }
    2121:   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
    2122:   PetscFunctionReturn(PETSC_SUCCESS);
    2123: }

    2125: /*
    2126:   DMPlexTensorPointLexicographic_Private - Returns all tuples of size 'len' with nonnegative integers that are all less than or equal to 'max' for that dimension.

    2128:   Input Parameters:
    2129: + len - The length of the tuple
    2130: . max - The maximum for each dimension, so values are in [0, max)
    2131: - tup - A tuple of length len+1: tup[len] > 0 indicates a stopping condition

    2133:   Output Parameter:
    2134: . tup - A tuple of `len` integers whose entries are at most `max`

    2136:   Level: developer

    2138:   Note:
    2139:   Ordering is lexicographic with lowest index as least significant in ordering.
    2140:   e.g. for len == 2 and max == 2, this will return, in order, {0,0}, {1,0}, {2,0}, {0,1}, {1,1}, {2,1}, {0,2}, {1,2}, {2,2}.

    2142: .seealso: PetscDualSpaceTensorPointLexicographic_Internal(), PetscDualSpaceLatticePointLexicographic_Internal()
    2143: */
    2144: static PetscErrorCode DMPlexTensorPointLexicographic_Private(PetscInt len, const PetscInt max[], PetscInt tup[])
    2145: {
    2146:   PetscInt i;

    2148:   PetscFunctionBegin;
    2149:   for (i = 0; i < len; ++i) {
    2150:     if (tup[i] < max[i] - 1) {
    2151:       break;
    2152:     } else {
    2153:       tup[i] = 0;
    2154:     }
    2155:   }
    2156:   if (i == len) tup[i - 1] = max[i - 1];
    2157:   else ++tup[i];
    2158:   PetscFunctionReturn(PETSC_SUCCESS);
    2159: }

    2161: static PetscInt TupleToIndex_Private(PetscInt len, const PetscInt max[], const PetscInt tup[])
    2162: {
    2163:   PetscInt idx = tup[len - 1];

    2165:   for (PetscInt i = len - 2; i >= 0; --i) {
    2166:     idx *= max[i];
    2167:     idx += tup[i];
    2168:   }
    2169:   return idx;
    2170: }

    2172: static void IndexToTuple_Private(PetscInt len, const PetscInt max[], PetscInt idx, PetscInt tup[])
    2173: {
    2174:   for (PetscInt i = 0; i < len; ++i) {
    2175:     tup[i] = idx % max[i];
    2176:     idx    = (idx - tup[i]) / max[i];
    2177:   }
    2178: }

    2180: static void TupleToRanks_Private(PetscInt len, const PetscInt max[], const PetscInt procs[], const PetscInt tup[], PetscInt ranks[])
    2181: {
    2182:   for (PetscInt i = 0; i < len; ++i) {
    2183:     const PetscInt div = max[i] / procs[i];
    2184:     const PetscInt rem = max[i] % procs[i];
    2185:     const PetscInt idx = (tup[i] < 0 ? max[i] + tup[i] : tup[i]) % max[i];

    2187:     if (idx < rem * (div + 1)) ranks[i] = idx / (div + 1);
    2188:     else ranks[i] = rem + (idx - rem * (div + 1)) / div;
    2189:   }
    2190: }

    2192: static void RanksToSizes_Private(PetscInt len, const PetscInt max[], const PetscInt procs[], const PetscInt ranks[], PetscInt sizes[])
    2193: {
    2194:   for (PetscInt i = 0; i < len; ++i) {
    2195:     const PetscInt div = max[i] / procs[i];
    2196:     const PetscInt rem = max[i] % procs[i];

    2198:     sizes[i] = ranks[i] < rem ? div + 1 : div;
    2199:   }
    2200: }

    2202: /*
    2203:   In serial, the mesh is completely periodic. In parallel, we will include a layer of ghost vertices around the patch, so our edges will not wrap around in parallel. This will instead be handled by the SF.

    2205:   The cone for all edges will always be complete. Even in the presence of boundaries, we will keep all support sizes constant. When an edge would not exist in the support, we will create one to wrap back periodically. This allows for uniform stencils, and that edge will not be used for computation.

    2207:   All point which do not attain the vertex lower or upper bound in any dimension are owned, the rest are leaves owned by another process and present in the SF.

    2209:   Parallel Layout:

    2211:   We create a cubic process grid of dimension P^{1/d} on each side. The IndexToTuple_Private() function maps the global rank to the local rank in each dimension. We divide edges using the PETSc distribution rule in each dimension, and then add overlap. TupleToRanks_Private() returns the local rank in ech dimension for a tuple. RanksToSizes_Private() gives the size in each dimension for the domain with those local ranks.
    2212: */
    2213: static PetscErrorCode DMPlexCreateHypercubicMesh_Internal(DM dm, PetscInt dim, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], PetscInt overlap, const DMBoundaryType bd[])
    2214: {
    2215:   const PetscInt debug = ((DM_Plex *)dm->data)->printAdj;
    2216:   PetscSF        sf;
    2217:   Vec            coordinates;
    2218:   PetscSection   coordSection;
    2219:   DMLabel        cutLabel    = NULL;
    2220:   PetscBool      cutMarker   = PETSC_FALSE;
    2221:   PetscBool      periodic    = PETSC_FALSE;
    2222:   PetscInt       numCells    = 1;
    2223:   PetscInt       numVertices = 1;
    2224:   PetscSFNode   *remotes;
    2225:   PetscScalar   *coords;
    2226:   PetscInt      *procs;     // The number of processes along each dimension
    2227:   PetscInt      *lrank;     // Rank in each dimension, lrank[d] \in [0, procs[d])
    2228:   PetscInt      *ledges;    // The number of edges along each dimension for this process
    2229:   PetscInt      *vstart;    // The first vertex along each dimension on this processes
    2230:   PetscInt      *vertices;  // The number of vertices along each dimension on this process
    2231:   PetscInt      *rvert;     // The global (not local) vertex number along each dimension
    2232:   PetscInt      *rrank;     // The rank along each dimension for the process owning rvert[]
    2233:   PetscInt      *rvertices; // The number of vertices along each dimension for the process rrank[]
    2234:   PetscInt      *vert, *vtmp, *supp, cone[2], *leaves;
    2235:   PetscInt       cell = 0, coordSize, Nl = 0, Nl2 = 0;
    2236:   PetscMPIInt    rank, size;
    2237:   MPI_Comm       comm;

    2239:   PetscFunctionBegin;
    2240:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
    2241:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
    2242:   PetscCallMPI(MPI_Comm_size(comm, &size));
    2243:   PetscCall(DMSetDimension(dm, dim));
    2244:   PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
    2245:   PetscCall(PetscCalloc4(dim, &procs, dim, &lrank, dim, &rrank, 2 * dim, &supp));
    2246:   PetscCall(PetscCalloc7(dim, &ledges, dim, &vertices, dim, &rvertices, dim, &vert, dim, &rvert, dim, &vstart, dim, &vtmp));
    2247:   PetscCall(DMCreateLabel(dm, "marker"));
    2248:   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
    2249:   for (PetscInt d = 0; d < dim; ++d) periodic = (periodic || bd[d] == DM_BOUNDARY_PERIODIC) ? PETSC_TRUE : PETSC_FALSE;
    2250:   if (periodic && cutMarker) {
    2251:     PetscCall(DMCreateLabel(dm, "periodic_cut"));
    2252:     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
    2253:   }
    2254:   for (PetscInt d = 0; d < dim; ++d) PetscCheck(bd[d] == DM_BOUNDARY_PERIODIC, comm, PETSC_ERR_SUP, "Hypercubic mesh must be periodic now");
    2255:   overlap = overlap == PETSC_DETERMINE ? 1 : overlap;
    2256:   PetscCheck(overlap >= 1, comm, PETSC_ERR_SUP, "Overlap %" PetscInt_FMT " must be greater than 0", overlap);
    2257:   if (size > 1) {
    2258:     PetscInt Npr = 1;

    2260:     // Make process grid
    2261:     if (debug) PetscCall(PetscPrintf(comm, "Process grid:"));
    2262:     for (PetscInt d = 0; d < dim; ++d) {
    2263:       procs[d] = PetscRintReal(PetscPowReal(size, 1. / dim));
    2264:       Npr *= procs[d];
    2265:       if (debug) PetscCall(PetscPrintf(comm, " %" PetscInt_FMT, procs[d]));
    2266:     }
    2267:     if (debug) PetscCall(PetscPrintf(comm, "\n"));
    2268:     PetscCheck(Npr == size, comm, PETSC_ERR_PLIB, "Process grid size %" PetscInt_FMT " != %d comm size", Npr, size);
    2269:     IndexToTuple_Private(dim, procs, rank, lrank);
    2270:     for (PetscInt d = 0; d < dim; ++d) {
    2271:       ledges[d] = edges[d] / procs[d] + (edges[d] % procs[d] > lrank[d] ? 1 : 0);
    2272:       vstart[d] = 0;
    2273:       for (PetscInt r = 0; r < lrank[d]; ++r) vstart[d] += edges[d] / procs[d] + (edges[d] % procs[d] > r ? 1 : 0);
    2274:       vstart[d] -= overlap; // For halo
    2275:     }
    2276:   } else {
    2277:     for (PetscInt d = 0; d < dim; ++d) {
    2278:       procs[d]  = 1;
    2279:       ledges[d] = edges[d];
    2280:     }
    2281:   }
    2282:   // Calculate local patch size
    2283:   for (PetscInt d = 0; d < dim; ++d) {
    2284:     vertices[d] = ledges[d] + (procs[d] > 1 ? 2 * overlap : 0);
    2285:     numVertices *= vertices[d];
    2286:   }
    2287:   numCells = numVertices * dim;
    2288:   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
    2289:   for (PetscInt c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, 2));
    2290:   for (PetscInt v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetSupportSize(dm, v, 2 * dim));
    2291:   PetscCall(DMSetUp(dm)); /* Allocate space for cones and supports */
    2292:   /* Build cell cones and vertex supports */
    2293:   PetscCall(DMCreateLabel(dm, "celltype"));
    2294:   if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Topology for rank %d:\n", rank));
    2295:   while (vert[dim - 1] < vertices[dim - 1]) {
    2296:     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert) + numCells;
    2297:     PetscInt       s      = 0;
    2298:     PetscBool      leaf   = PETSC_FALSE;

    2300:     if (debug) {
    2301:       PetscCall(PetscSynchronizedPrintf(comm, "Vertex %" PetscInt_FMT ":", vertex));
    2302:       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vert[d]));
    2303:       PetscCall(PetscSynchronizedPrintf(comm, "\n"));
    2304:     }
    2305:     PetscCall(DMPlexSetCellType(dm, vertex, DM_POLYTOPE_POINT));
    2306:     // Define edge cones
    2307:     for (PetscInt d = 0; d < dim; ++d) {
    2308:       for (PetscInt e = 0; e < dim; ++e) vtmp[e] = vert[e];
    2309:       vtmp[d] = (vert[d] + 1) % vertices[d];
    2310:       cone[0] = vertex;
    2311:       cone[1] = TupleToIndex_Private(dim, vertices, vtmp) + numCells;
    2312:       if (debug) {
    2313:         PetscCall(PetscSynchronizedPrintf(comm, "  Vertex %" PetscInt_FMT ":", cone[1]));
    2314:         for (PetscInt e = 0; e < dim; ++e) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vtmp[e]));
    2315:         PetscCall(PetscSynchronizedPrintf(comm, "\n"));
    2316:       }
    2317:       PetscCall(DMPlexSetCone(dm, cell, cone));
    2318:       PetscCall(DMPlexSetCellType(dm, cell, DM_POLYTOPE_SEGMENT));
    2319:       if (debug) PetscCall(PetscSynchronizedPrintf(comm, "  Edge %" PetscInt_FMT " (%" PetscInt_FMT " %" PetscInt_FMT ")\n", cell, cone[0], cone[1]));
    2320:       ++cell;
    2321:       // Shared vertices are any in the first or last overlap layers
    2322:       if (vert[d] < overlap || vert[d] >= vertices[d] - overlap) leaf = PETSC_TRUE;
    2323:     }
    2324:     if (size > 1 && leaf) ++Nl;
    2325:     // Define vertex supports
    2326:     for (PetscInt d = 0; d < dim; ++d) {
    2327:       for (PetscInt e = 0; e < dim; ++e) vtmp[e] = vert[e];
    2328:       vtmp[d]   = (vert[d] + vertices[d] - 1) % vertices[d];
    2329:       supp[s++] = TupleToIndex_Private(dim, vertices, vtmp) * dim + d;
    2330:       supp[s++] = (vertex - numCells) * dim + d;
    2331:       PetscCall(DMPlexSetSupport(dm, vertex, supp));
    2332:     }
    2333:     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
    2334:   }
    2335:   if (debug) PetscCall(PetscSynchronizedFlush(comm, NULL));
    2336:   PetscCall(DMPlexStratify(dm));
    2337:   // Allocate for SF
    2338:   PetscCall(PetscMalloc1(Nl, &leaves));
    2339:   PetscCall(PetscMalloc1(Nl, &remotes));
    2340:   // Build coordinates
    2341:   PetscCall(DMGetCoordinateSection(dm, &coordSection));
    2342:   PetscCall(PetscSectionSetNumFields(coordSection, 1));
    2343:   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
    2344:   PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
    2345:   for (PetscInt v = numCells; v < numCells + numVertices; ++v) {
    2346:     PetscCall(PetscSectionSetDof(coordSection, v, dim));
    2347:     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
    2348:   }
    2349:   PetscCall(PetscSectionSetUp(coordSection));
    2350:   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
    2351:   PetscCall(VecCreate(comm, &coordinates));
    2352:   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    2353:   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
    2354:   PetscCall(VecSetBlockSize(coordinates, dim));
    2355:   PetscCall(VecSetType(coordinates, VECSTANDARD));
    2356:   PetscCall(VecGetArray(coordinates, &coords));
    2357:   if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Geometry for rank %d:\n", rank));
    2358:   for (PetscInt d = 0; d < dim; ++d) vert[d] = 0;
    2359:   while (vert[dim - 1] < vertices[dim - 1]) {
    2360:     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert);
    2361:     PetscBool      leaf   = PETSC_FALSE;

    2363:     for (PetscInt d = 0; d < dim; ++d) {
    2364:       coords[vertex * dim + d] = lower[d] + ((upper[d] - lower[d]) / edges[d]) * (vert[d] + vstart[d]);
    2365:       if (vert[d] < overlap || vert[d] >= vertices[d] - overlap) leaf = PETSC_TRUE;
    2366:     }
    2367:     if (size > 1 && leaf) {
    2368:       PetscInt rnumCells = 1;

    2370:       for (PetscInt d = 0; d < dim; ++d) rvert[d] = vert[d] + vstart[d];
    2371:       TupleToRanks_Private(dim, edges, procs, rvert, rrank);
    2372:       leaves[Nl2]       = vertex + numCells;
    2373:       remotes[Nl2].rank = TupleToIndex_Private(dim, procs, rrank);
    2374:       RanksToSizes_Private(dim, edges, procs, rrank, rvertices);
    2375:       for (PetscInt d = 0; d < dim; ++d) {
    2376:         rvertices[d] += 2 * overlap; // Add halo
    2377:         rnumCells *= rvertices[d];
    2378:       }
    2379:       rnumCells *= dim;
    2380:       for (PetscInt d = 0; d < dim; ++d) {
    2381:         const PetscInt diff = rrank[d] - lrank[d];

    2383:         if (!diff) rvert[d] = vert[d];                                     // Vertex is local
    2384:         else if (rvert[d] < 0) rvert[d] = rvertices[d] - 1 + rvert[d];     // Wrap around at the bottom
    2385:         else if (rvert[d] >= edges[d]) rvert[d] = rvert[d] - edges[d] + 1; // Wrap around at the top
    2386:         else if (diff == -1) rvert[d] = rvertices[d] - 1 + (vert[d] - overlap);
    2387:         else if (diff == 1) rvert[d] = (vertices[d] - vert[d] - 1) + overlap;
    2388:         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Process distance %" PetscInt_FMT " in direction %" PetscInt_FMT " should not be possible", diff, d);
    2389:       }
    2390:       remotes[Nl2].index = TupleToIndex_Private(dim, rvertices, rvert) + rnumCells;
    2391:       if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Shared Vertex %" PetscInt_FMT " (%" PetscInt_FMT ", %" PetscInt_FMT ")\n", leaves[Nl2], remotes[Nl2].rank, remotes[Nl2].index));
    2392:       ++Nl2;
    2393:     }
    2394:     if (debug) {
    2395:       PetscCall(PetscSynchronizedPrintf(comm, "Vertex %" PetscInt_FMT ":", vertex));
    2396:       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vert[d] + vstart[d]));
    2397:       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %g", (double)PetscRealPart(coords[vertex * dim + d])));
    2398:       PetscCall(PetscSynchronizedPrintf(comm, "\n"));
    2399:     }
    2400:     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
    2401:   }
    2402:   if (debug) PetscCall(PetscSynchronizedFlush(comm, NULL));
    2403:   PetscCall(VecRestoreArray(coordinates, &coords));
    2404:   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
    2405:   PetscCall(VecDestroy(&coordinates));
    2406:   // Build SF
    2407:   PetscCheck(Nl == Nl2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Initial number of leaves %" PetscInt_FMT " != %" PetscInt_FMT " final number", Nl, Nl2);
    2408:   PetscCall(DMGetPointSF(dm, &sf));
    2409:   PetscCall(PetscSFSetGraph(sf, numCells + numVertices, Nl, leaves, PETSC_OWN_POINTER, remotes, PETSC_OWN_POINTER));
    2410:   if (debug) PetscCall(PetscSFView(sf, PETSC_VIEWER_STDOUT_WORLD));
    2411:   //PetscCall(DMSetPeriodicity(dm, NULL, lower, upper));
    2412:   // Attach the extent
    2413:   {
    2414:     PetscContainer c;
    2415:     PetscInt      *extent, *lextent;

    2417:     PetscCall(PetscMalloc1(dim, &extent));
    2418:     PetscCall(PetscMalloc1(dim, &lextent));
    2419:     for (PetscInt d = 0; d < dim; ++d) {
    2420:       extent[d]  = edges[d];
    2421:       lextent[d] = ledges[d];
    2422:     }
    2423:     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
    2424:     PetscCall(PetscContainerSetCtxDestroy(c, PetscCtxDestroyDefault));
    2425:     PetscCall(PetscContainerSetPointer(c, extent));
    2426:     PetscCall(PetscObjectCompose((PetscObject)dm, "_extent", (PetscObject)c));
    2427:     PetscCall(PetscContainerDestroy(&c));
    2428:     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
    2429:     PetscCall(PetscContainerSetCtxDestroy(c, PetscCtxDestroyDefault));
    2430:     PetscCall(PetscContainerSetPointer(c, lextent));
    2431:     PetscCall(PetscObjectCompose((PetscObject)dm, "_lextent", (PetscObject)c));
    2432:     PetscCall(PetscContainerDestroy(&c));
    2433:   }
    2434:   PetscCall(PetscFree4(procs, lrank, rrank, supp));
    2435:   PetscCall(PetscFree7(ledges, vertices, rvertices, vert, rvert, vstart, vtmp));
    2436:   PetscFunctionReturn(PETSC_SUCCESS);
    2437: }

    2439: /*@C
    2440:   DMPlexCreateHypercubicMesh - Creates a periodic mesh on the tensor product of unit intervals using only vertices and edges.

    2442:   Collective

    2444:   Input Parameters:
    2445: + comm    - The communicator for the `DM` object
    2446: . dim     - The spatial dimension
    2447: . edges   - Number of edges per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
    2448: . lower   - The lower left corner, or `NULL` for (0, 0, 0)
    2449: . upper   - The upper right corner, or `NULL` for (1, 1, 1)
    2450: - overlap - The number of vertices in each direction to include in the overlap (default is 1)

    2452:   Output Parameter:
    2453: . dm - The DM object

    2455:   Level: beginner

    2457:   Note:
    2458:   If you want to customize this mesh using options, you just need to
    2459: .vb
    2460:   DMCreate(comm, &dm);
    2461:   DMSetType(dm, DMPLEX);
    2462:   DMSetFromOptions(dm);
    2463: .ve
    2464:   and use the options on the `DMSetFromOptions()` page.

    2466:   The vertices are numbered is lexicographic order, and the dim edges exiting a vertex in the positive orthant are number consecutively,
    2467: .vb
    2468:  18--0-19--2-20--4-18
    2469:   |     |     |     |
    2470:  13    15    17    13
    2471:   |     |     |     |
    2472:  24-12-25-14-26-16-24
    2473:   |     |     |     |
    2474:   7     9    11     7
    2475:   |     |     |     |
    2476:  21--6-22--8-23-10-21
    2477:   |     |     |     |
    2478:   1     3     5     1
    2479:   |     |     |     |
    2480:  18--0-19--2-20--4-18
    2481: .ve

    2483: .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
    2484: @*/
    2485: PetscErrorCode DMPlexCreateHypercubicMesh(MPI_Comm comm, PetscInt dim, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], PetscInt overlap, DM *dm)
    2486: {
    2487:   PetscInt       *edg;
    2488:   PetscReal      *low, *upp;
    2489:   DMBoundaryType *bdt;
    2490:   PetscInt        d;

    2492:   PetscFunctionBegin;
    2493:   PetscCall(DMCreate(comm, dm));
    2494:   PetscCall(DMSetType(*dm, DMPLEX));
    2495:   PetscCall(PetscMalloc4(dim, &edg, dim, &low, dim, &upp, dim, &bdt));
    2496:   for (d = 0; d < dim; ++d) {
    2497:     edg[d] = edges ? edges[d] : 1;
    2498:     low[d] = lower ? lower[d] : 0.;
    2499:     upp[d] = upper ? upper[d] : 1.;
    2500:     bdt[d] = DM_BOUNDARY_PERIODIC;
    2501:   }
    2502:   PetscCall(DMPlexCreateHypercubicMesh_Internal(*dm, dim, low, upp, edg, overlap, bdt));
    2503:   PetscCall(PetscFree4(edg, low, upp, bdt));
    2504:   PetscFunctionReturn(PETSC_SUCCESS);
    2505: }

    2507: /*@
    2508:   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.

    2510:   Logically Collective

    2512:   Input Parameters:
    2513: + dm     - the `DM` context
    2514: - prefix - the prefix to prepend to all option names

    2516:   Level: advanced

    2518:   Note:
    2519:   A hyphen (-) must NOT be given at the beginning of the prefix name.
    2520:   The first character of all runtime options is AUTOMATICALLY the hyphen.

    2522: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
    2523: @*/
    2524: PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
    2525: {
    2526:   DM_Plex *mesh = (DM_Plex *)dm->data;

    2528:   PetscFunctionBegin;
    2530:   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
    2531:   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
    2532:   PetscFunctionReturn(PETSC_SUCCESS);
    2533: }

    2535: /* Remap geometry to cylinder
    2536:    TODO: This only works for a single refinement, then it is broken

    2538:      Interior square: Linear interpolation is correct
    2539:      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
    2540:      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance

    2542:        phi     = arctan(y/x)
    2543:        d_close = sqrt(1/8 + 1/4 sin^2(phi))
    2544:        d_far   = sqrt(1/2 + sin^2(phi))

    2546:      so we remap them using

    2548:        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
    2549:        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)

    2551:      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
    2552: */
    2553: static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
    2554: {
    2555:   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
    2556:   const PetscReal ds2 = 0.5 * dis;

    2558:   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
    2559:     f0[0] = u[0];
    2560:     f0[1] = u[1];
    2561:   } else {
    2562:     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;

    2564:     x    = PetscRealPart(u[0]);
    2565:     y    = PetscRealPart(u[1]);
    2566:     phi  = PetscAtan2Real(y, x);
    2567:     sinp = PetscSinReal(phi);
    2568:     cosp = PetscCosReal(phi);
    2569:     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
    2570:       dc = PetscAbsReal(ds2 / sinp);
    2571:       df = PetscAbsReal(dis / sinp);
    2572:       xc = ds2 * x / PetscAbsReal(y);
    2573:       yc = ds2 * PetscSignReal(y);
    2574:     } else {
    2575:       dc = PetscAbsReal(ds2 / cosp);
    2576:       df = PetscAbsReal(dis / cosp);
    2577:       xc = ds2 * PetscSignReal(x);
    2578:       yc = ds2 * y / PetscAbsReal(x);
    2579:     }
    2580:     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
    2581:     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
    2582:   }
    2583:   f0[2] = u[2];
    2584: }

    2586: static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ, PetscInt Nr)
    2587: {
    2588:   const PetscInt dim = 3;
    2589:   PetscInt       numCells, numVertices;
    2590:   PetscMPIInt    rank;

    2592:   PetscFunctionBegin;
    2593:   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
    2594:   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
    2595:   PetscCall(DMSetDimension(dm, dim));
    2596:   /* Create topology */
    2597:   {
    2598:     PetscInt cone[8], c;

    2600:     numCells    = rank == 0 ? 5 : 0;
    2601:     numVertices = rank == 0 ? 16 : 0;
    2602:     if (periodicZ == DM_BOUNDARY_PERIODIC) {
    2603:       numCells *= 3;
    2604:       numVertices = rank == 0 ? 24 : 0;
    2605:     }
    2606:     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
    2607:     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
    2608:     PetscCall(DMSetUp(dm));
    2609:     if (rank == 0) {
    2610:       if (periodicZ == DM_BOUNDARY_PERIODIC) {
    2611:         cone[0] = 15;
    2612:         cone[1] = 18;
    2613:         cone[2] = 17;
    2614:         cone[3] = 16;
    2615:         cone[4] = 31;
    2616:         cone[5] = 32;
    2617:         cone[6] = 33;
    2618:         cone[7] = 34;
    2619:         PetscCall(DMPlexSetCone(dm, 0, cone));
    2620:         cone[0] = 16;
    2621:         cone[1] = 17;
    2622:         cone[2] = 24;
    2623:         cone[3] = 23;
    2624:         cone[4] = 32;
    2625:         cone[5] = 36;
    2626:         cone[6] = 37;
    2627:         cone[7] = 33; /* 22 25 26 21 */
    2628:         PetscCall(DMPlexSetCone(dm, 1, cone));
    2629:         cone[0] = 18;
    2630:         cone[1] = 27;
    2631:         cone[2] = 24;
    2632:         cone[3] = 17;
    2633:         cone[4] = 34;
    2634:         cone[5] = 33;
    2635:         cone[6] = 37;
    2636:         cone[7] = 38;
    2637:         PetscCall(DMPlexSetCone(dm, 2, cone));
    2638:         cone[0] = 29;
    2639:         cone[1] = 27;
    2640:         cone[2] = 18;
    2641:         cone[3] = 15;
    2642:         cone[4] = 35;
    2643:         cone[5] = 31;
    2644:         cone[6] = 34;
    2645:         cone[7] = 38;
    2646:         PetscCall(DMPlexSetCone(dm, 3, cone));
    2647:         cone[0] = 29;
    2648:         cone[1] = 15;
    2649:         cone[2] = 16;
    2650:         cone[3] = 23;
    2651:         cone[4] = 35;
    2652:         cone[5] = 36;
    2653:         cone[6] = 32;
    2654:         cone[7] = 31;
    2655:         PetscCall(DMPlexSetCone(dm, 4, cone));

    2657:         cone[0] = 31;
    2658:         cone[1] = 34;
    2659:         cone[2] = 33;
    2660:         cone[3] = 32;
    2661:         cone[4] = 19;
    2662:         cone[5] = 22;
    2663:         cone[6] = 21;
    2664:         cone[7] = 20;
    2665:         PetscCall(DMPlexSetCone(dm, 5, cone));
    2666:         cone[0] = 32;
    2667:         cone[1] = 33;
    2668:         cone[2] = 37;
    2669:         cone[3] = 36;
    2670:         cone[4] = 22;
    2671:         cone[5] = 25;
    2672:         cone[6] = 26;
    2673:         cone[7] = 21;
    2674:         PetscCall(DMPlexSetCone(dm, 6, cone));
    2675:         cone[0] = 34;
    2676:         cone[1] = 38;
    2677:         cone[2] = 37;
    2678:         cone[3] = 33;
    2679:         cone[4] = 20;
    2680:         cone[5] = 21;
    2681:         cone[6] = 26;
    2682:         cone[7] = 28;
    2683:         PetscCall(DMPlexSetCone(dm, 7, cone));
    2684:         cone[0] = 35;
    2685:         cone[1] = 38;
    2686:         cone[2] = 34;
    2687:         cone[3] = 31;
    2688:         cone[4] = 30;
    2689:         cone[5] = 19;
    2690:         cone[6] = 20;
    2691:         cone[7] = 28;
    2692:         PetscCall(DMPlexSetCone(dm, 8, cone));
    2693:         cone[0] = 35;
    2694:         cone[1] = 31;
    2695:         cone[2] = 32;
    2696:         cone[3] = 36;
    2697:         cone[4] = 30;
    2698:         cone[5] = 25;
    2699:         cone[6] = 22;
    2700:         cone[7] = 19;
    2701:         PetscCall(DMPlexSetCone(dm, 9, cone));

    2703:         cone[0] = 19;
    2704:         cone[1] = 20;
    2705:         cone[2] = 21;
    2706:         cone[3] = 22;
    2707:         cone[4] = 15;
    2708:         cone[5] = 16;
    2709:         cone[6] = 17;
    2710:         cone[7] = 18;
    2711:         PetscCall(DMPlexSetCone(dm, 10, cone));
    2712:         cone[0] = 22;
    2713:         cone[1] = 21;
    2714:         cone[2] = 26;
    2715:         cone[3] = 25;
    2716:         cone[4] = 16;
    2717:         cone[5] = 23;
    2718:         cone[6] = 24;
    2719:         cone[7] = 17;
    2720:         PetscCall(DMPlexSetCone(dm, 11, cone));
    2721:         cone[0] = 20;
    2722:         cone[1] = 28;
    2723:         cone[2] = 26;
    2724:         cone[3] = 21;
    2725:         cone[4] = 18;
    2726:         cone[5] = 17;
    2727:         cone[6] = 24;
    2728:         cone[7] = 27;
    2729:         PetscCall(DMPlexSetCone(dm, 12, cone));
    2730:         cone[0] = 30;
    2731:         cone[1] = 28;
    2732:         cone[2] = 20;
    2733:         cone[3] = 19;
    2734:         cone[4] = 29;
    2735:         cone[5] = 15;
    2736:         cone[6] = 18;
    2737:         cone[7] = 27;
    2738:         PetscCall(DMPlexSetCone(dm, 13, cone));
    2739:         cone[0] = 30;
    2740:         cone[1] = 19;
    2741:         cone[2] = 22;
    2742:         cone[3] = 25;
    2743:         cone[4] = 29;
    2744:         cone[5] = 23;
    2745:         cone[6] = 16;
    2746:         cone[7] = 15;
    2747:         PetscCall(DMPlexSetCone(dm, 14, cone));
    2748:       } else {
    2749:         cone[0] = 5;
    2750:         cone[1] = 8;
    2751:         cone[2] = 7;
    2752:         cone[3] = 6;
    2753:         cone[4] = 9;
    2754:         cone[5] = 12;
    2755:         cone[6] = 11;
    2756:         cone[7] = 10;
    2757:         PetscCall(DMPlexSetCone(dm, 0, cone));
    2758:         cone[0] = 6;
    2759:         cone[1] = 7;
    2760:         cone[2] = 14;
    2761:         cone[3] = 13;
    2762:         cone[4] = 12;
    2763:         cone[5] = 15;
    2764:         cone[6] = 16;
    2765:         cone[7] = 11;
    2766:         PetscCall(DMPlexSetCone(dm, 1, cone));
    2767:         cone[0] = 8;
    2768:         cone[1] = 17;
    2769:         cone[2] = 14;
    2770:         cone[3] = 7;
    2771:         cone[4] = 10;
    2772:         cone[5] = 11;
    2773:         cone[6] = 16;
    2774:         cone[7] = 18;
    2775:         PetscCall(DMPlexSetCone(dm, 2, cone));
    2776:         cone[0] = 19;
    2777:         cone[1] = 17;
    2778:         cone[2] = 8;
    2779:         cone[3] = 5;
    2780:         cone[4] = 20;
    2781:         cone[5] = 9;
    2782:         cone[6] = 10;
    2783:         cone[7] = 18;
    2784:         PetscCall(DMPlexSetCone(dm, 3, cone));
    2785:         cone[0] = 19;
    2786:         cone[1] = 5;
    2787:         cone[2] = 6;
    2788:         cone[3] = 13;
    2789:         cone[4] = 20;
    2790:         cone[5] = 15;
    2791:         cone[6] = 12;
    2792:         cone[7] = 9;
    2793:         PetscCall(DMPlexSetCone(dm, 4, cone));
    2794:       }
    2795:     }
    2796:     PetscCall(DMPlexSymmetrize(dm));
    2797:     PetscCall(DMPlexStratify(dm));
    2798:   }
    2799:   /* Create cube geometry */
    2800:   {
    2801:     Vec             coordinates;
    2802:     PetscSection    coordSection;
    2803:     PetscScalar    *coords;
    2804:     PetscInt        coordSize, v;
    2805:     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
    2806:     const PetscReal ds2 = dis / 2.0;

    2808:     /* Build coordinates */
    2809:     PetscCall(DMGetCoordinateSection(dm, &coordSection));
    2810:     PetscCall(PetscSectionSetNumFields(coordSection, 1));
    2811:     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
    2812:     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
    2813:     for (v = numCells; v < numCells + numVertices; ++v) {
    2814:       PetscCall(PetscSectionSetDof(coordSection, v, dim));
    2815:       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
    2816:     }
    2817:     PetscCall(PetscSectionSetUp(coordSection));
    2818:     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
    2819:     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
    2820:     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    2821:     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
    2822:     PetscCall(VecSetBlockSize(coordinates, dim));
    2823:     PetscCall(VecSetType(coordinates, VECSTANDARD));
    2824:     PetscCall(VecGetArray(coordinates, &coords));
    2825:     if (rank == 0) {
    2826:       coords[0 * dim + 0]  = -ds2;
    2827:       coords[0 * dim + 1]  = -ds2;
    2828:       coords[0 * dim + 2]  = 0.0;
    2829:       coords[1 * dim + 0]  = ds2;
    2830:       coords[1 * dim + 1]  = -ds2;
    2831:       coords[1 * dim + 2]  = 0.0;
    2832:       coords[2 * dim + 0]  = ds2;
    2833:       coords[2 * dim + 1]  = ds2;
    2834:       coords[2 * dim + 2]  = 0.0;
    2835:       coords[3 * dim + 0]  = -ds2;
    2836:       coords[3 * dim + 1]  = ds2;
    2837:       coords[3 * dim + 2]  = 0.0;
    2838:       coords[4 * dim + 0]  = -ds2;
    2839:       coords[4 * dim + 1]  = -ds2;
    2840:       coords[4 * dim + 2]  = 1.0;
    2841:       coords[5 * dim + 0]  = -ds2;
    2842:       coords[5 * dim + 1]  = ds2;
    2843:       coords[5 * dim + 2]  = 1.0;
    2844:       coords[6 * dim + 0]  = ds2;
    2845:       coords[6 * dim + 1]  = ds2;
    2846:       coords[6 * dim + 2]  = 1.0;
    2847:       coords[7 * dim + 0]  = ds2;
    2848:       coords[7 * dim + 1]  = -ds2;
    2849:       coords[7 * dim + 2]  = 1.0;
    2850:       coords[8 * dim + 0]  = dis;
    2851:       coords[8 * dim + 1]  = -dis;
    2852:       coords[8 * dim + 2]  = 0.0;
    2853:       coords[9 * dim + 0]  = dis;
    2854:       coords[9 * dim + 1]  = dis;
    2855:       coords[9 * dim + 2]  = 0.0;
    2856:       coords[10 * dim + 0] = dis;
    2857:       coords[10 * dim + 1] = -dis;
    2858:       coords[10 * dim + 2] = 1.0;
    2859:       coords[11 * dim + 0] = dis;
    2860:       coords[11 * dim + 1] = dis;
    2861:       coords[11 * dim + 2] = 1.0;
    2862:       coords[12 * dim + 0] = -dis;
    2863:       coords[12 * dim + 1] = dis;
    2864:       coords[12 * dim + 2] = 0.0;
    2865:       coords[13 * dim + 0] = -dis;
    2866:       coords[13 * dim + 1] = dis;
    2867:       coords[13 * dim + 2] = 1.0;
    2868:       coords[14 * dim + 0] = -dis;
    2869:       coords[14 * dim + 1] = -dis;
    2870:       coords[14 * dim + 2] = 0.0;
    2871:       coords[15 * dim + 0] = -dis;
    2872:       coords[15 * dim + 1] = -dis;
    2873:       coords[15 * dim + 2] = 1.0;
    2874:       if (periodicZ == DM_BOUNDARY_PERIODIC) {
    2875:         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
    2876:         coords[16 * dim + 1]                = -ds2;
    2877:         coords[16 * dim + 2]                = 0.5;
    2878:         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
    2879:         coords[17 * dim + 1]                = -ds2;
    2880:         coords[17 * dim + 2]                = 0.5;
    2881:         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
    2882:         coords[18 * dim + 1]                = ds2;
    2883:         coords[18 * dim + 2]                = 0.5;
    2884:         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
    2885:         coords[19 * dim + 1]                = ds2;
    2886:         coords[19 * dim + 2]                = 0.5;
    2887:         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
    2888:         coords[20 * dim + 1]                = -dis;
    2889:         coords[20 * dim + 2]                = 0.5;
    2890:         /* 23 36 25 */ coords[21 * dim + 0] = dis;
    2891:         coords[21 * dim + 1]                = -dis;
    2892:         coords[21 * dim + 2]                = 0.5;
    2893:         /* 24 37 26 */ coords[22 * dim + 0] = dis;
    2894:         coords[22 * dim + 1]                = dis;
    2895:         coords[22 * dim + 2]                = 0.5;
    2896:         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
    2897:         coords[23 * dim + 1]                = dis;
    2898:         coords[23 * dim + 2]                = 0.5;
    2899:       }
    2900:     }
    2901:     PetscCall(VecRestoreArray(coordinates, &coords));
    2902:     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
    2903:     PetscCall(VecDestroy(&coordinates));
    2904:   }
    2905:   /* Create periodicity */
    2906:   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
    2907:     PetscReal L[3]       = {-1., -1., 0.};
    2908:     PetscReal maxCell[3] = {-1., -1., 0.};
    2909:     PetscReal lower[3]   = {0.0, 0.0, 0.0};
    2910:     PetscReal upper[3]   = {1.0, 1.0, 1.5};
    2911:     PetscInt  numZCells  = 3;

    2913:     L[2]       = upper[2] - lower[2];
    2914:     maxCell[2] = 1.1 * (L[2] / numZCells);
    2915:     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
    2916:   }
    2917:   {
    2918:     DM          cdm;
    2919:     PetscDS     cds;
    2920:     PetscScalar c[2] = {1.0, 1.0};

    2922:     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_FALSE, PETSC_TRUE));
    2923:     PetscCall(DMGetCoordinateDM(dm, &cdm));
    2924:     PetscCall(DMGetDS(cdm, &cds));
    2925:     PetscCall(PetscDSSetConstants(cds, 2, c));
    2926:   }
    2927:   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));

    2929:   /* Wait for coordinate creation before doing in-place modification */
    2930:   PetscCall(DMPlexInterpolateInPlace_Internal(dm));

    2932:   char        oldprefix[PETSC_MAX_PATH_LEN];
    2933:   const char *prefix;

    2935:   PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
    2936:   PetscCall(PetscStrncpy(oldprefix, prefix, PETSC_MAX_PATH_LEN));
    2937:   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, "petsc_cyl_ref_"));
    2938:   for (PetscInt r = 0; r < PetscMax(0, Nr); ++r) {
    2939:     DM rdm;

    2941:     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
    2942:     PetscCall(DMPlexReplace_Internal(dm, &rdm));
    2943:   }
    2944:   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldprefix));
    2945:   PetscCall(DMPlexRemapGeometry(dm, 0.0, snapToCylinder));

    2947:   DMLabel         bdlabel, edgelabel;
    2948:   IS              faceIS;
    2949:   const PetscInt *faces;
    2950:   PetscInt        Nf;

    2952:   PetscCall(DMCreateLabel(dm, "marker"));
    2953:   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
    2954:   PetscCall(DMCreateLabel(dm, "generatrix"));
    2955:   PetscCall(DMGetLabel(dm, "generatrix", &edgelabel));
    2956:   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
    2957:   // Remove faces on top and bottom
    2958:   PetscCall(DMLabelGetStratumIS(bdlabel, 1, &faceIS));
    2959:   if (faceIS) {
    2960:     PetscCall(ISGetLocalSize(faceIS, &Nf));
    2961:     PetscCall(ISGetIndices(faceIS, &faces));
    2962:     for (PetscInt f = 0; f < Nf; ++f) {
    2963:       PetscReal vol, normal[3];

    2965:       PetscCall(DMPlexComputeCellGeometryFVM(dm, faces[f], &vol, NULL, normal));
    2966:       if (PetscAbsReal(normal[2]) < PETSC_SMALL) PetscCall(DMLabelSetValue(edgelabel, faces[f], 1));
    2967:     }
    2968:     PetscCall(ISRestoreIndices(faceIS, &faces));
    2969:     PetscCall(ISDestroy(&faceIS));
    2970:   }
    2971:   PetscCall(DMPlexLabelComplete(dm, bdlabel));
    2972:   PetscCall(DMPlexLabelComplete(dm, edgelabel));
    2973:   PetscFunctionReturn(PETSC_SUCCESS);
    2974: }

    2976: /*@
    2977:   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.

    2979:   Collective

    2981:   Input Parameters:
    2982: + comm      - The communicator for the `DM` object
    2983: . periodicZ - The boundary type for the Z direction
    2984: - Nr        - The number of refinements to carry out

    2986:   Output Parameter:
    2987: . dm - The `DM` object

    2989:   Level: beginner

    2991:   Note:
    2992:   Here is the output numbering looking from the bottom of the cylinder\:
    2993: .vb
    2994:        17-----14
    2995:         |     |
    2996:         |  2  |
    2997:         |     |
    2998:  17-----8-----7-----14
    2999:   |     |     |     |
    3000:   |  3  |  0  |  1  |
    3001:   |     |     |     |
    3002:  19-----5-----6-----13
    3003:         |     |
    3004:         |  4  |
    3005:         |     |
    3006:        19-----13

    3008:  and up through the top

    3010:        18-----16
    3011:         |     |
    3012:         |  2  |
    3013:         |     |
    3014:  18----10----11-----16
    3015:   |     |     |     |
    3016:   |  3  |  0  |  1  |
    3017:   |     |     |     |
    3018:  20-----9----12-----15
    3019:         |     |
    3020:         |  4  |
    3021:         |     |
    3022:        20-----15
    3023: .ve

    3025: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
    3026: @*/
    3027: PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, PetscInt Nr, DM *dm)
    3028: {
    3029:   PetscFunctionBegin;
    3030:   PetscAssertPointer(dm, 4);
    3031:   PetscCall(DMCreate(comm, dm));
    3032:   PetscCall(DMSetType(*dm, DMPLEX));
    3033:   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ, Nr));
    3034:   PetscFunctionReturn(PETSC_SUCCESS);
    3035: }

    3037: static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
    3038: {
    3039:   const PetscInt dim = 3;
    3040:   PetscInt       numCells, numVertices, v;
    3041:   PetscMPIInt    rank;

    3043:   PetscFunctionBegin;
    3044:   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
    3045:   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
    3046:   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
    3047:   PetscCall(DMSetDimension(dm, dim));
    3048:   /* Must create the celltype label here so that we do not automatically try to compute the types */
    3049:   PetscCall(DMCreateLabel(dm, "celltype"));
    3050:   /* Create topology */
    3051:   {
    3052:     PetscInt cone[6], c;

    3054:     numCells    = rank == 0 ? n : 0;
    3055:     numVertices = rank == 0 ? 2 * (n + 1) : 0;
    3056:     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
    3057:     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
    3058:     PetscCall(DMSetUp(dm));
    3059:     for (c = 0; c < numCells; c++) {
    3060:       cone[0] = c + n * 1;
    3061:       cone[1] = (c + 1) % n + n * 1;
    3062:       cone[2] = 0 + 3 * n;
    3063:       cone[3] = c + n * 2;
    3064:       cone[4] = (c + 1) % n + n * 2;
    3065:       cone[5] = 1 + 3 * n;
    3066:       PetscCall(DMPlexSetCone(dm, c, cone));
    3067:       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
    3068:     }
    3069:     PetscCall(DMPlexSymmetrize(dm));
    3070:     PetscCall(DMPlexStratify(dm));
    3071:   }
    3072:   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
    3073:   /* Create cylinder geometry */
    3074:   {
    3075:     Vec          coordinates;
    3076:     PetscSection coordSection;
    3077:     PetscScalar *coords;
    3078:     PetscInt     coordSize, c;

    3080:     /* Build coordinates */
    3081:     PetscCall(DMGetCoordinateSection(dm, &coordSection));
    3082:     PetscCall(PetscSectionSetNumFields(coordSection, 1));
    3083:     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
    3084:     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
    3085:     for (v = numCells; v < numCells + numVertices; ++v) {
    3086:       PetscCall(PetscSectionSetDof(coordSection, v, dim));
    3087:       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
    3088:     }
    3089:     PetscCall(PetscSectionSetUp(coordSection));
    3090:     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
    3091:     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
    3092:     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    3093:     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
    3094:     PetscCall(VecSetBlockSize(coordinates, dim));
    3095:     PetscCall(VecSetType(coordinates, VECSTANDARD));
    3096:     PetscCall(VecGetArray(coordinates, &coords));
    3097:     for (c = 0; c < numCells; c++) {
    3098:       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
    3099:       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
    3100:       coords[(c + 0 * n) * dim + 2] = 1.0;
    3101:       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
    3102:       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
    3103:       coords[(c + 1 * n) * dim + 2] = 0.0;
    3104:     }
    3105:     if (rank == 0) {
    3106:       coords[(2 * n + 0) * dim + 0] = 0.0;
    3107:       coords[(2 * n + 0) * dim + 1] = 0.0;
    3108:       coords[(2 * n + 0) * dim + 2] = 1.0;
    3109:       coords[(2 * n + 1) * dim + 0] = 0.0;
    3110:       coords[(2 * n + 1) * dim + 1] = 0.0;
    3111:       coords[(2 * n + 1) * dim + 2] = 0.0;
    3112:     }
    3113:     PetscCall(VecRestoreArray(coordinates, &coords));
    3114:     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
    3115:     PetscCall(VecDestroy(&coordinates));
    3116:   }
    3117:   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
    3118:   /* Interpolate */
    3119:   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
    3120:   PetscFunctionReturn(PETSC_SUCCESS);
    3121: }

    3123: /*@
    3124:   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.

    3126:   Collective

    3128:   Input Parameters:
    3129: + comm        - The communicator for the `DM` object
    3130: . n           - The number of wedges around the origin
    3131: - interpolate - Create edges and faces

    3133:   Output Parameter:
    3134: . dm - The `DM` object

    3136:   Level: beginner

    3138: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
    3139: @*/
    3140: PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
    3141: {
    3142:   PetscFunctionBegin;
    3143:   PetscAssertPointer(dm, 4);
    3144:   PetscCall(DMCreate(comm, dm));
    3145:   PetscCall(DMSetType(*dm, DMPLEX));
    3146:   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
    3147:   PetscFunctionReturn(PETSC_SUCCESS);
    3148: }

    3150: static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
    3151: {
    3152:   PetscReal prod = 0.0;
    3153:   PetscInt  i;
    3154:   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
    3155:   return PetscSqrtReal(prod);
    3156: }

    3158: static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
    3159: {
    3160:   PetscReal prod = 0.0;
    3161:   PetscInt  i;
    3162:   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
    3163:   return prod;
    3164: }

    3166: /* The first constant is the sphere radius */
    3167: static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
    3168: {
    3169:   PetscReal r     = PetscRealPart(constants[0]);
    3170:   PetscReal norm2 = 0.0, fac;
    3171:   PetscInt  n     = uOff[1] - uOff[0], d;

    3173:   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
    3174:   fac = r / PetscSqrtReal(norm2);
    3175:   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
    3176: }

    3178: static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
    3179: {
    3180:   const PetscInt embedDim = dim + 1;
    3181:   PetscSection   coordSection;
    3182:   Vec            coordinates;
    3183:   PetscScalar   *coords;
    3184:   PetscReal     *coordsIn;
    3185:   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, e;
    3186:   PetscMPIInt    rank;

    3188:   PetscFunctionBegin;
    3190:   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
    3191:   PetscCall(DMSetDimension(dm, dim));
    3192:   PetscCall(DMSetCoordinateDim(dm, dim + 1));
    3193:   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
    3194:   switch (dim) {
    3195:   case 1:
    3196:     numCells = 16;
    3197:     numVerts = numCells;

    3199:     // Build Topology
    3200:     PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
    3201:     for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
    3202:     PetscCall(DMSetUp(dm));
    3203:     for (PetscInt c = 0; c < numCells; ++c) {
    3204:       PetscInt cone[2];

    3206:       cone[0] = c + numCells;
    3207:       cone[1] = (c + 1) % numVerts + numCells;
    3208:       PetscCall(DMPlexSetCone(dm, c, cone));
    3209:     }
    3210:     PetscCall(DMPlexSymmetrize(dm));
    3211:     PetscCall(DMPlexStratify(dm));
    3212:     PetscCall(PetscMalloc1(numVerts * embedDim, &coordsIn));
    3213:     for (PetscInt v = 0; v < numVerts; ++v) {
    3214:       const PetscReal rad = 2. * PETSC_PI * v / numVerts;

    3216:       coordsIn[v * embedDim + 0] = PetscCosReal(rad);
    3217:       coordsIn[v * embedDim + 1] = PetscSinReal(rad);
    3218:     }
    3219:     break;
    3220:   case 2:
    3221:     if (simplex) {
    3222:       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
    3223:       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
    3224:       const PetscInt  degree    = 5;
    3225:       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
    3226:       PetscInt        s[3]      = {1, 1, 1};
    3227:       PetscInt        cone[3];
    3228:       PetscInt       *graph;

    3230:       vertex[0] *= R / radius;
    3231:       vertex[1] *= R / radius;
    3232:       vertex[2] *= R / radius;
    3233:       numCells    = rank == 0 ? 20 : 0;
    3234:       numVerts    = rank == 0 ? 12 : 0;
    3235:       firstVertex = numCells;
    3236:       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of

    3238:            (0, \pm 1/\phi+1, \pm \phi/\phi+1)

    3240:          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
    3241:          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
    3242:       */
    3243:       /* Construct vertices */
    3244:       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
    3245:       if (rank == 0) {
    3246:         for (PetscInt p = 0, i = 0; p < embedDim; ++p) {
    3247:           for (s[1] = -1; s[1] < 2; s[1] += 2) {
    3248:             for (s[2] = -1; s[2] < 2; s[2] += 2) {
    3249:               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
    3250:               ++i;
    3251:             }
    3252:           }
    3253:         }
    3254:       }
    3255:       /* Construct graph */
    3256:       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
    3257:       for (PetscInt i = 0; i < numVerts; ++i) {
    3258:         PetscInt k = 0;
    3259:         for (PetscInt j = 0; j < numVerts; ++j) {
    3260:           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
    3261:             graph[i * numVerts + j] = 1;
    3262:             ++k;
    3263:           }
    3264:         }
    3265:         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
    3266:       }
    3267:       /* Build Topology */
    3268:       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
    3269:       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
    3270:       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
    3271:       /* Cells */
    3272:       for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
    3273:         for (PetscInt j = 0; j < i; ++j) {
    3274:           for (PetscInt k = 0; k < j; ++k) {
    3275:             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
    3276:               cone[0] = firstVertex + i;
    3277:               cone[1] = firstVertex + j;
    3278:               cone[2] = firstVertex + k;
    3279:               /* Check orientation */
    3280:               {
    3281:                 const PetscInt epsilon[3][3][3] = {
    3282:                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
    3283:                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
    3284:                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
    3285:                 };
    3286:                 PetscReal normal[3];
    3287:                 PetscInt  e, f;

    3289:                 for (d = 0; d < embedDim; ++d) {
    3290:                   normal[d] = 0.0;
    3291:                   for (e = 0; e < embedDim; ++e) {
    3292:                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
    3293:                   }
    3294:                 }
    3295:                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
    3296:                   PetscInt tmp = cone[1];
    3297:                   cone[1]      = cone[2];
    3298:                   cone[2]      = tmp;
    3299:                 }
    3300:               }
    3301:               PetscCall(DMPlexSetCone(dm, c++, cone));
    3302:             }
    3303:           }
    3304:         }
    3305:       }
    3306:       PetscCall(DMPlexSymmetrize(dm));
    3307:       PetscCall(DMPlexStratify(dm));
    3308:       PetscCall(PetscFree(graph));
    3309:     } else {
    3310:       /*
    3311:         12-21--13
    3312:          |     |
    3313:         25  4  24
    3314:          |     |
    3315:   12-25--9-16--8-24--13
    3316:    |     |     |     |
    3317:   23  5 17  0 15  3  22
    3318:    |     |     |     |
    3319:   10-20--6-14--7-19--11
    3320:          |     |
    3321:         20  1  19
    3322:          |     |
    3323:         10-18--11
    3324:          |     |
    3325:         23  2  22
    3326:          |     |
    3327:         12-21--13
    3328:        */
    3329:       PetscInt cone[4], ornt[4];

    3331:       numCells    = rank == 0 ? 6 : 0;
    3332:       numEdges    = rank == 0 ? 12 : 0;
    3333:       numVerts    = rank == 0 ? 8 : 0;
    3334:       firstVertex = numCells;
    3335:       firstEdge   = numCells + numVerts;
    3336:       /* Build Topology */
    3337:       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
    3338:       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
    3339:       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
    3340:       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
    3341:       if (rank == 0) {
    3342:         /* Cell 0 */
    3343:         cone[0] = 14;
    3344:         cone[1] = 15;
    3345:         cone[2] = 16;
    3346:         cone[3] = 17;
    3347:         PetscCall(DMPlexSetCone(dm, 0, cone));
    3348:         ornt[0] = 0;
    3349:         ornt[1] = 0;
    3350:         ornt[2] = 0;
    3351:         ornt[3] = 0;
    3352:         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
    3353:         /* Cell 1 */
    3354:         cone[0] = 18;
    3355:         cone[1] = 19;
    3356:         cone[2] = 14;
    3357:         cone[3] = 20;
    3358:         PetscCall(DMPlexSetCone(dm, 1, cone));
    3359:         ornt[0] = 0;
    3360:         ornt[1] = 0;
    3361:         ornt[2] = -1;
    3362:         ornt[3] = 0;
    3363:         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
    3364:         /* Cell 2 */
    3365:         cone[0] = 21;
    3366:         cone[1] = 22;
    3367:         cone[2] = 18;
    3368:         cone[3] = 23;
    3369:         PetscCall(DMPlexSetCone(dm, 2, cone));
    3370:         ornt[0] = 0;
    3371:         ornt[1] = 0;
    3372:         ornt[2] = -1;
    3373:         ornt[3] = 0;
    3374:         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
    3375:         /* Cell 3 */
    3376:         cone[0] = 19;
    3377:         cone[1] = 22;
    3378:         cone[2] = 24;
    3379:         cone[3] = 15;
    3380:         PetscCall(DMPlexSetCone(dm, 3, cone));
    3381:         ornt[0] = -1;
    3382:         ornt[1] = -1;
    3383:         ornt[2] = 0;
    3384:         ornt[3] = -1;
    3385:         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
    3386:         /* Cell 4 */
    3387:         cone[0] = 16;
    3388:         cone[1] = 24;
    3389:         cone[2] = 21;
    3390:         cone[3] = 25;
    3391:         PetscCall(DMPlexSetCone(dm, 4, cone));
    3392:         ornt[0] = -1;
    3393:         ornt[1] = -1;
    3394:         ornt[2] = -1;
    3395:         ornt[3] = 0;
    3396:         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
    3397:         /* Cell 5 */
    3398:         cone[0] = 20;
    3399:         cone[1] = 17;
    3400:         cone[2] = 25;
    3401:         cone[3] = 23;
    3402:         PetscCall(DMPlexSetCone(dm, 5, cone));
    3403:         ornt[0] = -1;
    3404:         ornt[1] = -1;
    3405:         ornt[2] = -1;
    3406:         ornt[3] = -1;
    3407:         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
    3408:         /* Edges */
    3409:         cone[0] = 6;
    3410:         cone[1] = 7;
    3411:         PetscCall(DMPlexSetCone(dm, 14, cone));
    3412:         cone[0] = 7;
    3413:         cone[1] = 8;
    3414:         PetscCall(DMPlexSetCone(dm, 15, cone));
    3415:         cone[0] = 8;
    3416:         cone[1] = 9;
    3417:         PetscCall(DMPlexSetCone(dm, 16, cone));
    3418:         cone[0] = 9;
    3419:         cone[1] = 6;
    3420:         PetscCall(DMPlexSetCone(dm, 17, cone));
    3421:         cone[0] = 10;
    3422:         cone[1] = 11;
    3423:         PetscCall(DMPlexSetCone(dm, 18, cone));
    3424:         cone[0] = 11;
    3425:         cone[1] = 7;
    3426:         PetscCall(DMPlexSetCone(dm, 19, cone));
    3427:         cone[0] = 6;
    3428:         cone[1] = 10;
    3429:         PetscCall(DMPlexSetCone(dm, 20, cone));
    3430:         cone[0] = 12;
    3431:         cone[1] = 13;
    3432:         PetscCall(DMPlexSetCone(dm, 21, cone));
    3433:         cone[0] = 13;
    3434:         cone[1] = 11;
    3435:         PetscCall(DMPlexSetCone(dm, 22, cone));
    3436:         cone[0] = 10;
    3437:         cone[1] = 12;
    3438:         PetscCall(DMPlexSetCone(dm, 23, cone));
    3439:         cone[0] = 13;
    3440:         cone[1] = 8;
    3441:         PetscCall(DMPlexSetCone(dm, 24, cone));
    3442:         cone[0] = 12;
    3443:         cone[1] = 9;
    3444:         PetscCall(DMPlexSetCone(dm, 25, cone));
    3445:       }
    3446:       PetscCall(DMPlexSymmetrize(dm));
    3447:       PetscCall(DMPlexStratify(dm));
    3448:       /* Build coordinates */
    3449:       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
    3450:       if (rank == 0) {
    3451:         coordsIn[0 * embedDim + 0] = -R;
    3452:         coordsIn[0 * embedDim + 1] = R;
    3453:         coordsIn[0 * embedDim + 2] = -R;
    3454:         coordsIn[1 * embedDim + 0] = R;
    3455:         coordsIn[1 * embedDim + 1] = R;
    3456:         coordsIn[1 * embedDim + 2] = -R;
    3457:         coordsIn[2 * embedDim + 0] = R;
    3458:         coordsIn[2 * embedDim + 1] = -R;
    3459:         coordsIn[2 * embedDim + 2] = -R;
    3460:         coordsIn[3 * embedDim + 0] = -R;
    3461:         coordsIn[3 * embedDim + 1] = -R;
    3462:         coordsIn[3 * embedDim + 2] = -R;
    3463:         coordsIn[4 * embedDim + 0] = -R;
    3464:         coordsIn[4 * embedDim + 1] = R;
    3465:         coordsIn[4 * embedDim + 2] = R;
    3466:         coordsIn[5 * embedDim + 0] = R;
    3467:         coordsIn[5 * embedDim + 1] = R;
    3468:         coordsIn[5 * embedDim + 2] = R;
    3469:         coordsIn[6 * embedDim + 0] = -R;
    3470:         coordsIn[6 * embedDim + 1] = -R;
    3471:         coordsIn[6 * embedDim + 2] = R;
    3472:         coordsIn[7 * embedDim + 0] = R;
    3473:         coordsIn[7 * embedDim + 1] = -R;
    3474:         coordsIn[7 * embedDim + 2] = R;
    3475:       }
    3476:     }
    3477:     break;
    3478:   case 3:
    3479:     if (simplex) {
    3480:       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
    3481:       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
    3482:       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
    3483:       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
    3484:       const PetscInt  degree          = 12;
    3485:       PetscInt        s[4]            = {1, 1, 1};
    3486:       PetscInt        evenPerm[12][4] = {
    3487:         {0, 1, 2, 3},
    3488:         {0, 2, 3, 1},
    3489:         {0, 3, 1, 2},
    3490:         {1, 0, 3, 2},
    3491:         {1, 2, 0, 3},
    3492:         {1, 3, 2, 0},
    3493:         {2, 0, 1, 3},
    3494:         {2, 1, 3, 0},
    3495:         {2, 3, 0, 1},
    3496:         {3, 0, 2, 1},
    3497:         {3, 1, 0, 2},
    3498:         {3, 2, 1, 0}
    3499:       };
    3500:       PetscInt  cone[4];
    3501:       PetscInt *graph, p, i, j, k, l;

    3503:       vertexA[0] *= R;
    3504:       vertexA[1] *= R;
    3505:       vertexA[2] *= R;
    3506:       vertexA[3] *= R;
    3507:       vertexB[0] *= R;
    3508:       vertexB[1] *= R;
    3509:       vertexB[2] *= R;
    3510:       vertexB[3] *= R;
    3511:       vertexC[0] *= R;
    3512:       vertexC[1] *= R;
    3513:       vertexC[2] *= R;
    3514:       vertexC[3] *= R;
    3515:       numCells    = rank == 0 ? 600 : 0;
    3516:       numVerts    = rank == 0 ? 120 : 0;
    3517:       firstVertex = numCells;
    3518:       /* Use the 600-cell, which for a unit sphere has coordinates which are

    3520:            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
    3521:                (\pm 1,    0,       0,      0)  all cyclic permutations   8
    3522:            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96

    3524:          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
    3525:          length is then given by 1/\phi = 0.61803.

    3527:          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
    3528:          http://mathworld.wolfram.com/600-Cell.html
    3529:       */
    3530:       /* Construct vertices */
    3531:       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
    3532:       i = 0;
    3533:       if (rank == 0) {
    3534:         for (s[0] = -1; s[0] < 2; s[0] += 2) {
    3535:           for (s[1] = -1; s[1] < 2; s[1] += 2) {
    3536:             for (s[2] = -1; s[2] < 2; s[2] += 2) {
    3537:               for (s[3] = -1; s[3] < 2; s[3] += 2) {
    3538:                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
    3539:                 ++i;
    3540:               }
    3541:             }
    3542:           }
    3543:         }
    3544:         for (p = 0; p < embedDim; ++p) {
    3545:           s[1] = s[2] = s[3] = 1;
    3546:           for (s[0] = -1; s[0] < 2; s[0] += 2) {
    3547:             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
    3548:             ++i;
    3549:           }
    3550:         }
    3551:         for (p = 0; p < 12; ++p) {
    3552:           s[3] = 1;
    3553:           for (s[0] = -1; s[0] < 2; s[0] += 2) {
    3554:             for (s[1] = -1; s[1] < 2; s[1] += 2) {
    3555:               for (s[2] = -1; s[2] < 2; s[2] += 2) {
    3556:                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
    3557:                 ++i;
    3558:               }
    3559:             }
    3560:           }
    3561:         }
    3562:       }
    3563:       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
    3564:       /* Construct graph */
    3565:       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
    3566:       for (i = 0; i < numVerts; ++i) {
    3567:         for (j = 0, k = 0; j < numVerts; ++j) {
    3568:           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
    3569:             graph[i * numVerts + j] = 1;
    3570:             ++k;
    3571:           }
    3572:         }
    3573:         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
    3574:       }
    3575:       /* Build Topology */
    3576:       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
    3577:       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
    3578:       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
    3579:       /* Cells */
    3580:       if (rank == 0) {
    3581:         for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
    3582:           for (j = 0; j < i; ++j) {
    3583:             for (k = 0; k < j; ++k) {
    3584:               for (l = 0; l < k; ++l) {
    3585:                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
    3586:                   cone[0] = firstVertex + i;
    3587:                   cone[1] = firstVertex + j;
    3588:                   cone[2] = firstVertex + k;
    3589:                   cone[3] = firstVertex + l;
    3590:                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
    3591:                   {
    3592:                     const PetscInt epsilon[4][4][4][4] = {
    3593:                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},

    3595:                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},

    3597:                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},

    3599:                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
    3600:                     };
    3601:                     PetscReal normal[4];
    3602:                     PetscInt  e, f, g;

    3604:                     for (d = 0; d < embedDim; ++d) {
    3605:                       normal[d] = 0.0;
    3606:                       for (e = 0; e < embedDim; ++e) {
    3607:                         for (f = 0; f < embedDim; ++f) {
    3608:                           for (g = 0; g < embedDim; ++g) {
    3609:                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
    3610:                           }
    3611:                         }
    3612:                       }
    3613:                     }
    3614:                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
    3615:                       PetscInt tmp = cone[1];
    3616:                       cone[1]      = cone[2];
    3617:                       cone[2]      = tmp;
    3618:                     }
    3619:                   }
    3620:                   PetscCall(DMPlexSetCone(dm, c++, cone));
    3621:                 }
    3622:               }
    3623:             }
    3624:           }
    3625:         }
    3626:       }
    3627:       PetscCall(DMPlexSymmetrize(dm));
    3628:       PetscCall(DMPlexStratify(dm));
    3629:       PetscCall(PetscFree(graph));
    3630:     }
    3631:     break;
    3632:   default:
    3633:     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
    3634:   }
    3635:   /* Create coordinates */
    3636:   PetscCall(DMGetCoordinateSection(dm, &coordSection));
    3637:   PetscCall(PetscSectionSetNumFields(coordSection, 1));
    3638:   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
    3639:   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
    3640:   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
    3641:     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
    3642:     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
    3643:   }
    3644:   PetscCall(PetscSectionSetUp(coordSection));
    3645:   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
    3646:   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
    3647:   PetscCall(VecSetBlockSize(coordinates, embedDim));
    3648:   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    3649:   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
    3650:   PetscCall(VecSetType(coordinates, VECSTANDARD));
    3651:   PetscCall(VecGetArray(coordinates, &coords));
    3652:   for (v = 0; v < numVerts; ++v)
    3653:     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
    3654:   PetscCall(VecRestoreArray(coordinates, &coords));
    3655:   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
    3656:   PetscCall(VecDestroy(&coordinates));
    3657:   PetscCall(PetscFree(coordsIn));
    3658:   {
    3659:     DM          cdm;
    3660:     PetscDS     cds;
    3661:     PetscScalar c = R;

    3663:     PetscCall(DMPlexSetCoordinateMap(dm, snapToSphere));
    3664:     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_FALSE, PETSC_TRUE));
    3665:     PetscCall(DMGetCoordinateDM(dm, &cdm));
    3666:     PetscCall(DMGetDS(cdm, &cds));
    3667:     PetscCall(PetscDSSetConstants(cds, 1, &c));
    3668:   }
    3669:   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
    3670:   /* Wait for coordinate creation before doing in-place modification */
    3671:   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
    3672:   PetscFunctionReturn(PETSC_SUCCESS);
    3673: }

    3675: typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);

    3677: /*
    3678:  The Schwarz P implicit surface is

    3680:      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
    3681: */
    3682: static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
    3683: {
    3684:   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
    3685:   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
    3686:   f[0]           = c[0] + c[1] + c[2];
    3687:   for (PetscInt i = 0; i < 3; i++) {
    3688:     grad[i] = PETSC_PI * g[i];
    3689:     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
    3690:   }
    3691: }

    3693: // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
    3694: static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
    3695: {
    3696:   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
    3697:   return PETSC_SUCCESS;
    3698: }

    3700: /*
    3701:  The Gyroid implicit surface is

    3703:  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)

    3705: */
    3706: static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
    3707: {
    3708:   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
    3709:   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
    3710:   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
    3711:   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
    3712:   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
    3713:   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
    3714:   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
    3715:   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
    3716:   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
    3717:   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
    3718:   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
    3719:   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
    3720:   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
    3721:   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
    3722:   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
    3723: }

    3725: // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
    3726: static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
    3727: {
    3728:   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
    3729:   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
    3730:   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
    3731:   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
    3732:   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
    3733:   return PETSC_SUCCESS;
    3734: }

    3736: /*
    3737:    We wish to solve

    3739:          min_y || y - x ||^2  subject to f(y) = 0

    3741:    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
    3742:    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
    3743:    tangent space and ask for both components in the tangent space to be zero.

    3745:    Take g to be a column vector and compute the "full QR" factorization Q R = g,
    3746:    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
    3747:    The first column of Q is parallel to g so the remaining two columns span the null space.
    3748:    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
    3749:    Since Q is symmetric, this is equivalent to multiplying by Q and taking the last two entries.
    3750:    In total, we have a system of 3 equations in 3 unknowns:

    3752:      f(y) = 0                       1 equation
    3753:      Qn^T (y - x) = 0               2 equations

    3755:    Here, we compute the residual and Jacobian of this system.
    3756: */
    3757: static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
    3758: {
    3759:   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
    3760:   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
    3761:   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
    3762:   PetscReal n_y[3][3] = {
    3763:     {0, 0, 0},
    3764:     {0, 0, 0},
    3765:     {0, 0, 0}
    3766:   };

    3768:   feval(yreal, &f, grad, n_y);

    3770:   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
    3771:   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
    3772:   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];

    3774:   // Define the Householder reflector
    3775:   sign = n[0] >= 0 ? 1. : -1.;
    3776:   n[0] += norm * sign;
    3777:   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;

    3779:   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
    3780:   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
    3781:   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
    3782:   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);

    3784:   for (PetscInt i = 0; i < 3; i++) {
    3785:     n[i] /= norm;
    3786:     for (PetscInt j = 0; j < 3; j++) {
    3787:       // note that n[i] is n_old[i]/norm when executing the code below
    3788:       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
    3789:     }
    3790:   }

    3792:   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
    3793:   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];

    3795:   res[0] = f;
    3796:   res[1] = d[1] - 2 * n[1] * nd;
    3797:   res[2] = d[2] - 2 * n[2] * nd;
    3798:   // J[j][i] is J_{ij} (column major)
    3799:   for (PetscInt j = 0; j < 3; j++) {
    3800:     J[0 + j * 3] = grad[j];
    3801:     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
    3802:     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
    3803:   }
    3804: }

    3806: /*
    3807:    Project x to the nearest point on the implicit surface using Newton's method.
    3808: */
    3809: static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
    3810: {
    3811:   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess

    3813:   PetscFunctionBegin;
    3814:   for (PetscInt iter = 0; iter < 10; iter++) {
    3815:     PetscScalar res[3], J[9];
    3816:     PetscReal   resnorm;
    3817:     TPSNearestPointResJac(feval, x, y, res, J);
    3818:     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
    3819:     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
    3820:       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
    3821:     }
    3822:     if (resnorm < PETSC_SMALL) break;

    3824:     // Take the Newton step
    3825:     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
    3826:     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
    3827:   }
    3828:   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
    3829:   PetscFunctionReturn(PETSC_SUCCESS);
    3830: }

    3832: const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};

    3834: static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
    3835: {
    3836:   PetscMPIInt rank;
    3837:   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
    3838:   PetscInt (*edges)[2] = NULL, *edgeSets = NULL;
    3839:   PetscInt           *cells_flat = NULL;
    3840:   PetscReal          *vtxCoords  = NULL;
    3841:   TPSEvaluateFunc     evalFunc   = NULL;
    3842:   PetscSimplePointFn *normalFunc = NULL;
    3843:   DMLabel             label;

    3845:   PetscFunctionBegin;
    3846:   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
    3847:   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
    3848:   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
    3849:   switch (tpstype) {
    3850:   case DMPLEX_TPS_SCHWARZ_P:
    3851:     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
    3852:     if (rank == 0) {
    3853:       PetscInt (*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
    3854:       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
    3855:       PetscReal L = 1;

    3857:       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
    3858:       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
    3859:       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
    3860:       Njunctions  = extent[0] * extent[1] * extent[2];
    3861:       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
    3862:       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
    3863:       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
    3864:       PetscCall(PetscMalloc1(Njunctions, &cells));
    3865:       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
    3866:       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
    3867:       // x-normal pipes
    3868:       vcount = 0;
    3869:       for (PetscInt i = 0; i < extent[0] + 1; i++) {
    3870:         for (PetscInt j = 0; j < extent[1]; j++) {
    3871:           for (PetscInt k = 0; k < extent[2]; k++) {
    3872:             for (PetscInt l = 0; l < 4; l++) {
    3873:               vtxCoords[vcount++] = (2 * i - 1) * L;
    3874:               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
    3875:               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
    3876:             }
    3877:           }
    3878:         }
    3879:       }
    3880:       // y-normal pipes
    3881:       for (PetscInt i = 0; i < extent[0]; i++) {
    3882:         for (PetscInt j = 0; j < extent[1] + 1; j++) {
    3883:           for (PetscInt k = 0; k < extent[2]; k++) {
    3884:             for (PetscInt l = 0; l < 4; l++) {
    3885:               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
    3886:               vtxCoords[vcount++] = (2 * j - 1) * L;
    3887:               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
    3888:             }
    3889:           }
    3890:         }
    3891:       }
    3892:       // z-normal pipes
    3893:       for (PetscInt i = 0; i < extent[0]; i++) {
    3894:         for (PetscInt j = 0; j < extent[1]; j++) {
    3895:           for (PetscInt k = 0; k < extent[2] + 1; k++) {
    3896:             for (PetscInt l = 0; l < 4; l++) {
    3897:               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
    3898:               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
    3899:               vtxCoords[vcount++] = (2 * k - 1) * L;
    3900:             }
    3901:           }
    3902:         }
    3903:       }
    3904:       // junctions
    3905:       for (PetscInt i = 0; i < extent[0]; i++) {
    3906:         for (PetscInt j = 0; j < extent[1]; j++) {
    3907:           for (PetscInt k = 0; k < extent[2]; k++) {
    3908:             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
    3909:             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
    3910:             for (PetscInt ii = 0; ii < 2; ii++) {
    3911:               for (PetscInt jj = 0; jj < 2; jj++) {
    3912:                 for (PetscInt kk = 0; kk < 2; kk++) {
    3913:                   double Ls           = (1 - sqrt(2) / 4) * L;
    3914:                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
    3915:                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
    3916:                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
    3917:                 }
    3918:               }
    3919:             }
    3920:             const PetscInt jfaces[3][2][4] = {
    3921:               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
    3922:               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
    3923:               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
    3924:             };
    3925:             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
    3926:                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
    3927:             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
    3928:                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
    3929:             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
    3930:               const PetscInt ijk[3] = {i, j, k};
    3931:               for (PetscInt l = 0; l < 4; l++) { // rotations
    3932:                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
    3933:                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
    3934:                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
    3935:                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
    3936:                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
    3937:                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
    3938:                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
    3939:                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
    3940:                 if (ijk[dir] == 0) {
    3941:                   edges[numEdges][0] = pipe_lo[dir] + l;
    3942:                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
    3943:                   edgeSets[numEdges] = dir * 2 + 1;
    3944:                   numEdges++;
    3945:                 }
    3946:                 if (ijk[dir] + 1 == extent[dir]) {
    3947:                   edges[numEdges][0] = pipe_hi[dir] + l;
    3948:                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
    3949:                   edgeSets[numEdges] = dir * 2 + 2;
    3950:                   numEdges++;
    3951:                 }
    3952:               }
    3953:             }
    3954:           }
    3955:         }
    3956:       }
    3957:       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
    3958:       numFaces   = 24 * Njunctions;
    3959:       cells_flat = cells[0][0][0];
    3960:     }
    3961:     evalFunc   = TPSEvaluate_SchwarzP;
    3962:     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
    3963:     break;
    3964:   case DMPLEX_TPS_GYROID:
    3965:     if (rank == 0) {
    3966:       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
    3967:       //
    3968:       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
    3969:       //
    3970:       // on the cell [0,2]^3.
    3971:       //
    3972:       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
    3973:       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
    3974:       // like a boomerang:
    3975:       //
    3976:       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
    3977:       //     -----          -------        -------        -------     //
    3978:       //                                                              //
    3979:       //     +       +      +       +      +       +      +   \   +   //
    3980:       //      \                                   /            \      //
    3981:       //       \            `-_   _-'            /              }     //
    3982:       //        *-_            `-'            _-'              /      //
    3983:       //     +     `-+      +       +      +-'     +      +   /   +   //
    3984:       //                                                              //
    3985:       //                                                              //
    3986:       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
    3987:       //     -----          -------        -------        -------     //
    3988:       //                                                              //
    3989:       //     +-_     +      +       +      +     _-+      +   /   +   //
    3990:       //        `-_            _-_            _-`            /        //
    3991:       //           \        _-'   `-_        /              {         //
    3992:       //            \                       /                \        //
    3993:       //     +       +      +       +      +       +      +   \   +   //
    3994:       //
    3995:       //
    3996:       // This course mesh approximates each of these slices by two line segments,
    3997:       // and then connects the segments in consecutive layers with quadrilateral faces.
    3998:       // All of the end points of the segments are multiples of 1/4 except for the
    3999:       // point * in the picture for z = 0 above and the similar points in other layers.
    4000:       // That point is at (gamma, gamma, 0), where gamma is calculated below.
    4001:       //
    4002:       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
    4003:       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
    4004:       //
    4005:       // As for how this method turned into the names given to the vertices:
    4006:       // that was not systematic, it was just the way it worked out in my handwritten notes.

    4008:       PetscInt facesPerBlock = 64;
    4009:       PetscInt vertsPerBlock = 56;
    4010:       PetscInt extentPlus[3];
    4011:       PetscInt numBlocks, numBlocksPlus;
    4012:       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
    4013:       const PetscInt pattern[64][4] = {
    4014:         /* face to vertex within the coarse discretization of a single gyroid block */
    4015:         /* layer 0 */
    4016:         {A,           C,           K,           G          },
    4017:         {C,           B,           II,          K          },
    4018:         {D,           A,           H,           L          },
    4019:         {B + 56 * 1,  D,           L,           J          },
    4020:         {E,           B + 56 * 1,  J,           N          },
    4021:         {A + 56 * 2,  E,           N,           H + 56 * 2 },
    4022:         {F,           A + 56 * 2,  G + 56 * 2,  M          },
    4023:         {B,           F,           M,           II         },
    4024:         /* layer 1 */
    4025:         {G,           K,           Q,           O          },
    4026:         {K,           II,          P,           Q          },
    4027:         {L,           H,           O + 56 * 1,  R          },
    4028:         {J,           L,           R,           P          },
    4029:         {N,           J,           P,           S          },
    4030:         {H + 56 * 2,  N,           S,           O + 56 * 3 },
    4031:         {M,           G + 56 * 2,  O + 56 * 2,  T          },
    4032:         {II,          M,           T,           P          },
    4033:         /* layer 2 */
    4034:         {O,           Q,           Y,           U          },
    4035:         {Q,           P,           W,           Y          },
    4036:         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
    4037:         {P,           R,           Ap,          W          },
    4038:         {S,           P,           X,           Bp         },
    4039:         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
    4040:         {T,           O + 56 * 2,  V,           Z          },
    4041:         {P,           T,           Z,           X          },
    4042:         /* layer 3 */
    4043:         {U,           Y,           Ep,          Dp         },
    4044:         {Y,           W,           Cp,          Ep         },
    4045:         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
    4046:         {W,           Ap,          Gp,          Cp         },
    4047:         {Bp,          X,           Cp + 56 * 2, Fp         },
    4048:         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
    4049:         {Z,           V,           Dp,          Hp         },
    4050:         {X,           Z,           Hp,          Cp + 56 * 2},
    4051:         /* layer 4 */
    4052:         {Dp,          Ep,          Mp,          Kp         },
    4053:         {Ep,          Cp,          Ip,          Mp         },
    4054:         {Gp,          Dp + 56 * 1, Lp,          Np         },
    4055:         {Cp,          Gp,          Np,          Jp         },
    4056:         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
    4057:         {Dp + 56 * 1, Fp,          Pp,          Lp         },
    4058:         {Hp,          Dp,          Kp,          Op         },
    4059:         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
    4060:         /* layer 5 */
    4061:         {Kp,          Mp,          Sp,          Rp         },
    4062:         {Mp,          Ip,          Qp,          Sp         },
    4063:         {Np,          Lp,          Rp,          Tp         },
    4064:         {Jp,          Np,          Tp,          Qp + 56 * 1},
    4065:         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
    4066:         {Lp,          Pp,          Up,          Rp         },
    4067:         {Op,          Kp,          Rp,          Vp         },
    4068:         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
    4069:         /* layer 6 */
    4070:         {Rp,          Sp,          Aq,          Yp         },
    4071:         {Sp,          Qp,          Wp,          Aq         },
    4072:         {Tp,          Rp,          Yp,          Cq         },
    4073:         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
    4074:         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
    4075:         {Rp,          Up,          Dq,          Zp         },
    4076:         {Vp,          Rp,          Zp,          Bq         },
    4077:         {Qp + 56 * 2, Vp,          Bq,          Xp         },
    4078:         /* layer 7 (the top is the periodic image of the bottom of layer 0) */
    4079:         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
    4080:         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
    4081:         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
    4082:         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
    4083:         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
    4084:         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
    4085:         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
    4086:         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
    4087:       };
    4088:       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
    4089:       const PetscReal patternCoords[56][3] = {
    4090:         {1.,        0.,        0.  }, /* A  */
    4091:         {0.,        1.,        0.  }, /* B  */
    4092:         {gamma,     gamma,     0.  }, /* C  */
    4093:         {1 + gamma, 1 - gamma, 0.  }, /* D  */
    4094:         {2 - gamma, 2 - gamma, 0.  }, /* E  */
    4095:         {1 - gamma, 1 + gamma, 0.  }, /* F  */

    4097:         {.5,        0,         .25 }, /* G  */
    4098:         {1.5,       0.,        .25 }, /* H  */
    4099:         {.5,        1.,        .25 }, /* II */
    4100:         {1.5,       1.,        .25 }, /* J  */
    4101:         {.25,       .5,        .25 }, /* K  */
    4102:         {1.25,      .5,        .25 }, /* L  */
    4103:         {.75,       1.5,       .25 }, /* M  */
    4104:         {1.75,      1.5,       .25 }, /* N  */

    4106:         {0.,        0.,        .5  }, /* O  */
    4107:         {1.,        1.,        .5  }, /* P  */
    4108:         {gamma,     1 - gamma, .5  }, /* Q  */
    4109:         {1 + gamma, gamma,     .5  }, /* R  */
    4110:         {2 - gamma, 1 + gamma, .5  }, /* S  */
    4111:         {1 - gamma, 2 - gamma, .5  }, /* T  */

    4113:         {0.,        .5,        .75 }, /* U  */
    4114:         {0.,        1.5,       .75 }, /* V  */
    4115:         {1.,        .5,        .75 }, /* W  */
    4116:         {1.,        1.5,       .75 }, /* X  */
    4117:         {.5,        .75,       .75 }, /* Y  */
    4118:         {.5,        1.75,      .75 }, /* Z  */
    4119:         {1.5,       .25,       .75 }, /* Ap */
    4120:         {1.5,       1.25,      .75 }, /* Bp */

    4122:         {1.,        0.,        1.  }, /* Cp */
    4123:         {0.,        1.,        1.  }, /* Dp */
    4124:         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
    4125:         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
    4126:         {2 - gamma, gamma,     1.  }, /* Gp */
    4127:         {gamma,     2 - gamma, 1.  }, /* Hp */

    4129:         {.5,        0.,        1.25}, /* Ip */
    4130:         {1.5,       0.,        1.25}, /* Jp */
    4131:         {.5,        1.,        1.25}, /* Kp */
    4132:         {1.5,       1.,        1.25}, /* Lp */
    4133:         {.75,       .5,        1.25}, /* Mp */
    4134:         {1.75,      .5,        1.25}, /* Np */
    4135:         {.25,       1.5,       1.25}, /* Op */
    4136:         {1.25,      1.5,       1.25}, /* Pp */

    4138:         {0.,        0.,        1.5 }, /* Qp */
    4139:         {1.,        1.,        1.5 }, /* Rp */
    4140:         {1 - gamma, gamma,     1.5 }, /* Sp */
    4141:         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
    4142:         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
    4143:         {gamma,     1 + gamma, 1.5 }, /* Vp */

    4145:         {0.,        .5,        1.75}, /* Wp */
    4146:         {0.,        1.5,       1.75}, /* Xp */
    4147:         {1.,        .5,        1.75}, /* Yp */
    4148:         {1.,        1.5,       1.75}, /* Zp */
    4149:         {.5,        .25,       1.75}, /* Aq */
    4150:         {.5,        1.25,      1.75}, /* Bq */
    4151:         {1.5,       .75,       1.75}, /* Cq */
    4152:         {1.5,       1.75,      1.75}, /* Dq */
    4153:       };
    4154:       PetscInt (*cells)[64][4] = NULL;
    4155:       PetscBool *seen;
    4156:       PetscInt  *vertToTrueVert;
    4157:       PetscInt   count;

    4159:       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
    4160:       numBlocks = 1;
    4161:       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
    4162:       numBlocksPlus = 1;
    4163:       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
    4164:       numFaces = numBlocks * facesPerBlock;
    4165:       PetscCall(PetscMalloc1(numBlocks, &cells));
    4166:       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
    4167:       for (PetscInt k = 0; k < extent[2]; k++) {
    4168:         for (PetscInt j = 0; j < extent[1]; j++) {
    4169:           for (PetscInt i = 0; i < extent[0]; i++) {
    4170:             for (PetscInt f = 0; f < facesPerBlock; f++) {
    4171:               for (PetscInt v = 0; v < 4; v++) {
    4172:                 PetscInt vertRaw     = pattern[f][v];
    4173:                 PetscInt blockidx    = vertRaw / 56;
    4174:                 PetscInt patternvert = vertRaw % 56;
    4175:                 PetscInt xplus       = (blockidx & 1);
    4176:                 PetscInt yplus       = (blockidx & 2) >> 1;
    4177:                 PetscInt zplus       = (blockidx & 4) >> 2;
    4178:                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
    4179:                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
    4180:                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
    4181:                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;

    4183:                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
    4184:                 seen[vert]                                       = PETSC_TRUE;
    4185:               }
    4186:             }
    4187:           }
    4188:         }
    4189:       }
    4190:       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
    4191:         if (seen[i]) numVertices++;
    4192:       count = 0;
    4193:       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
    4194:       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
    4195:       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
    4196:       for (PetscInt k = 0; k < extentPlus[2]; k++) {
    4197:         for (PetscInt j = 0; j < extentPlus[1]; j++) {
    4198:           for (PetscInt i = 0; i < extentPlus[0]; i++) {
    4199:             for (PetscInt v = 0; v < vertsPerBlock; v++) {
    4200:               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;

    4202:               if (seen[vIdx]) {
    4203:                 PetscInt thisVert;

    4205:                 vertToTrueVert[vIdx] = thisVert = count++;

    4207:                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
    4208:                 vtxCoords[3 * thisVert + 0] += i * 2;
    4209:                 vtxCoords[3 * thisVert + 1] += j * 2;
    4210:                 vtxCoords[3 * thisVert + 2] += k * 2;
    4211:               }
    4212:             }
    4213:           }
    4214:         }
    4215:       }
    4216:       for (PetscInt i = 0; i < numBlocks; i++) {
    4217:         for (PetscInt f = 0; f < facesPerBlock; f++) {
    4218:           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
    4219:         }
    4220:       }
    4221:       PetscCall(PetscFree(vertToTrueVert));
    4222:       PetscCall(PetscFree(seen));
    4223:       cells_flat = cells[0][0];
    4224:       numEdges   = 0;
    4225:       for (PetscInt i = 0; i < numFaces; i++) {
    4226:         for (PetscInt e = 0; e < 4; e++) {
    4227:           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
    4228:           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};

    4230:           for (PetscInt d = 0; d < 3; d++) {
    4231:             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
    4232:               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
    4233:               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
    4234:             }
    4235:           }
    4236:         }
    4237:       }
    4238:       PetscCall(PetscMalloc1(numEdges, &edges));
    4239:       PetscCall(PetscMalloc1(numEdges, &edgeSets));
    4240:       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
    4241:         for (PetscInt e = 0; e < 4; e++) {
    4242:           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
    4243:           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};

    4245:           for (PetscInt d = 0; d < 3; d++) {
    4246:             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
    4247:               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
    4248:                 edges[edge][0]   = ev[0];
    4249:                 edges[edge][1]   = ev[1];
    4250:                 edgeSets[edge++] = 2 * d;
    4251:               }
    4252:               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
    4253:                 edges[edge][0]   = ev[0];
    4254:                 edges[edge][1]   = ev[1];
    4255:                 edgeSets[edge++] = 2 * d + 1;
    4256:               }
    4257:             }
    4258:           }
    4259:         }
    4260:       }
    4261:     }
    4262:     evalFunc   = TPSEvaluate_Gyroid;
    4263:     normalFunc = TPSExtrudeNormalFunc_Gyroid;
    4264:     break;
    4265:   }

    4267:   PetscCall(DMSetDimension(dm, topoDim));
    4268:   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
    4269:   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
    4270:   PetscCall(PetscFree(cells_flat));
    4271:   {
    4272:     DM idm;
    4273:     PetscCall(DMPlexInterpolate(dm, &idm));
    4274:     PetscCall(DMPlexReplace_Internal(dm, &idm));
    4275:   }
    4276:   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
    4277:   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
    4278:   PetscCall(PetscFree(vtxCoords));

    4280:   PetscCall(DMCreateLabel(dm, "Face Sets"));
    4281:   PetscCall(DMGetLabel(dm, "Face Sets", &label));
    4282:   for (PetscInt e = 0; e < numEdges; e++) {
    4283:     PetscInt        njoin;
    4284:     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
    4285:     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
    4286:     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
    4287:     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
    4288:     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
    4289:   }
    4290:   PetscCall(PetscFree(edges));
    4291:   PetscCall(PetscFree(edgeSets));
    4292:   if (tps_distribute) {
    4293:     DM               pdm = NULL;
    4294:     PetscPartitioner part;

    4296:     PetscCall(DMPlexGetPartitioner(dm, &part));
    4297:     PetscCall(PetscPartitionerSetFromOptions(part));
    4298:     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
    4299:     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
    4300:     // Do not auto-distribute again
    4301:     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
    4302:   }

    4304:   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
    4305:   for (PetscInt refine = 0; refine < refinements; refine++) {
    4306:     PetscInt     m;
    4307:     DM           dmf, cdm, cdmf;
    4308:     Vec          X;
    4309:     PetscScalar *x;

    4311:     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
    4312:     PetscCall(DMGetCoordinateDM(dm, &cdm));
    4313:     PetscCall(DMGetCoordinateDM(dmf, &cdmf));
    4314:     PetscCall(DMCopyDisc(cdm, cdmf));
    4315:     PetscCall(DMPlexReplace_Internal(dm, &dmf));

    4317:     PetscCall(DMGetCoordinatesLocal(dm, &X));
    4318:     PetscCall(VecGetLocalSize(X, &m));
    4319:     PetscCall(VecGetArray(X, &x));
    4320:     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
    4321:     PetscCall(VecRestoreArray(X, &x));
    4322:   }

    4324:   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
    4325:   PetscCall(DMGetLabel(dm, "Face Sets", &label));
    4326:   PetscCall(DMPlexLabelComplete(dm, label));

    4328:   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));

    4330:   if (thickness > 0) {
    4331:     DM              edm, cdm, ecdm;
    4332:     DMPlexTransform tr;
    4333:     const char     *prefix;
    4334:     PetscOptions    options;
    4335:     // Code from DMPlexExtrude
    4336:     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
    4337:     PetscCall(DMPlexTransformSetDM(tr, dm));
    4338:     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDETYPE));
    4339:     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
    4340:     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
    4341:     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
    4342:     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
    4343:     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
    4344:     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
    4345:     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
    4346:     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
    4347:     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
    4348:     PetscCall(DMPlexTransformSetFromOptions(tr));
    4349:     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
    4350:     PetscCall(DMPlexTransformSetUp(tr));
    4351:     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
    4352:     PetscCall(DMPlexTransformApply(tr, dm, &edm));
    4353:     PetscCall(DMCopyDisc(dm, edm));
    4354:     PetscCall(DMGetCoordinateDM(dm, &cdm));
    4355:     PetscCall(DMGetCoordinateDM(edm, &ecdm));
    4356:     PetscCall(DMCopyDisc(cdm, ecdm));
    4357:     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
    4358:     PetscCall(DMPlexTransformDestroy(&tr));
    4359:     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, edm));
    4360:     PetscCall(DMPlexReplace_Internal(dm, &edm));
    4361:   }
    4362:   PetscFunctionReturn(PETSC_SUCCESS);
    4363: }

    4365: /*@
    4366:   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface

    4368:   Collective

    4370:   Input Parameters:
    4371: + comm           - The communicator for the `DM` object
    4372: . tpstype        - Type of triply-periodic surface
    4373: . extent         - Array of length 3 containing number of periods in each direction
    4374: . periodic       - array of length 3 with periodicity, or `NULL` for non-periodic
    4375: . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
    4376: . refinements    - Number of factor-of-2 refinements of 2D manifold mesh
    4377: . layers         - Number of cell layers extruded in normal direction
    4378: - thickness      - Thickness in normal direction

    4380:   Output Parameter:
    4381: . dm - The `DM` object

    4383:   Level: beginner

    4385:   Notes:
    4386:   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is the simplest member of the triply-periodic minimal surfaces.
    4387:   <https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22)> and can be cut with "clean" boundaries.
    4388:   The Gyroid <https://en.wikipedia.org/wiki/Gyroid> is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
    4389:   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
    4390:   On each refinement, all vertices are projected to their nearest point on the surface.
    4391:   This projection could readily be extended to related surfaces.

    4393:   See {cite}`maskery2018insights`

    4395:   The face (edge) sets for the Schwarz P surface are numbered $1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z)$.
    4396:   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).
    4397:   Use `DMPlexLabelComplete()` to propagate to coarse-level vertices.

    4399:   Developer Notes:
    4400:   The Gyroid mesh does not currently mark boundary sets.

    4402: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
    4403: @*/
    4404: PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
    4405: {
    4406:   PetscFunctionBegin;
    4407:   PetscCall(DMCreate(comm, dm));
    4408:   PetscCall(DMSetType(*dm, DMPLEX));
    4409:   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
    4410:   PetscFunctionReturn(PETSC_SUCCESS);
    4411: }

    4413: /*@
    4414:   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.

    4416:   Collective

    4418:   Input Parameters:
    4419: + comm    - The communicator for the `DM` object
    4420: . dim     - The dimension
    4421: . simplex - Use simplices, or tensor product cells
    4422: - R       - The radius

    4424:   Output Parameter:
    4425: . dm - The `DM` object

    4427:   Level: beginner

    4429: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
    4430: @*/
    4431: PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
    4432: {
    4433:   PetscFunctionBegin;
    4434:   PetscAssertPointer(dm, 5);
    4435:   PetscCall(DMCreate(comm, dm));
    4436:   PetscCall(DMSetType(*dm, DMPLEX));
    4437:   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
    4438:   PetscFunctionReturn(PETSC_SUCCESS);
    4439: }

    4441: static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
    4442: {
    4443:   DM          sdm, vol;
    4444:   DMLabel     bdlabel;
    4445:   const char *prefix;

    4447:   PetscFunctionBegin;
    4448:   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
    4449:   PetscCall(DMSetType(sdm, DMPLEX));
    4450:   PetscCall(DMGetOptionsPrefix(dm, &prefix));
    4451:   PetscCall(DMSetOptionsPrefix(sdm, prefix));
    4452:   PetscCall(DMAppendOptionsPrefix(sdm, "bd_"));
    4453:   PetscCall(DMPlexDistributeSetDefault(sdm, PETSC_FALSE));
    4454:   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
    4455:   PetscCall(DMSetFromOptions(sdm));
    4456:   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
    4457:   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
    4458:   PetscCall(DMDestroy(&sdm));
    4459:   PetscCall(DMPlexReplace_Internal(dm, &vol));
    4460:   PetscCall(DMCreateLabel(dm, "marker"));
    4461:   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
    4462:   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
    4463:   PetscCall(DMPlexLabelComplete(dm, bdlabel));
    4464:   PetscFunctionReturn(PETSC_SUCCESS);
    4465: }

    4467: /*@
    4468:   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.

    4470:   Collective

    4472:   Input Parameters:
    4473: + comm - The communicator for the `DM` object
    4474: . dim  - The dimension
    4475: - R    - The radius

    4477:   Output Parameter:
    4478: . dm - The `DM` object

    4480:   Options Database Key:
    4481: . bd_dm_refine - This will refine the surface mesh preserving the sphere geometry

    4483:   Level: beginner

    4485: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
    4486: @*/
    4487: PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
    4488: {
    4489:   PetscFunctionBegin;
    4490:   PetscCall(DMCreate(comm, dm));
    4491:   PetscCall(DMSetType(*dm, DMPLEX));
    4492:   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
    4493:   PetscFunctionReturn(PETSC_SUCCESS);
    4494: }

    4496: static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
    4497: {
    4498:   PetscFunctionBegin;
    4499:   switch (ct) {
    4500:   case DM_POLYTOPE_POINT: {
    4501:     PetscInt    numPoints[1]        = {1};
    4502:     PetscInt    coneSize[1]         = {0};
    4503:     PetscInt    cones[1]            = {0};
    4504:     PetscInt    coneOrientations[1] = {0};
    4505:     PetscScalar vertexCoords[1]     = {0.0};

    4507:     PetscCall(DMSetDimension(rdm, 0));
    4508:     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    4509:   } break;
    4510:   case DM_POLYTOPE_SEGMENT: {
    4511:     PetscInt    numPoints[2]        = {2, 1};
    4512:     PetscInt    coneSize[3]         = {2, 0, 0};
    4513:     PetscInt    cones[2]            = {1, 2};
    4514:     PetscInt    coneOrientations[2] = {0, 0};
    4515:     PetscScalar vertexCoords[2]     = {-1.0, 1.0};

    4517:     PetscCall(DMSetDimension(rdm, 1));
    4518:     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    4519:   } break;
    4520:   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
    4521:     PetscInt    numPoints[2]        = {2, 1};
    4522:     PetscInt    coneSize[3]         = {2, 0, 0};
    4523:     PetscInt    cones[2]            = {1, 2};
    4524:     PetscInt    coneOrientations[2] = {0, 0};
    4525:     PetscScalar vertexCoords[2]     = {-1.0, 1.0};

    4527:     PetscCall(DMSetDimension(rdm, 1));
    4528:     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    4529:   } break;
    4530:   case DM_POLYTOPE_TRIANGLE: {
    4531:     PetscInt    numPoints[2]        = {3, 1};
    4532:     PetscInt    coneSize[4]         = {3, 0, 0, 0};
    4533:     PetscInt    cones[3]            = {1, 2, 3};
    4534:     PetscInt    coneOrientations[3] = {0, 0, 0};
    4535:     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};

    4537:     PetscCall(DMSetDimension(rdm, 2));
    4538:     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    4539:   } break;
    4540:   case DM_POLYTOPE_QUADRILATERAL: {
    4541:     PetscInt    numPoints[2]        = {4, 1};
    4542:     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
    4543:     PetscInt    cones[4]            = {1, 2, 3, 4};
    4544:     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
    4545:     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};

    4547:     PetscCall(DMSetDimension(rdm, 2));
    4548:     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    4549:   } break;
    4550:   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
    4551:     PetscInt    numPoints[2]        = {4, 1};
    4552:     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
    4553:     PetscInt    cones[4]            = {1, 2, 3, 4};
    4554:     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
    4555:     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};

    4557:     PetscCall(DMSetDimension(rdm, 2));
    4558:     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    4559:   } break;
    4560:   case DM_POLYTOPE_TETRAHEDRON: {
    4561:     PetscInt    numPoints[2]        = {4, 1};
    4562:     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
    4563:     PetscInt    cones[4]            = {1, 2, 3, 4};
    4564:     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
    4565:     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};

    4567:     PetscCall(DMSetDimension(rdm, 3));
    4568:     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    4569:   } break;
    4570:   case DM_POLYTOPE_HEXAHEDRON: {
    4571:     PetscInt    numPoints[2]        = {8, 1};
    4572:     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
    4573:     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
    4574:     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
    4575:     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};

    4577:     PetscCall(DMSetDimension(rdm, 3));
    4578:     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    4579:   } break;
    4580:   case DM_POLYTOPE_TRI_PRISM: {
    4581:     PetscInt    numPoints[2]        = {6, 1};
    4582:     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
    4583:     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
    4584:     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
    4585:     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};

    4587:     PetscCall(DMSetDimension(rdm, 3));
    4588:     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    4589:   } break;
    4590:   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
    4591:     PetscInt    numPoints[2]        = {6, 1};
    4592:     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
    4593:     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
    4594:     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
    4595:     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};

    4597:     PetscCall(DMSetDimension(rdm, 3));
    4598:     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    4599:   } break;
    4600:   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
    4601:     PetscInt    numPoints[2]        = {8, 1};
    4602:     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
    4603:     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
    4604:     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
    4605:     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};

    4607:     PetscCall(DMSetDimension(rdm, 3));
    4608:     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    4609:   } break;
    4610:   case DM_POLYTOPE_PYRAMID: {
    4611:     PetscInt    numPoints[2]        = {5, 1};
    4612:     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
    4613:     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
    4614:     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
    4615:     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};

    4617:     PetscCall(DMSetDimension(rdm, 3));
    4618:     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
    4619:   } break;
    4620:   default:
    4621:     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
    4622:   }
    4623:   {
    4624:     PetscInt Nv, v;

    4626:     /* Must create the celltype label here so that we do not automatically try to compute the types */
    4627:     PetscCall(DMCreateLabel(rdm, "celltype"));
    4628:     PetscCall(DMPlexSetCellType(rdm, 0, ct));
    4629:     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
    4630:     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
    4631:   }
    4632:   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
    4633:   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
    4634:   PetscFunctionReturn(PETSC_SUCCESS);
    4635: }

    4637: /*@
    4638:   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell

    4640:   Collective

    4642:   Input Parameters:
    4643: + comm - The communicator
    4644: - ct   - The cell type of the reference cell

    4646:   Output Parameter:
    4647: . refdm - The reference cell

    4649:   Level: intermediate

    4651: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`
    4652: @*/
    4653: PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
    4654: {
    4655:   PetscFunctionBegin;
    4656:   PetscCall(DMCreate(comm, refdm));
    4657:   PetscCall(DMSetType(*refdm, DMPLEX));
    4658:   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
    4659:   PetscFunctionReturn(PETSC_SUCCESS);
    4660: }

    4662: static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
    4663: {
    4664:   DM        plex;
    4665:   DMLabel   label;
    4666:   PetscBool hasLabel;

    4668:   PetscFunctionBegin;
    4669:   PetscCall(DMHasLabel(dm, name, &hasLabel));
    4670:   if (hasLabel) PetscFunctionReturn(PETSC_SUCCESS);
    4671:   PetscCall(DMCreateLabel(dm, name));
    4672:   PetscCall(DMGetLabel(dm, name, &label));
    4673:   PetscCall(DMConvert(dm, DMPLEX, &plex));
    4674:   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
    4675:   PetscCall(DMPlexLabelComplete(plex, label));
    4676:   PetscCall(DMDestroy(&plex));
    4677:   PetscFunctionReturn(PETSC_SUCCESS);
    4678: }

    4680: /*
    4681:   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.

    4683:     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
    4684: */
    4685: static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
    4686: {
    4687:   const PetscReal low = PetscRealPart(constants[0]);
    4688:   const PetscReal upp = PetscRealPart(constants[1]);
    4689:   const PetscReal r   = PetscRealPart(u[1]);
    4690:   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);

    4692:   f0[0] = r * PetscCosReal(th);
    4693:   f0[1] = r * PetscSinReal(th);
    4694: }

    4696: // Insert vertices and their joins, marked by depth
    4697: static PetscErrorCode ProcessCohesiveLabel_Vertices(DM dm, DMLabel label, DMLabel vlabel, PetscInt val, PetscInt n, const PetscInt vertices[])
    4698: {
    4699:   PetscFunctionBegin;
    4700:   PetscCall(DMPlexMarkSubmesh_Interpolated(dm, vlabel, val, PETSC_FALSE, PETSC_FALSE, label, NULL));
    4701:   PetscFunctionReturn(PETSC_SUCCESS);
    4702: }

    4704: // Insert faces and their closures, marked by depth
    4705: static PetscErrorCode ProcessCohesiveLabel_Faces(DM dm, DMLabel label, PetscInt n, const PetscInt faces[])
    4706: {
    4707:   PetscFunctionBegin;
    4708:   for (PetscInt p = 0; p < n; ++p) {
    4709:     const PetscInt point   = faces[p];
    4710:     PetscInt      *closure = NULL;
    4711:     PetscInt       clSize, pdepth;

    4713:     PetscCall(DMPlexGetPointDepth(dm, point, &pdepth));
    4714:     PetscCall(DMLabelSetValue(label, point, pdepth));
    4715:     PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
    4716:     for (PetscInt cl = 0; cl < clSize * 2; cl += 2) {
    4717:       PetscCall(DMPlexGetPointDepth(dm, closure[cl], &pdepth));
    4718:       PetscCall(DMLabelSetValue(label, closure[cl], pdepth));
    4719:     }
    4720:     PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
    4721:   }
    4722:   PetscFunctionReturn(PETSC_SUCCESS);
    4723: }

    4725: PETSC_EXTERN PetscErrorCode PetscOptionsFindPairPrefix_Private(PetscOptions, const char pre[], const char name[], const char *option[], const char *value[], PetscBool *flg);

    4727: const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "hypercubic", "zbox", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};

    4729: static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
    4730: {
    4731:   DMPlexShape    shape   = DM_SHAPE_BOX;
    4732:   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
    4733:   PetscInt       dim     = 2;
    4734:   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, orient = PETSC_FALSE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
    4735:   PetscBool      flg, flg2, fflg, strflg, bdfflg, nameflg;
    4736:   MPI_Comm       comm;
    4737:   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
    4738:   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
    4739:   char           plexname[PETSC_MAX_PATH_LEN]   = "";
    4740:   const char    *option;

    4742:   PetscFunctionBegin;
    4743:   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
    4744:   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
    4745:   /* TODO Turn this into a registration interface */
    4746:   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
    4747:   PetscCall(PetscOptionsString("-dm_plex_file_contents", "Contents of a file format in a string", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &strflg));
    4748:   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
    4749:   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
    4750:   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
    4751:   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
    4752:   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
    4753:   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
    4754:   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
    4755:   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
    4756:   PetscCall(PetscOptionsBool("-dm_plex_orient", "Orient the constructed mesh", "DMPlexOrient", orient, &orient, &flg));
    4757:   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
    4758:   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
    4759:   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
    4760:   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_adj", "Debug output level all adjacency computations", "", 0, &((DM_Plex *)dm->data)->printAdj, NULL, 0));

    4762:   switch (cell) {
    4763:   case DM_POLYTOPE_POINT:
    4764:   case DM_POLYTOPE_SEGMENT:
    4765:   case DM_POLYTOPE_POINT_PRISM_TENSOR:
    4766:   case DM_POLYTOPE_TRIANGLE:
    4767:   case DM_POLYTOPE_QUADRILATERAL:
    4768:   case DM_POLYTOPE_TETRAHEDRON:
    4769:   case DM_POLYTOPE_HEXAHEDRON:
    4770:     *useCoordSpace = PETSC_TRUE;
    4771:     break;
    4772:   default:
    4773:     *useCoordSpace = PETSC_FALSE;
    4774:     break;
    4775:   }

    4777:   if (fflg) {
    4778:     DM          dmnew;
    4779:     const char *name;

    4781:     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
    4782:     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, nameflg ? plexname : name, interpolate, &dmnew));
    4783:     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
    4784:     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
    4785:   } else if (refDomain) {
    4786:     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
    4787:   } else if (bdfflg) {
    4788:     DM          bdm, dmnew;
    4789:     const char *name;

    4791:     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
    4792:     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, nameflg ? plexname : name, interpolate, &bdm));
    4793:     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
    4794:     PetscCall(DMSetFromOptions(bdm));
    4795:     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
    4796:     PetscCall(DMDestroy(&bdm));
    4797:     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
    4798:     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
    4799:   } else if (strflg) {
    4800:     DM          dmnew;
    4801:     PetscViewer viewer;
    4802:     const char *contents;
    4803:     char       *strname;
    4804:     char        tmpdir[PETSC_MAX_PATH_LEN];
    4805:     char        tmpfilename[PETSC_MAX_PATH_LEN];
    4806:     char        name[PETSC_MAX_PATH_LEN];
    4807:     MPI_Comm    comm;
    4808:     PetscMPIInt rank;

    4810:     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
    4811:     PetscCallMPI(MPI_Comm_rank(comm, &rank));
    4812:     PetscCall(PetscStrchr(filename, ':', &strname));
    4813:     PetscCheck(strname, comm, PETSC_ERR_ARG_WRONG, "File contents must have the form \"ext:string_name\", not %s", filename);
    4814:     strname[0] = '\0';
    4815:     ++strname;
    4816:     PetscCall(PetscDLSym(NULL, strname, (void **)&contents));
    4817:     PetscCheck(contents, comm, PETSC_ERR_ARG_WRONG, "Could not locate mesh string %s", strname);
    4818:     PetscCall(PetscGetTmp(comm, tmpdir, PETSC_MAX_PATH_LEN));
    4819:     PetscCall(PetscStrlcat(tmpdir, "/meshXXXXXX", PETSC_MAX_PATH_LEN));
    4820:     PetscCall(PetscMkdtemp(tmpdir));
    4821:     PetscCall(PetscSNPrintf(tmpfilename, PETSC_MAX_PATH_LEN, "%s/mesh.%s", tmpdir, filename));
    4822:     PetscCall(PetscViewerASCIIOpen(comm, tmpfilename, &viewer));
    4823:     PetscCall(PetscViewerASCIIPrintf(viewer, "%s\n", contents));
    4824:     PetscCall(PetscViewerDestroy(&viewer));
    4825:     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), tmpfilename, plexname, interpolate, &dmnew));
    4826:     PetscCall(PetscRMTree(tmpdir));
    4827:     PetscCall(PetscSNPrintf(name, PETSC_MAX_PATH_LEN, "%s Mesh", strname));
    4828:     PetscCall(PetscObjectSetName((PetscObject)dm, name));
    4829:     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
    4830:     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
    4831:   } else {
    4832:     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
    4833:     switch (shape) {
    4834:     case DM_SHAPE_BOX:
    4835:     case DM_SHAPE_ZBOX:
    4836:     case DM_SHAPE_ANNULUS: {
    4837:       PetscInt       faces[3]  = {0, 0, 0};
    4838:       PetscReal      lower[3]  = {0, 0, 0};
    4839:       PetscReal      upper[3]  = {1, 1, 1};
    4840:       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
    4841:       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
    4842:       PetscInt       i, n;

    4844:       n = dim;
    4845:       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
    4846:       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
    4847:       n = 3;
    4848:       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
    4849:       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
    4850:       n = 3;
    4851:       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
    4852:       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
    4853:       n = 3;
    4854:       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
    4855:       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);

    4857:       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
    4858:       if (isAnnular)
    4859:         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;

    4861:       switch (cell) {
    4862:       case DM_POLYTOPE_TRI_PRISM_TENSOR:
    4863:         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
    4864:         if (!interpolate) {
    4865:           DM udm;

    4867:           PetscCall(DMPlexUninterpolate(dm, &udm));
    4868:           PetscCall(DMPlexReplace_Internal(dm, &udm));
    4869:         }
    4870:         break;
    4871:       default:
    4872:         PetscCall(DMPlexCreateBoxMesh_Internal(dm, shape, dim, simplex, faces, lower, upper, bdt, interpolate));
    4873:         break;
    4874:       }
    4875:       if (isAnnular) {
    4876:         DM          cdm;
    4877:         PetscDS     cds;
    4878:         PetscScalar bounds[2] = {lower[0], upper[0]};

    4880:         // Fix coordinates for annular region
    4881:         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
    4882:         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
    4883:         PetscCall(DMSetCellCoordinates(dm, NULL));
    4884:         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_FALSE, PETSC_TRUE));
    4885:         PetscCall(DMGetCoordinateDM(dm, &cdm));
    4886:         PetscCall(DMGetDS(cdm, &cds));
    4887:         PetscCall(PetscDSSetConstants(cds, 2, bounds));
    4888:         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
    4889:       }
    4890:     } break;
    4891:     case DM_SHAPE_BOX_SURFACE: {
    4892:       PetscInt  faces[3] = {0, 0, 0};
    4893:       PetscReal lower[3] = {0, 0, 0};
    4894:       PetscReal upper[3] = {1, 1, 1};
    4895:       PetscInt  i, n;

    4897:       n = dim + 1;
    4898:       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
    4899:       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
    4900:       n = 3;
    4901:       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
    4902:       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
    4903:       n = 3;
    4904:       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
    4905:       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
    4906:       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
    4907:     } break;
    4908:     case DM_SHAPE_SPHERE: {
    4909:       PetscReal R = 1.0;

    4911:       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
    4912:       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
    4913:     } break;
    4914:     case DM_SHAPE_BALL: {
    4915:       PetscReal R = 1.0;

    4917:       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
    4918:       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
    4919:     } break;
    4920:     case DM_SHAPE_CYLINDER: {
    4921:       DMBoundaryType bdt = DM_BOUNDARY_NONE;
    4922:       PetscInt       Nw  = 6;
    4923:       PetscInt       Nr  = 0;

    4925:       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
    4926:       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
    4927:       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_refine", "Number of refinements before projection", "", Nr, &Nr, NULL));
    4928:       switch (cell) {
    4929:       case DM_POLYTOPE_TRI_PRISM_TENSOR:
    4930:         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
    4931:         break;
    4932:       default:
    4933:         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt, Nr));
    4934:         break;
    4935:       }
    4936:     } break;
    4937:     case DM_SHAPE_SCHWARZ_P: // fallthrough
    4938:     case DM_SHAPE_GYROID: {
    4939:       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
    4940:       PetscReal      thickness   = 0.;
    4941:       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
    4942:       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
    4943:       PetscBool      tps_distribute;
    4944:       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
    4945:       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
    4946:       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
    4947:       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
    4948:       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
    4949:       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
    4950:       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
    4951:       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
    4952:     } break;
    4953:     case DM_SHAPE_DOUBLET: {
    4954:       DM        dmnew;
    4955:       PetscReal rl = 0.0;

    4957:       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
    4958:       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
    4959:       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
    4960:       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
    4961:     } break;
    4962:     case DM_SHAPE_HYPERCUBIC: {
    4963:       PetscInt       *edges, overlap = 1;
    4964:       PetscReal      *lower, *upper;
    4965:       DMBoundaryType *bdt;
    4966:       PetscInt        n, d;

    4968:       *useCoordSpace = PETSC_FALSE;
    4969:       PetscCall(PetscMalloc4(dim, &edges, dim, &lower, dim, &upper, dim, &bdt));
    4970:       for (d = 0; d < dim; ++d) {
    4971:         edges[d] = 1;
    4972:         lower[d] = 0.;
    4973:         upper[d] = 1.;
    4974:         bdt[d]   = DM_BOUNDARY_PERIODIC;
    4975:       }
    4976:       n = dim;
    4977:       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", edges, &n, &flg));
    4978:       n = dim;
    4979:       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
    4980:       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
    4981:       n = dim;
    4982:       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
    4983:       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
    4984:       n = dim;
    4985:       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
    4986:       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
    4987:       PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", overlap, &overlap, NULL, 0));
    4988:       PetscCall(DMPlexCreateHypercubicMesh_Internal(dm, dim, lower, upper, edges, overlap, bdt));
    4989:       PetscCall(PetscFree4(edges, lower, upper, bdt));
    4990:     } break;
    4991:     default:
    4992:       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
    4993:     }
    4994:   }
    4995:   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
    4996:   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
    4997:   if (orient) PetscCall(DMPlexOrient(dm));
    4998:   // Allow label creation
    4999:   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_label_", &option, NULL, &flg));
    5000:   if (flg) {
    5001:     DMLabel     label;
    5002:     PetscInt   *points, cStart, cEnd, n;
    5003:     char        fulloption[PETSC_MAX_PATH_LEN];
    5004:     const char *name = &option[14];

    5006:     PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
    5007:     n = PetscMax(cEnd - cStart, 1024);
    5008:     PetscCall(PetscMalloc1(n, &points));
    5009:     PetscCall(DMCreateLabel(dm, name));
    5010:     PetscCall(DMGetLabel(dm, name, &label));
    5011:     fulloption[0] = '-';
    5012:     fulloption[1] = 0;
    5013:     PetscCall(PetscStrlcat(fulloption, option, PETSC_MAX_PATH_LEN));
    5014:     PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, NULL));
    5015:     for (PetscInt p = 0; p < n; ++p) PetscCall(DMLabelSetValue(label, points[p], 1));
    5016:     PetscCall(PetscFree(points));
    5017:   }
    5018:   // Allow cohesive label creation
    5019:   //   Faces are input, completed, and all points are marked with their depth
    5020:   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_cohesive_label_", &option, NULL, &flg));
    5021:   if (flg) {
    5022:     DMLabel   label;
    5023:     PetscInt  points[1024], n, pStart, pEnd, Nl = 1;
    5024:     PetscBool noCreate = PETSC_FALSE;
    5025:     char      fulloption[PETSC_MAX_PATH_LEN];
    5026:     char      name[PETSC_MAX_PATH_LEN];
    5027:     size_t    len;

    5029:     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
    5030:     PetscCall(PetscStrncpy(name, &option[23], PETSC_MAX_PATH_LEN));
    5031:     PetscCall(PetscStrlen(name, &len));
    5032:     if (name[len - 1] == '0') Nl = 10;
    5033:     for (PetscInt l = 0; l < Nl; ++l) {
    5034:       if (l > 0) name[len - 1] = (char)('0' + l);
    5035:       fulloption[0] = 0;
    5036:       PetscCall(PetscStrlcat(fulloption, "-dm_plex_cohesive_label_", 32));
    5037:       PetscCall(PetscStrlcat(fulloption, name, PETSC_MAX_PATH_LEN - 32));
    5038:       n = 1024;
    5039:       PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, &flg));
    5040:       if (!flg) break;
    5041:       PetscCall(DMHasLabel(dm, name, &noCreate));
    5042:       if (noCreate) {
    5043:         DMLabel         inlabel;
    5044:         IS              pointIS;
    5045:         const PetscInt *lpoints;
    5046:         PetscInt        pdep, ln, inval = points[0];
    5047:         char            newname[PETSC_MAX_PATH_LEN];

    5049:         PetscCheck(n == 1, comm, PETSC_ERR_ARG_WRONG, "Must specify a label value with this option");
    5050:         PetscCall(DMGetLabel(dm, name, &inlabel));
    5051:         PetscCall(DMLabelGetStratumIS(inlabel, inval, &pointIS));
    5052:         PetscCall(ISGetLocalSize(pointIS, &ln));
    5053:         PetscCall(ISGetIndices(pointIS, &lpoints));
    5054:         PetscCall(DMPlexGetPointDepth(dm, lpoints[0], &pdep));
    5055:         PetscCall(PetscSNPrintf(newname, PETSC_MAX_PATH_LEN, "%s%" PetscInt_FMT, name, points[0]));
    5056:         PetscCall(DMCreateLabel(dm, newname));
    5057:         PetscCall(DMGetLabel(dm, newname, &label));
    5058:         if (!pdep) PetscCall(ProcessCohesiveLabel_Vertices(dm, label, inlabel, inval, ln, lpoints));
    5059:         else PetscCall(ProcessCohesiveLabel_Faces(dm, label, ln, lpoints));
    5060:         PetscCall(ISRestoreIndices(pointIS, &lpoints));
    5061:         PetscCall(ISDestroy(&pointIS));
    5062:       } else {
    5063:         PetscCall(DMCreateLabel(dm, name));
    5064:         PetscCall(DMGetLabel(dm, name, &label));
    5065:         if (pStart >= pEnd) n = 0;
    5066:         PetscCall(ProcessCohesiveLabel_Faces(dm, label, n, points));
    5067:       }
    5068:       PetscCall(DMPlexOrientLabel(dm, label));
    5069:       PetscCall(DMPlexLabelCohesiveComplete(dm, label, NULL, 1, PETSC_FALSE, PETSC_FALSE, NULL));
    5070:     }
    5071:   }
    5072:   PetscCall(DMViewFromOptions(dm, NULL, "-created_dm_view"));
    5073:   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
    5074:   PetscFunctionReturn(PETSC_SUCCESS);
    5075: }

    5077: PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems PetscOptionsObject)
    5078: {
    5079:   DM_Plex  *mesh = (DM_Plex *)dm->data;
    5080:   PetscBool flg, flg2;
    5081:   char      bdLabel[PETSC_MAX_PATH_LEN];
    5082:   char      method[PETSC_MAX_PATH_LEN];

    5084:   PetscFunctionBegin;
    5085:   /* Handle viewing */
    5086:   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
    5087:   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level for all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
    5088:   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fvm", "Debug output level for all fvm computations", "DMPlexSNESComputeResidualFVM", 0, &mesh->printFVM, NULL, 0));
    5089:   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
    5090:   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
    5091:   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
    5092:   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_project", "Debug output level all projection computations", "DMPlexProject", 0, &mesh->printProject, NULL, 0));
    5093:   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
    5094:   if (flg) PetscCall(PetscLogDefaultBegin());
    5095:   // Interpolation
    5096:   PetscCall(PetscOptionsBool("-dm_plex_interpolate_prefer_tensor", "When different orderings exist, prefer the tensor order", "DMPlexSetInterpolationPreferTensor", mesh->interpolatePreferTensor, &mesh->interpolatePreferTensor, NULL));
    5097:   /* Labeling */
    5098:   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", NULL, bdLabel, sizeof(bdLabel), &flg));
    5099:   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
    5100:   /* Point Location */
    5101:   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
    5102:   /* Partitioning and distribution */
    5103:   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
    5104:   /* Reordering */
    5105:   PetscCall(PetscOptionsBool("-dm_reorder_section", "Compute point permutation for local section", "DMReorderSectionSetDefault", PETSC_FALSE, &flg2, &flg));
    5106:   if (flg) PetscCall(DMReorderSectionSetDefault(dm, flg2 ? DM_REORDER_DEFAULT_TRUE : DM_REORDER_DEFAULT_FALSE));
    5107:   PetscCall(PetscOptionsString("-dm_reorder_section_type", "Reordering method for local section", "DMReorderSectionSetType", NULL, method, PETSC_MAX_PATH_LEN, &flg));
    5108:   if (flg) PetscCall(DMReorderSectionSetType(dm, method));
    5109:   /* Generation and remeshing */
    5110:   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
    5111:   PetscCall(PetscOptionsBool("-dm_plex_save_transform", "Save the transform producing this mesh", "DMAdapt", PETSC_FALSE, &mesh->saveTransform, NULL));
    5112:   /* Projection behavior */
    5113:   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
    5114:   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
    5115:   /* Checking structure */
    5116:   {
    5117:     PetscBool all = PETSC_FALSE;

    5119:     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
    5120:     if (all) {
    5121:       PetscCall(DMPlexCheck(dm));
    5122:     } else {
    5123:       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
    5124:       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
    5125:       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
    5126:       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
    5127:       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
    5128:       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
    5129:       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
    5130:       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
    5131:       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
    5132:       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
    5133:       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
    5134:       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
    5135:     }
    5136:     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
    5137:     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
    5138:   }
    5139:   {
    5140:     PetscReal scale = 1.0;

    5142:     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
    5143:     if (flg) {
    5144:       Vec coordinates, coordinatesLocal;

    5146:       PetscCall(DMGetCoordinates(dm, &coordinates));
    5147:       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
    5148:       PetscCall(VecScale(coordinates, scale));
    5149:       PetscCall(VecScale(coordinatesLocal, scale));
    5150:     }
    5151:   }
    5152:   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
    5153:   PetscFunctionReturn(PETSC_SUCCESS);
    5154: }

    5156: PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems PetscOptionsObject, PetscInt *overlap)
    5157: {
    5158:   PetscInt  numOvLabels = 16, numOvExLabels = 16;
    5159:   char     *ovLabelNames[16], *ovExLabelNames[16];
    5160:   PetscInt  numOvValues = 16, numOvExValues = 16, l;
    5161:   PetscBool flg;

    5163:   PetscFunctionBegin;
    5164:   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
    5165:   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
    5166:   if (!flg) numOvLabels = 0;
    5167:   if (numOvLabels) {
    5168:     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
    5169:     for (l = 0; l < numOvLabels; ++l) {
    5170:       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
    5171:       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
    5172:       PetscCall(PetscFree(ovLabelNames[l]));
    5173:     }
    5174:     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
    5175:     if (!flg) numOvValues = 0;
    5176:     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);

    5178:     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
    5179:     if (!flg) numOvExLabels = 0;
    5180:     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
    5181:     for (l = 0; l < numOvExLabels; ++l) {
    5182:       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
    5183:       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
    5184:       PetscCall(PetscFree(ovExLabelNames[l]));
    5185:     }
    5186:     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
    5187:     if (!flg) numOvExValues = 0;
    5188:     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
    5189:   }
    5190:   PetscFunctionReturn(PETSC_SUCCESS);
    5191: }

    5193: static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems PetscOptionsObject)
    5194: {
    5195:   PetscFunctionList    ordlist;
    5196:   char                 oname[256];
    5197:   char                 sublabelname[PETSC_MAX_PATH_LEN] = "";
    5198:   DMReorderDefaultFlag reorder;
    5199:   PetscReal            volume    = -1.0;
    5200:   PetscInt             prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
    5201:   PetscBool            uniformOrig = PETSC_FALSE, uniform = PETSC_TRUE, distribute, saveSF = PETSC_FALSE, interpolate = PETSC_TRUE, coordSpace = PETSC_FALSE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, flg;

    5203:   PetscFunctionBegin;
    5204:   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
    5205:   if (dm->cloneOpts) goto non_refine;
    5206:   /* Handle automatic creation */
    5207:   PetscCall(DMGetDimension(dm, &dim));
    5208:   if (dim < 0) PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
    5209:   PetscCall(DMGetDimension(dm, &dim));
    5210:   /* Handle interpolation before distribution */
    5211:   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
    5212:   if (flg) {
    5213:     DMPlexInterpolatedFlag interpolated;

    5215:     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
    5216:     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
    5217:       DM udm;

    5219:       PetscCall(DMPlexUninterpolate(dm, &udm));
    5220:       PetscCall(DMPlexReplace_Internal(dm, &udm));
    5221:     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
    5222:       DM idm;

    5224:       PetscCall(DMPlexInterpolate(dm, &idm));
    5225:       PetscCall(DMPlexReplace_Internal(dm, &idm));
    5226:     }
    5227:   }
    5228:   // Handle submesh selection before distribution
    5229:   PetscCall(PetscOptionsString("-dm_plex_submesh", "Label to use for submesh selection", "", sublabelname, sublabelname, PETSC_MAX_PATH_LEN, &flg));
    5230:   if (flg) {
    5231:     DM              subdm;
    5232:     DMLabel         label;
    5233:     IS              valueIS, pointIS;
    5234:     const PetscInt *values, *points;
    5235:     PetscBool       markedFaces = PETSC_FALSE;
    5236:     PetscInt        Nv, value, Np;

    5238:     PetscCall(DMGetLabel(dm, sublabelname, &label));
    5239:     PetscCall(DMLabelGetNumValues(label, &Nv));
    5240:     PetscCheck(Nv == 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Only a single label value is currently supported for submesh selection, not %" PetscInt_FMT, Nv);
    5241:     PetscCall(DMLabelGetValueIS(label, &valueIS));
    5242:     PetscCall(ISGetIndices(valueIS, &values));
    5243:     value = values[0];
    5244:     PetscCall(ISRestoreIndices(valueIS, &values));
    5245:     PetscCall(ISDestroy(&valueIS));
    5246:     PetscCall(DMLabelGetStratumSize(label, value, &Np));
    5247:     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
    5248:     PetscCall(ISGetIndices(pointIS, &points));
    5249:     for (PetscInt p = 0; p < Np; ++p) {
    5250:       PetscInt pdepth;

    5252:       PetscCall(DMPlexGetPointDepth(dm, points[p], &pdepth));
    5253:       if (pdepth) {
    5254:         markedFaces = PETSC_TRUE;
    5255:         break;
    5256:       }
    5257:     }
    5258:     PetscCall(ISRestoreIndices(pointIS, &points));
    5259:     PetscCall(ISDestroy(&pointIS));
    5260:     PetscCall(DMPlexCreateSubmesh(dm, label, value, markedFaces, &subdm));
    5261:     PetscCall(DMPlexReplace_Internal(dm, &subdm));
    5262:     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
    5263:   }
    5264:   /* Handle DMPlex refinement before distribution */
    5265:   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
    5266:   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
    5267:   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
    5268:   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
    5269:   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
    5270:   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
    5271:   if (flg) {
    5272:     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
    5273:     PetscCall(DMPlexSetRefinementLimit(dm, volume));
    5274:     prerefine = PetscMax(prerefine, 1);
    5275:   }
    5276:   if (prerefine) PetscCall(DMLocalizeCoordinates(dm));
    5277:   for (r = 0; r < prerefine; ++r) {
    5278:     DM            rdm;
    5279:     PetscPointFn *coordFunc;

    5281:     PetscCall(DMPlexGetCoordinateMap(dm, &coordFunc));
    5282:     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
    5283:     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
    5284:     PetscCall(DMPlexReplace_Internal(dm, &rdm));
    5285:     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
    5286:     if (coordFunc && remap) {
    5287:       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
    5288:       PetscCall(DMPlexSetCoordinateMap(dm, coordFunc));
    5289:     }
    5290:   }
    5291:   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
    5292:   /* Handle DMPlex extrusion before distribution */
    5293:   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
    5294:   if (extLayers) {
    5295:     DM edm;

    5297:     PetscCall(DMExtrude(dm, extLayers, &edm));
    5298:     PetscCall(DMPlexReplace_Internal(dm, &edm));
    5299:     PetscCall(DMPlexSetCoordinateMap(dm, NULL));
    5300:     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
    5301:     extLayers = 0;
    5302:     PetscCall(DMGetDimension(dm, &dim));
    5303:   }
    5304:   /* Handle DMPlex reordering before distribution */
    5305:   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
    5306:   PetscCall(MatGetOrderingList(&ordlist));
    5307:   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
    5308:   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
    5309:   if (reorder == DM_REORDER_DEFAULT_TRUE || flg) {
    5310:     DM pdm;
    5311:     IS perm;

    5313:     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
    5314:     PetscCall(DMPlexPermute(dm, perm, &pdm));
    5315:     PetscCall(ISDestroy(&perm));
    5316:     PetscCall(DMPlexReplace_Internal(dm, &pdm));
    5317:     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
    5318:   }
    5319:   /* Handle DMPlex distribution */
    5320:   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
    5321:   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
    5322:   PetscCall(PetscOptionsBool("-dm_distribute_save_sf", "Flag to save the migration SF", "DMPlexSetMigrationSF", saveSF, &saveSF, NULL));
    5323:   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
    5324:   if (distribute) {
    5325:     DM               pdm = NULL;
    5326:     PetscPartitioner part;
    5327:     PetscSF          sfMigration;
    5328:     PetscBool        use_partition_balance;

    5330:     PetscCall(DMPlexGetPartitioner(dm, &part));
    5331:     PetscCall(PetscPartitionerSetFromOptions(part));
    5332:     PetscCall(DMPlexGetPartitionBalance(dm, &use_partition_balance));
    5333:     PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", use_partition_balance, &use_partition_balance, &flg));
    5334:     if (flg) PetscCall(DMPlexSetPartitionBalance(dm, use_partition_balance));
    5335:     PetscCall(DMPlexDistribute(dm, overlap, &sfMigration, &pdm));
    5336:     if (pdm) {
    5337:       // Delete the local section to force the existing one to be rebuilt with the distributed DM
    5338:       PetscCall(DMSetLocalSection(dm, pdm->localSection));
    5339:       PetscCall(DMPlexReplace_Internal(dm, &pdm));
    5340:     }
    5341:     if (saveSF) PetscCall(DMPlexSetMigrationSF(dm, sfMigration));
    5342:     PetscCall(PetscSFDestroy(&sfMigration));
    5343:   }

    5345:   {
    5346:     PetscBool useBoxLabel = PETSC_FALSE;
    5347:     PetscCall(PetscOptionsBool("-dm_plex_box_label", "Create 'Face Sets' assuming boundary faces align with cartesian directions", "DMCreate", useBoxLabel, &useBoxLabel, NULL));
    5348:     if (useBoxLabel) {
    5349:       PetscInt       n      = 3;
    5350:       DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};

    5352:       PetscCall(PetscOptionsEnumArray("-dm_plex_box_label_bd", "Boundary type for each dimension when using -dm_plex_box_label", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
    5353:       PetscCheck(!flg || !(n != dim), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
    5354:       PetscCall(DMPlexSetBoxLabel_Internal(dm, bdt));
    5355:     }
    5356:   }
    5357:   /* Must check CEED options before creating function space for coordinates */
    5358:   {
    5359:     PetscBool useCeed = PETSC_FALSE, flg;

    5361:     PetscCall(PetscOptionsBool("-dm_plex_use_ceed", "Use LibCEED as the FEM backend", "DMPlexSetUseCeed", useCeed, &useCeed, &flg));
    5362:     if (flg) PetscCall(DMPlexSetUseCeed(dm, useCeed));
    5363:   }
    5364:   /* Create coordinate space */
    5365:   PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, NULL));
    5366:   if (coordSpace) {
    5367:     PetscInt  degree = 1, deg;
    5368:     PetscInt  height = 0;
    5369:     DM        cdm;
    5370:     PetscBool localize = PETSC_TRUE, sparseLocalize = PETSC_TRUE;

    5372:     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
    5373:     PetscCall(DMGetCoordinateDegree_Internal(dm, &deg));
    5374:     if (coordSpace && deg <= 1) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, PETSC_FALSE, PETSC_TRUE));
    5375:     PetscCall(DMGetCoordinateDM(dm, &cdm));
    5376:     if (!coordSpace) {
    5377:       PetscDS      cds;
    5378:       PetscObject  obj;
    5379:       PetscClassId id;

    5381:       PetscCall(DMGetDS(cdm, &cds));
    5382:       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
    5383:       PetscCall(PetscObjectGetClassId(obj, &id));
    5384:       if (id == PETSCFE_CLASSID) {
    5385:         PetscContainer dummy;

    5387:         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
    5388:         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
    5389:         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
    5390:         PetscCall(PetscContainerDestroy(&dummy));
    5391:         PetscCall(DMClearDS(cdm));
    5392:       }
    5393:       PetscCall(DMPlexSetCoordinateMap(dm, NULL));
    5394:     }
    5395:     PetscCall(PetscOptionsBool("-dm_localize", "Localize mesh coordinates", "", localize, &localize, NULL));
    5396:     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "DMSetSparseLocalize", sparseLocalize, &sparseLocalize, &flg));
    5397:     if (flg) PetscCall(DMSetSparseLocalize(dm, sparseLocalize));
    5398:     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
    5399:     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
    5400:     if (localize) PetscCall(DMLocalizeCoordinates(dm));
    5401:   }
    5402:   /* Handle DMPlex refinement */
    5403:   remap = PETSC_TRUE;
    5404:   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
    5405:   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
    5406:   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
    5407:   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
    5408:   if (refine && isHierarchy) {
    5409:     DM *dms, coarseDM;

    5411:     PetscCall(DMGetCoarseDM(dm, &coarseDM));
    5412:     PetscCall(PetscObjectReference((PetscObject)coarseDM));
    5413:     PetscCall(PetscMalloc1(refine, &dms));
    5414:     PetscCall(DMRefineHierarchy(dm, refine, dms));
    5415:     /* Total hack since we do not pass in a pointer */
    5416:     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
    5417:     if (refine == 1) {
    5418:       PetscCall(DMSetCoarseDM(dm, dms[0]));
    5419:       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
    5420:     } else {
    5421:       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
    5422:       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
    5423:       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
    5424:       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
    5425:     }
    5426:     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
    5427:     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
    5428:     /* Free DMs */
    5429:     for (r = 0; r < refine; ++r) {
    5430:       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
    5431:       PetscCall(DMDestroy(&dms[r]));
    5432:     }
    5433:     PetscCall(PetscFree(dms));
    5434:   } else {
    5435:     for (r = 0; r < refine; ++r) {
    5436:       DM            rdm;
    5437:       PetscPointFn *coordFunc;

    5439:       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
    5440:       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
    5441:       /* Total hack since we do not pass in a pointer */
    5442:       PetscCall(DMPlexReplace_Internal(dm, &rdm));
    5443:       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
    5444:       PetscCall(DMPlexGetCoordinateMap(dm, &coordFunc));
    5445:       if (coordFunc && remap) {
    5446:         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
    5447:         PetscCall(DMPlexSetCoordinateMap(dm, coordFunc));
    5448:       }
    5449:     }
    5450:   }
    5451:   /* Handle DMPlex coarsening */
    5452:   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
    5453:   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
    5454:   if (coarsen && isHierarchy) {
    5455:     DM *dms;

    5457:     PetscCall(PetscMalloc1(coarsen, &dms));
    5458:     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
    5459:     /* Free DMs */
    5460:     for (r = 0; r < coarsen; ++r) {
    5461:       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
    5462:       PetscCall(DMDestroy(&dms[r]));
    5463:     }
    5464:     PetscCall(PetscFree(dms));
    5465:   } else {
    5466:     for (r = 0; r < coarsen; ++r) {
    5467:       DM            cdm;
    5468:       PetscPointFn *coordFunc;

    5470:       PetscCall(DMPlexGetCoordinateMap(dm, &coordFunc));
    5471:       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
    5472:       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
    5473:       /* Total hack since we do not pass in a pointer */
    5474:       PetscCall(DMPlexReplace_Internal(dm, &cdm));
    5475:       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
    5476:       if (coordFunc) {
    5477:         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
    5478:         PetscCall(DMPlexSetCoordinateMap(dm, coordFunc));
    5479:       }
    5480:     }
    5481:   }
    5482:   // Handle coordinate remapping
    5483:   remap = PETSC_FALSE;
    5484:   PetscCall(PetscOptionsBool("-dm_coord_remap", "Flag to control coordinate remapping", "", remap, &remap, NULL));
    5485:   if (remap) {
    5486:     DMPlexCoordMap map     = DM_COORD_MAP_NONE;
    5487:     PetscPointFn  *mapFunc = NULL;
    5488:     PetscScalar    params[16];
    5489:     PetscInt       Np = PETSC_STATIC_ARRAY_LENGTH(params), cdim;
    5490:     MPI_Comm       comm;

    5492:     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
    5493:     PetscCall(DMGetCoordinateDim(dm, &cdim));
    5494:     PetscCall(PetscOptionsScalarArray("-dm_coord_map_params", "Parameters for the coordinate remapping", "", params, &Np, &flg));
    5495:     if (!flg) Np = 0;
    5496:     // TODO Allow user to pass a map function by name
    5497:     PetscCall(PetscOptionsEnum("-dm_coord_map", "Coordinate mapping for built-in mesh", "", DMPlexCoordMaps, (PetscEnum)map, (PetscEnum *)&map, &flg));
    5498:     if (flg) {
    5499:       switch (map) {
    5500:       case DM_COORD_MAP_NONE:
    5501:         mapFunc = coordMap_identity;
    5502:         break;
    5503:       case DM_COORD_MAP_SHEAR:
    5504:         mapFunc = coordMap_shear;
    5505:         if (!Np) {
    5506:           Np        = cdim + 1;
    5507:           params[0] = 0;
    5508:           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
    5509:         }
    5510:         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The shear coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
    5511:         break;
    5512:       case DM_COORD_MAP_FLARE:
    5513:         mapFunc = coordMap_flare;
    5514:         if (!Np) {
    5515:           Np        = cdim + 1;
    5516:           params[0] = 0;
    5517:           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
    5518:         }
    5519:         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The flare coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
    5520:         break;
    5521:       case DM_COORD_MAP_ANNULUS:
    5522:         mapFunc = coordMap_annulus;
    5523:         if (!Np) {
    5524:           Np        = 2;
    5525:           params[0] = 1.;
    5526:           params[1] = 2.;
    5527:         }
    5528:         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The annulus coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
    5529:         break;
    5530:       case DM_COORD_MAP_SHELL:
    5531:         mapFunc = coordMap_shell;
    5532:         if (!Np) {
    5533:           Np        = 2;
    5534:           params[0] = 1.;
    5535:           params[1] = 2.;
    5536:         }
    5537:         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The spherical shell coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
    5538:         break;
    5539:       case DM_COORD_MAP_SINUSOID:
    5540:         mapFunc = coordMap_sinusoid;
    5541:         if (!Np) {
    5542:           Np        = 3;
    5543:           params[0] = 1.;
    5544:           params[1] = 1.;
    5545:           params[2] = 1.;
    5546:         }
    5547:         PetscCheck(Np == 3, comm, PETSC_ERR_ARG_WRONG, "The sinusoidal coordinate map must have 3 parameters, not %" PetscInt_FMT, Np);
    5548:         break;
    5549:       default:
    5550:         mapFunc = coordMap_identity;
    5551:       }
    5552:     }
    5553:     if (Np) {
    5554:       DM      cdm;
    5555:       PetscDS cds;

    5557:       PetscCall(DMGetCoordinateDM(dm, &cdm));
    5558:       PetscCall(DMGetDS(cdm, &cds));
    5559:       PetscCall(PetscDSSetConstants(cds, Np, params));
    5560:     }
    5561:     PetscCall(DMPlexRemapGeometry(dm, 0.0, mapFunc));
    5562:   }
    5563:   /* Handle ghost cells */
    5564:   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
    5565:   if (ghostCells) {
    5566:     DM   gdm;
    5567:     char lname[PETSC_MAX_PATH_LEN];

    5569:     lname[0] = '\0';
    5570:     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
    5571:     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
    5572:     PetscCall(DMPlexReplace_Internal(dm, &gdm));
    5573:   }
    5574:   /* Handle 1D order */
    5575:   if (reorder != DM_REORDER_DEFAULT_FALSE && dim == 1) {
    5576:     DM           cdm, rdm;
    5577:     PetscDS      cds;
    5578:     PetscObject  obj;
    5579:     PetscClassId id = PETSC_OBJECT_CLASSID;
    5580:     IS           perm;
    5581:     PetscInt     Nf;
    5582:     PetscBool    distributed;

    5584:     PetscCall(DMPlexIsDistributed(dm, &distributed));
    5585:     PetscCall(DMGetCoordinateDM(dm, &cdm));
    5586:     PetscCall(DMGetDS(cdm, &cds));
    5587:     PetscCall(PetscDSGetNumFields(cds, &Nf));
    5588:     if (Nf) {
    5589:       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
    5590:       PetscCall(PetscObjectGetClassId(obj, &id));
    5591:     }
    5592:     if (!distributed && id != PETSCFE_CLASSID) {
    5593:       PetscCall(DMPlexGetOrdering1D(dm, &perm));
    5594:       PetscCall(DMPlexPermute(dm, perm, &rdm));
    5595:       PetscCall(DMPlexReplace_Internal(dm, &rdm));
    5596:       PetscCall(ISDestroy(&perm));
    5597:     }
    5598:   }
    5599: /* Handle */
    5600: non_refine:
    5601:   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
    5602:   char    *phases[16];
    5603:   PetscInt Nphases = 16;
    5604:   PetscCall(PetscOptionsStringArray("-dm_plex_option_phases", "Option phase prefixes", "DMSetFromOptions", phases, &Nphases, &flg));
    5605:   PetscOptionsHeadEnd();

    5607:   // Phases
    5608:   if (flg) {
    5609:     DM          cdm;
    5610:     char       *oldPrefix, *oldCoordPrefix;
    5611:     const char *tmp;

    5613:     PetscCall(DMGetCoordinateDM(dm, &cdm));
    5614:     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &tmp));
    5615:     PetscCall(PetscStrallocpy(tmp, &oldPrefix));
    5616:     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)cdm, &tmp));
    5617:     PetscCall(PetscStrallocpy(tmp, &oldCoordPrefix));
    5618:     for (PetscInt ph = 0; ph < Nphases; ++ph) {
    5619:       PetscCall(PetscObjectAppendOptionsPrefix((PetscObject)dm, phases[ph]));
    5620:       PetscCall(PetscObjectAppendOptionsPrefix((PetscObject)cdm, phases[ph]));
    5621:       PetscCall(PetscInfo(dm, "Options phase %s for DM %s\n", phases[ph], dm->hdr.name));
    5622:       PetscCall(DMSetFromOptions(dm));
    5623:       PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldPrefix));
    5624:       PetscCall(DMGetCoordinateDM(dm, &cdm));
    5625:       PetscCall(PetscObjectSetOptionsPrefix((PetscObject)cdm, oldCoordPrefix));
    5626:       PetscCall(PetscFree(phases[ph]));
    5627:     }
    5628:     PetscCall(PetscFree(oldPrefix));
    5629:     PetscCall(PetscFree(oldCoordPrefix));
    5630:   }
    5631:   PetscFunctionReturn(PETSC_SUCCESS);
    5632: }

    5634: static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
    5635: {
    5636:   PetscFunctionBegin;
    5637:   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
    5638:   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
    5639:   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (PetscErrorCodeFn *)VecView_Plex));
    5640:   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (PetscErrorCodeFn *)VecView_Plex_Native));
    5641:   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (PetscErrorCodeFn *)VecLoad_Plex));
    5642:   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (PetscErrorCodeFn *)VecLoad_Plex_Native));
    5643:   PetscFunctionReturn(PETSC_SUCCESS);
    5644: }

    5646: static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
    5647: {
    5648:   PetscFunctionBegin;
    5649:   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
    5650:   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (PetscErrorCodeFn *)VecView_Plex_Local));
    5651:   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (PetscErrorCodeFn *)VecLoad_Plex_Local));
    5652:   PetscFunctionReturn(PETSC_SUCCESS);
    5653: }

    5655: static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
    5656: {
    5657:   PetscInt depth, d;

    5659:   PetscFunctionBegin;
    5660:   PetscCall(DMPlexGetDepth(dm, &depth));
    5661:   if (depth == 1) {
    5662:     PetscCall(DMGetDimension(dm, &d));
    5663:     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
    5664:     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
    5665:     else {
    5666:       *pStart = 0;
    5667:       *pEnd   = 0;
    5668:     }
    5669:   } else {
    5670:     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
    5671:   }
    5672:   PetscFunctionReturn(PETSC_SUCCESS);
    5673: }

    5675: static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
    5676: {
    5677:   PetscSF            sf;
    5678:   PetscMPIInt        niranks, njranks;
    5679:   PetscInt           n;
    5680:   const PetscMPIInt *iranks, *jranks;
    5681:   DM_Plex           *data = (DM_Plex *)dm->data;

    5683:   PetscFunctionBegin;
    5684:   PetscCall(DMGetPointSF(dm, &sf));
    5685:   if (!data->neighbors) {
    5686:     PetscCall(PetscSFSetUp(sf));
    5687:     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
    5688:     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
    5689:     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
    5690:     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
    5691:     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
    5692:     n = njranks + niranks;
    5693:     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
    5694:     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
    5695:     PetscCall(PetscMPIIntCast(n, data->neighbors));
    5696:   }
    5697:   if (nranks) *nranks = data->neighbors[0];
    5698:   if (ranks) {
    5699:     if (data->neighbors[0]) *ranks = data->neighbors + 1;
    5700:     else *ranks = NULL;
    5701:   }
    5702:   PetscFunctionReturn(PETSC_SUCCESS);
    5703: }

    5705: PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);

    5707: static PetscErrorCode DMInitialize_Plex(DM dm)
    5708: {
    5709:   PetscFunctionBegin;
    5710:   dm->ops->view                      = DMView_Plex;
    5711:   dm->ops->load                      = DMLoad_Plex;
    5712:   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
    5713:   dm->ops->clone                     = DMClone_Plex;
    5714:   dm->ops->setup                     = DMSetUp_Plex;
    5715:   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
    5716:   dm->ops->createsectionpermutation  = DMCreateSectionPermutation_Plex;
    5717:   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
    5718:   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
    5719:   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
    5720:   dm->ops->getlocaltoglobalmapping   = NULL;
    5721:   dm->ops->createfieldis             = NULL;
    5722:   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
    5723:   dm->ops->createcellcoordinatedm    = DMCreateCellCoordinateDM_Plex;
    5724:   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
    5725:   dm->ops->getcoloring               = NULL;
    5726:   dm->ops->creatematrix              = DMCreateMatrix_Plex;
    5727:   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
    5728:   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
    5729:   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
    5730:   dm->ops->creategradientmatrix      = DMCreateGradientMatrix_Plex;
    5731:   dm->ops->createinjection           = DMCreateInjection_Plex;
    5732:   dm->ops->refine                    = DMRefine_Plex;
    5733:   dm->ops->coarsen                   = DMCoarsen_Plex;
    5734:   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
    5735:   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
    5736:   dm->ops->extrude                   = DMExtrude_Plex;
    5737:   dm->ops->globaltolocalbegin        = NULL;
    5738:   dm->ops->globaltolocalend          = NULL;
    5739:   dm->ops->localtoglobalbegin        = NULL;
    5740:   dm->ops->localtoglobalend          = NULL;
    5741:   dm->ops->destroy                   = DMDestroy_Plex;
    5742:   dm->ops->createsubdm               = DMCreateSubDM_Plex;
    5743:   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
    5744:   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
    5745:   dm->ops->locatepoints              = DMLocatePoints_Plex;
    5746:   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
    5747:   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
    5748:   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
    5749:   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
    5750:   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
    5751:   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
    5752:   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
    5753:   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
    5754:   dm->ops->getneighbors              = DMGetNeighbors_Plex;
    5755:   dm->ops->getlocalboundingbox       = DMGetLocalBoundingBox_Coordinates;
    5756:   dm->ops->createdomaindecomposition = DMCreateDomainDecomposition_Plex;
    5757:   dm->ops->createddscatters          = DMCreateDomainDecompositionScatters_Plex;
    5758:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
    5759:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerivativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
    5760:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBounds_C", DMPlexInsertBounds_Plex));
    5761:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
    5762:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
    5763:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
    5764:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
    5765:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
    5766:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
    5767:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetDefault_C", DMReorderSectionGetDefault_Plex));
    5768:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetDefault_C", DMReorderSectionSetDefault_Plex));
    5769:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetType_C", DMReorderSectionGetType_Plex));
    5770:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetType_C", DMReorderSectionSetType_Plex));
    5771:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
    5772:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
    5773:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
    5774:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetUseCeed_C", DMPlexGetUseCeed_Plex));
    5775:   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetUseCeed_C", DMPlexSetUseCeed_Plex));
    5776:   PetscFunctionReturn(PETSC_SUCCESS);
    5777: }

    5779: PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
    5780: {
    5781:   DM_Plex       *mesh = (DM_Plex *)dm->data;
    5782:   const PetscSF *face_sfs;
    5783:   PetscInt       num_face_sfs;

    5785:   PetscFunctionBegin;
    5786:   mesh->refct++;
    5787:   (*newdm)->data = mesh;
    5788:   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &face_sfs));
    5789:   PetscCall(DMPlexSetIsoperiodicFaceSF(*newdm, num_face_sfs, (PetscSF *)face_sfs));
    5790:   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
    5791:   PetscCall(DMInitialize_Plex(*newdm));
    5792:   PetscFunctionReturn(PETSC_SUCCESS);
    5793: }

    5795: /*MC
    5796:   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh (or grid), or CW Complex {cite}`cwcomplex`,
    5797:            which can be expressed using a Hasse Diagram {cite}`hassediagram`.
    5798:            In the local representation, `Vec`s contain all unknowns in the interior and shared boundary. This is
    5799:            specified by a `PetscSection` object. Ownership in the global representation is determined by
    5800:            ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.

    5802:   Options Database Keys:
    5803: + -dm_refine_pre                     - Refine mesh before distribution
    5804: + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
    5805: + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
    5806: . -dm_distribute                     - Distribute mesh across processes
    5807: . -dm_distribute_overlap             - Number of cells to overlap for distribution
    5808: . -dm_refine                         - Refine mesh after distribution
    5809: . -dm_localize <bool>                - Whether to localize coordinates for periodic meshes
    5810: . -dm_sparse_localize <bool>         - Whether to only localize cells on the periodic boundary
    5811: . -dm_plex_hash_location             - Use grid hashing for point location
    5812: . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
    5813: . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
    5814: . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
    5815: . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
    5816: . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
    5817: . -dm_plex_reorder_section           - Use specialized blocking if available
    5818: . -dm_plex_check_all                 - Perform all checks below
    5819: . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
    5820: . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
    5821: . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
    5822: . -dm_plex_check_geometry            - Check that cells have positive volume
    5823: . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
    5824: . -dm_plex_view_scale <num>          - Scale the TikZ
    5825: . -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
    5826: - -dm_plex_print_fvm <num>           - View FVM assembly information, such as flux updates

    5828:   Level: intermediate

    5830: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
    5831: M*/

    5833: PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
    5834: {
    5835:   DM_Plex *mesh;
    5836:   PetscInt unit;

    5838:   PetscFunctionBegin;
    5839:   PetscCall(PetscCitationsRegister(PlexCitation, &Plexcite));
    5841:   PetscCall(PetscNew(&mesh));
    5842:   dm->reorderSection = DM_REORDER_DEFAULT_NOTSET;
    5843:   dm->data           = mesh;

    5845:   mesh->refct = 1;
    5846:   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
    5847:   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
    5848:   mesh->refinementUniform       = PETSC_TRUE;
    5849:   mesh->refinementLimit         = -1.0;
    5850:   mesh->distDefault             = PETSC_TRUE;
    5851:   mesh->reorderDefault          = DM_REORDER_DEFAULT_NOTSET;
    5852:   mesh->distributionName        = NULL;
    5853:   mesh->interpolated            = DMPLEX_INTERPOLATED_INVALID;
    5854:   mesh->interpolatedCollective  = DMPLEX_INTERPOLATED_INVALID;
    5855:   mesh->interpolatePreferTensor = PETSC_TRUE;

    5857:   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
    5858:   mesh->remeshBd = PETSC_FALSE;

    5860:   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;

    5862:   mesh->depthState    = -1;
    5863:   mesh->celltypeState = -1;
    5864:   mesh->printTol      = 1.0e-10;
    5865:   mesh->nonempty_comm = MPI_COMM_SELF;

    5867:   PetscCall(DMInitialize_Plex(dm));
    5868:   PetscFunctionReturn(PETSC_SUCCESS);
    5869: }

    5871: /*@
    5872:   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.

    5874:   Collective

    5876:   Input Parameter:
    5877: . comm - The communicator for the `DMPLEX` object

    5879:   Output Parameter:
    5880: . mesh - The `DMPLEX` object

    5882:   Level: beginner

    5884: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMCreate()`, `DMSetType()`
    5885: @*/
    5886: PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
    5887: {
    5888:   PetscFunctionBegin;
    5889:   PetscAssertPointer(mesh, 2);
    5890:   PetscCall(DMCreate(comm, mesh));
    5891:   PetscCall(DMSetType(*mesh, DMPLEX));
    5892:   PetscFunctionReturn(PETSC_SUCCESS);
    5893: }

    5895: /*@C
    5896:   DMPlexBuildFromCellListParallel - Build a distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output) where all cells have the same celltype

    5898:   Collective; No Fortran Support

    5900:   Input Parameters:
    5901: + dm          - The `DM`
    5902: . numCells    - The number of cells owned by this process
    5903: . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
    5904: . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
    5905: . numCorners  - The number of vertices for each cell
    5906: - cells       - An array of $ numCells \times numCorners$ numbers, the global vertex numbers for each cell

    5908:   Output Parameters:
    5909: + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
    5910: - verticesAdjSaved - (Optional) vertex adjacency array

    5912:   Level: advanced

    5914:   Notes:
    5915:   Two triangles sharing a face
    5916: .vb

    5918:         2
    5919:       / | \
    5920:      /  |  \
    5921:     /   |   \
    5922:    0  0 | 1  3
    5923:     \   |   /
    5924:      \  |  /
    5925:       \ | /
    5926:         1
    5927: .ve
    5928:   would have input
    5929: .vb
    5930:   numCells = 2, numVertices = 4
    5931:   cells = [0 1 2  1 3 2]
    5932: .ve
    5933:   which would result in the `DMPLEX`
    5934: .vb

    5936:         4
    5937:       / | \
    5938:      /  |  \
    5939:     /   |   \
    5940:    2  0 | 1  5
    5941:     \   |   /
    5942:      \  |  /
    5943:       \ | /
    5944:         3
    5945: .ve

    5947:   Vertices are implicitly numbered consecutively $0, \ldots, \mathrm{NVertices}$.

    5949:   Each process owns a chunk of `numVertices` consecutive vertices.

    5951:   If `numVertices` is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using `PetscLayout`.

    5953:   If `NVertices` is `PETSC_DETERMINE` and `numVertices` is `PETSC_DECIDE`, `NVertices` is computed by PETSc as the maximum vertex index in $ cells + 1 $.

    5955:   If only `NVertices` is `PETSC_DETERMINE`, it is computed as the sum of `numVertices` over all processes.

    5957:   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.

    5959: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
    5960:           `PetscSF`
    5961: @*/
    5962: PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdjSaved[])
    5963: {
    5964:   PetscSF     sfPoint;
    5965:   PetscLayout layout;
    5966:   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;

    5968:   PetscFunctionBegin;
    5970:   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
    5971:   /* Get/check global number of vertices */
    5972:   {
    5973:     PetscInt       NVerticesInCells, i;
    5974:     const PetscInt len = numCells * numCorners;

    5976:     /* NVerticesInCells = max(cells) + 1 */
    5977:     NVerticesInCells = PETSC_INT_MIN;
    5978:     for (i = 0; i < len; i++)
    5979:       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
    5980:     ++NVerticesInCells;
    5981:     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));

    5983:     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
    5984:     else
    5985:       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
    5986:   }
    5987:   /* Count locally unique vertices */
    5988:   {
    5989:     PetscHSetI vhash;
    5990:     PetscInt   off = 0;

    5992:     PetscCall(PetscHSetICreate(&vhash));
    5993:     for (c = 0; c < numCells; ++c) {
    5994:       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
    5995:     }
    5996:     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
    5997:     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
    5998:     else verticesAdj = *verticesAdjSaved;
    5999:     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
    6000:     PetscCall(PetscHSetIDestroy(&vhash));
    6001:     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
    6002:   }
    6003:   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
    6004:   /* Create cones */
    6005:   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
    6006:   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
    6007:   PetscCall(DMSetUp(dm));
    6008:   PetscCall(DMPlexGetCones(dm, &cones));
    6009:   for (c = 0; c < numCells; ++c) {
    6010:     for (p = 0; p < numCorners; ++p) {
    6011:       const PetscInt gv = cells[c * numCorners + p];
    6012:       PetscInt       lv;

    6014:       /* Positions within verticesAdj form 0-based local vertex numbering;
    6015:          we need to shift it by numCells to get correct DAG points (cells go first) */
    6016:       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
    6017:       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
    6018:       cones[c * numCorners + p] = lv + numCells;
    6019:     }
    6020:   }
    6021:   /* Build point sf */
    6022:   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
    6023:   PetscCall(PetscLayoutSetSize(layout, NVertices));
    6024:   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
    6025:   PetscCall(PetscLayoutSetBlockSize(layout, 1));
    6026:   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
    6027:   PetscCall(PetscLayoutDestroy(&layout));
    6028:   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
    6029:   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
    6030:   if (dm->sf) {
    6031:     const char *prefix;

    6033:     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
    6034:     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
    6035:   }
    6036:   PetscCall(DMSetPointSF(dm, sfPoint));
    6037:   PetscCall(PetscSFDestroy(&sfPoint));
    6038:   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
    6039:   /* Fill in the rest of the topology structure */
    6040:   PetscCall(DMPlexSymmetrize(dm));
    6041:   PetscCall(DMPlexStratify(dm));
    6042:   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
    6043:   PetscFunctionReturn(PETSC_SUCCESS);
    6044: }

    6046: /*@C
    6047:   DMPlexBuildFromCellSectionParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output) allowing multiple celltypes

    6049:   Collective; No Fortran Support

    6051:   Input Parameters:
    6052: + dm          - The `DM`
    6053: . numCells    - The number of cells owned by this process
    6054: . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
    6055: . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
    6056: . cellSection - The `PetscSection` giving the number of vertices for each cell (layout of cells)
    6057: - cells       - An array of the global vertex numbers for each cell

    6059:   Output Parameters:
    6060: + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
    6061: - verticesAdjSaved - (Optional) vertex adjacency array

    6063:   Level: advanced

    6065:   Notes:
    6066:   A triangle and quadrilateral sharing a face
    6067: .vb
    6068:         2----------3
    6069:       / |          |
    6070:      /  |          |
    6071:     /   |          |
    6072:    0  0 |     1    |
    6073:     \   |          |
    6074:      \  |          |
    6075:       \ |          |
    6076:         1----------4
    6077: .ve
    6078:   would have input
    6079: .vb
    6080:   numCells = 2, numVertices = 5
    6081:   cells = [0 1 2  1 4 3 2]
    6082: .ve
    6083:   which would result in the `DMPLEX`
    6084: .vb
    6085:         4----------5
    6086:       / |          |
    6087:      /  |          |
    6088:     /   |          |
    6089:    2  0 |     1    |
    6090:     \   |          |
    6091:      \  |          |
    6092:       \ |          |
    6093:         3----------6
    6094: .ve

    6096:   Vertices are implicitly numbered consecutively 0,...,NVertices.
    6097:   Each rank owns a chunk of numVertices consecutive vertices.
    6098:   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
    6099:   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
    6100:   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.

    6102:   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.

    6104: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexCreateFromCellSectionParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
    6105:           `PetscSF`
    6106: @*/
    6107: PetscErrorCode DMPlexBuildFromCellSectionParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscSection cellSection, const PetscInt cells[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdjSaved[])
    6108: {
    6109:   PetscSF     sfPoint;
    6110:   PetscLayout layout;
    6111:   PetscInt    numVerticesAdj, *verticesAdj, *cones, cStart, cEnd, len;

    6113:   PetscFunctionBegin;
    6115:   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
    6116:   PetscCall(PetscSectionGetChart(cellSection, &cStart, &cEnd));
    6117:   PetscCall(PetscSectionGetStorageSize(cellSection, &len));
    6118:   PetscCheck(cStart == 0 && cEnd == numCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section chart [%" PetscInt_FMT ", %" PetscInt_FMT ") should be [0, %" PetscInt_FMT ")", cStart, cEnd, numCells);
    6119:   /* Get/check global number of vertices */
    6120:   {
    6121:     PetscInt NVerticesInCells;

    6123:     /* NVerticesInCells = max(cells) + 1 */
    6124:     NVerticesInCells = PETSC_MIN_INT;
    6125:     for (PetscInt i = 0; i < len; i++)
    6126:       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
    6127:     ++NVerticesInCells;
    6128:     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));

    6130:     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
    6131:     else
    6132:       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
    6133:   }
    6134:   /* Count locally unique vertices */
    6135:   {
    6136:     PetscHSetI vhash;
    6137:     PetscInt   off = 0;

    6139:     PetscCall(PetscHSetICreate(&vhash));
    6140:     for (PetscInt i = 0; i < len; i++) PetscCall(PetscHSetIAdd(vhash, cells[i]));
    6141:     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
    6142:     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
    6143:     else verticesAdj = *verticesAdjSaved;
    6144:     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
    6145:     PetscCall(PetscHSetIDestroy(&vhash));
    6146:     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
    6147:   }
    6148:   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
    6149:   /* Create cones */
    6150:   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
    6151:   for (PetscInt c = 0; c < numCells; ++c) {
    6152:     PetscInt dof;

    6154:     PetscCall(PetscSectionGetDof(cellSection, c, &dof));
    6155:     PetscCall(DMPlexSetConeSize(dm, c, dof));
    6156:   }
    6157:   PetscCall(DMSetUp(dm));
    6158:   PetscCall(DMPlexGetCones(dm, &cones));
    6159:   for (PetscInt c = 0; c < numCells; ++c) {
    6160:     PetscInt dof, off;

    6162:     PetscCall(PetscSectionGetDof(cellSection, c, &dof));
    6163:     PetscCall(PetscSectionGetOffset(cellSection, c, &off));
    6164:     for (PetscInt p = off; p < off + dof; ++p) {
    6165:       const PetscInt gv = cells[p];
    6166:       PetscInt       lv;

    6168:       /* Positions within verticesAdj form 0-based local vertex numbering;
    6169:          we need to shift it by numCells to get correct DAG points (cells go first) */
    6170:       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
    6171:       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
    6172:       cones[p] = lv + numCells;
    6173:     }
    6174:   }
    6175:   /* Build point sf */
    6176:   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
    6177:   PetscCall(PetscLayoutSetSize(layout, NVertices));
    6178:   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
    6179:   PetscCall(PetscLayoutSetBlockSize(layout, 1));
    6180:   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
    6181:   PetscCall(PetscLayoutDestroy(&layout));
    6182:   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
    6183:   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
    6184:   if (dm->sf) {
    6185:     const char *prefix;

    6187:     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
    6188:     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
    6189:   }
    6190:   PetscCall(DMSetPointSF(dm, sfPoint));
    6191:   PetscCall(PetscSFDestroy(&sfPoint));
    6192:   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
    6193:   /* Fill in the rest of the topology structure */
    6194:   PetscCall(DMPlexSymmetrize(dm));
    6195:   PetscCall(DMPlexStratify(dm));
    6196:   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
    6197:   PetscFunctionReturn(PETSC_SUCCESS);
    6198: }

    6200: /*@
    6201:   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)

    6203:   Collective; No Fortran Support

    6205:   Input Parameters:
    6206: + dm           - The `DM`
    6207: . spaceDim     - The spatial dimension used for coordinates
    6208: . sfVert       - `PetscSF` describing complete vertex ownership
    6209: - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex

    6211:   Level: advanced

    6213: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
    6214: @*/
    6215: PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
    6216: {
    6217:   PetscSection coordSection;
    6218:   Vec          coordinates;
    6219:   PetscScalar *coords;
    6220:   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
    6221:   PetscMPIInt  spaceDimi;

    6223:   PetscFunctionBegin;
    6224:   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
    6225:   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
    6226:   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
    6227:   PetscCall(DMSetCoordinateDim(dm, spaceDim));
    6228:   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
    6229:   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
    6230:   PetscCall(DMGetCoordinateSection(dm, &coordSection));
    6231:   PetscCall(PetscSectionSetNumFields(coordSection, 1));
    6232:   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
    6233:   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
    6234:   for (v = vStart; v < vEnd; ++v) {
    6235:     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
    6236:     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
    6237:   }
    6238:   PetscCall(PetscSectionSetUp(coordSection));
    6239:   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
    6240:   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
    6241:   PetscCall(VecSetBlockSize(coordinates, spaceDim));
    6242:   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    6243:   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
    6244:   PetscCall(VecSetType(coordinates, VECSTANDARD));
    6245:   PetscCall(VecGetArray(coordinates, &coords));
    6246:   {
    6247:     MPI_Datatype coordtype;

    6249:     /* Need a temp buffer for coords if we have complex/single */
    6250:     PetscCall(PetscMPIIntCast(spaceDim, &spaceDimi));
    6251:     PetscCallMPI(MPI_Type_contiguous(spaceDimi, MPIU_SCALAR, &coordtype));
    6252:     PetscCallMPI(MPI_Type_commit(&coordtype));
    6253: #if defined(PETSC_USE_COMPLEX)
    6254:     {
    6255:       PetscScalar *svertexCoords;
    6256:       PetscInt     i;
    6257:       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
    6258:       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
    6259:       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
    6260:       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
    6261:       PetscCall(PetscFree(svertexCoords));
    6262:     }
    6263: #else
    6264:     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
    6265:     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
    6266: #endif
    6267:     PetscCallMPI(MPI_Type_free(&coordtype));
    6268:   }
    6269:   PetscCall(VecRestoreArray(coordinates, &coords));
    6270:   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
    6271:   PetscCall(VecDestroy(&coordinates));
    6272:   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
    6273:   PetscFunctionReturn(PETSC_SUCCESS);
    6274: }

    6276: /*@
    6277:   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output) where all cells have the same celltype

    6279:   Collective

    6281:   Input Parameters:
    6282: + comm         - The communicator
    6283: . dim          - The topological dimension of the mesh
    6284: . numCells     - The number of cells owned by this process
    6285: . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
    6286: . NVertices    - The global number of vertices, or `PETSC_DECIDE`
    6287: . numCorners   - The number of vertices for each cell
    6288: . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
    6289: . cells        - An array of numCells*numCorners numbers, the global vertex numbers for each cell
    6290: . spaceDim     - The spatial dimension used for coordinates
    6291: - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex

    6293:   Output Parameters:
    6294: + dm          - The `DM`
    6295: . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
    6296: - verticesAdj - (Optional) vertex adjacency array

    6298:   Level: intermediate

    6300:   Notes:
    6301:   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
    6302:   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`

    6304:   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.

    6306:   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.

    6308: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
    6309: @*/
    6310: PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdj[], DM *dm)
    6311: {
    6312:   PetscSF sfVert;

    6314:   PetscFunctionBegin;
    6315:   PetscCall(DMCreate(comm, dm));
    6316:   PetscCall(DMSetType(*dm, DMPLEX));
    6319:   PetscCall(DMSetDimension(*dm, dim));
    6320:   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
    6321:   if (interpolate) {
    6322:     DM idm;

    6324:     PetscCall(DMPlexInterpolate(*dm, &idm));
    6325:     PetscCall(DMDestroy(dm));
    6326:     *dm = idm;
    6327:   }
    6328:   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
    6329:   if (vertexSF) *vertexSF = sfVert;
    6330:   else PetscCall(PetscSFDestroy(&sfVert));
    6331:   PetscFunctionReturn(PETSC_SUCCESS);
    6332: }

    6334: /*@
    6335:   DMPlexCreateFromCellSectionParallel - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output) and supports multiple celltypes

    6337:   Collective

    6339:   Input Parameters:
    6340: + comm         - The communicator
    6341: . dim          - The topological dimension of the mesh
    6342: . numCells     - The number of cells owned by this process
    6343: . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
    6344: . NVertices    - The global number of vertices, or `PETSC_DECIDE`
    6345: . cellSection  - The `PetscSection` giving the number of vertices for each cell (layout of cells)
    6346: . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
    6347: . cells        - An array of the global vertex numbers for each cell
    6348: . spaceDim     - The spatial dimension used for coordinates
    6349: - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex

    6351:   Output Parameters:
    6352: + dm          - The `DM`
    6353: . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
    6354: - verticesAdj - (Optional) vertex adjacency array

    6356:   Level: intermediate

    6358:   Notes:
    6359:   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
    6360:   `DMPlexBuildFromCellSectionParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`

    6362:   See `DMPlexBuildFromCellSectionParallel()` for an example and details about the topology-related parameters.

    6364:   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.

    6366: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
    6367: @*/
    6368: PetscErrorCode DMPlexCreateFromCellSectionParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscSection cellSection, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdj[], DM *dm)
    6369: {
    6370:   PetscSF sfVert;

    6372:   PetscFunctionBegin;
    6373:   PetscCall(DMCreate(comm, dm));
    6374:   PetscCall(DMSetType(*dm, DMPLEX));
    6377:   PetscCall(DMSetDimension(*dm, dim));
    6378:   PetscCall(DMPlexBuildFromCellSectionParallel(*dm, numCells, numVertices, NVertices, cellSection, cells, &sfVert, verticesAdj));
    6379:   if (interpolate) {
    6380:     DM idm;

    6382:     PetscCall(DMPlexInterpolate(*dm, &idm));
    6383:     PetscCall(DMDestroy(dm));
    6384:     *dm = idm;
    6385:   }
    6386:   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
    6387:   if (vertexSF) *vertexSF = sfVert;
    6388:   else PetscCall(PetscSFDestroy(&sfVert));
    6389:   PetscFunctionReturn(PETSC_SUCCESS);
    6390: }

    6392: /*@
    6393:   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)

    6395:   Collective; No Fortran Support

    6397:   Input Parameters:
    6398: + dm          - The `DM`
    6399: . numCells    - The number of cells owned by this process
    6400: . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
    6401: . numCorners  - The number of vertices for each cell
    6402: - cells       - An array of `numCells` x `numCorners` numbers, the global vertex numbers for each cell

    6404:   Level: advanced

    6406:   Notes:
    6407:   Two triangles sharing a face
    6408: .vb

    6410:         2
    6411:       / | \
    6412:      /  |  \
    6413:     /   |   \
    6414:    0  0 | 1  3
    6415:     \   |   /
    6416:      \  |  /
    6417:       \ | /
    6418:         1
    6419: .ve
    6420:   would have input
    6421: .vb
    6422:   numCells = 2, numVertices = 4
    6423:   cells = [0 1 2  1 3 2]
    6424: .ve
    6425:   which would result in the `DMPLEX`
    6426: .vb

    6428:         4
    6429:       / | \
    6430:      /  |  \
    6431:     /   |   \
    6432:    2  0 | 1  5
    6433:     \   |   /
    6434:      \  |  /
    6435:       \ | /
    6436:         3
    6437: .ve

    6439:   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.

    6441: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
    6442: @*/
    6443: PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
    6444: {
    6445:   PetscInt *cones, c, p, dim;

    6447:   PetscFunctionBegin;
    6448:   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
    6449:   PetscCall(DMGetDimension(dm, &dim));
    6450:   /* Get/check global number of vertices */
    6451:   {
    6452:     PetscInt       NVerticesInCells, i;
    6453:     const PetscInt len = numCells * numCorners;

    6455:     /* NVerticesInCells = max(cells) + 1 */
    6456:     NVerticesInCells = PETSC_INT_MIN;
    6457:     for (i = 0; i < len; i++)
    6458:       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
    6459:     ++NVerticesInCells;

    6461:     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
    6462:     else
    6463:       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
    6464:   }
    6465:   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
    6466:   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
    6467:   PetscCall(DMSetUp(dm));
    6468:   PetscCall(DMPlexGetCones(dm, &cones));
    6469:   for (c = 0; c < numCells; ++c) {
    6470:     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
    6471:   }
    6472:   PetscCall(DMPlexSymmetrize(dm));
    6473:   PetscCall(DMPlexStratify(dm));
    6474:   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
    6475:   PetscFunctionReturn(PETSC_SUCCESS);
    6476: }

    6478: /*@
    6479:   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)

    6481:   Collective

    6483:   Input Parameters:
    6484: + dm           - The `DM`
    6485: . spaceDim     - The spatial dimension used for coordinates
    6486: - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex

    6488:   Level: advanced

    6490: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
    6491: @*/
    6492: PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
    6493: {
    6494:   PetscSection coordSection;
    6495:   Vec          coordinates;
    6496:   DM           cdm;
    6497:   PetscScalar *coords;
    6498:   PetscInt     v, vStart, vEnd, d;

    6500:   PetscFunctionBegin;
    6501:   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
    6502:   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
    6503:   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
    6504:   PetscCall(DMSetCoordinateDim(dm, spaceDim));
    6505:   PetscCall(DMGetCoordinateSection(dm, &coordSection));
    6506:   PetscCall(PetscSectionSetNumFields(coordSection, 1));
    6507:   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
    6508:   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
    6509:   for (v = vStart; v < vEnd; ++v) {
    6510:     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
    6511:     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
    6512:   }
    6513:   PetscCall(PetscSectionSetUp(coordSection));

    6515:   PetscCall(DMGetCoordinateDM(dm, &cdm));
    6516:   PetscCall(DMCreateLocalVector(cdm, &coordinates));
    6517:   PetscCall(VecSetBlockSize(coordinates, spaceDim));
    6518:   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    6519:   PetscCall(VecGetArrayWrite(coordinates, &coords));
    6520:   for (v = 0; v < vEnd - vStart; ++v) {
    6521:     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
    6522:   }
    6523:   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
    6524:   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
    6525:   PetscCall(VecDestroy(&coordinates));
    6526:   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
    6527:   PetscFunctionReturn(PETSC_SUCCESS);
    6528: }

    6530: /*@
    6531:   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input

    6533:   Collective

    6535:   Input Parameters:
    6536: + comm         - The communicator
    6537: . dim          - The topological dimension of the mesh
    6538: . numCells     - The number of cells, only on process 0
    6539: . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
    6540: . numCorners   - The number of vertices for each cell, only on process 0
    6541: . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
    6542: . cells        - An array of $ numCells \times numCorners$ numbers, the vertices for each cell, only on process 0
    6543: . spaceDim     - The spatial dimension used for coordinates
    6544: - vertexCoords - An array of $ numVertices \times spaceDim$ numbers, the coordinates of each vertex, only on process 0

    6546:   Output Parameter:
    6547: . dm - The `DM`, which only has points on process 0

    6549:   Level: intermediate

    6551:   Notes:
    6552:   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
    6553:   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`

    6555:   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
    6556:   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
    6557:   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input

    6559: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
    6560: @*/
    6561: PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
    6562: {
    6563:   PetscMPIInt rank;

    6565:   PetscFunctionBegin;
    6566:   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
    6567:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
    6568:   PetscCall(DMCreate(comm, dm));
    6569:   PetscCall(DMSetType(*dm, DMPLEX));
    6570:   PetscCall(DMSetDimension(*dm, dim));
    6571:   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
    6572:   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
    6573:   if (interpolate) {
    6574:     DM idm;

    6576:     PetscCall(DMPlexInterpolate(*dm, &idm));
    6577:     PetscCall(DMDestroy(dm));
    6578:     *dm = idm;
    6579:   }
    6580:   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
    6581:   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
    6582:   PetscFunctionReturn(PETSC_SUCCESS);
    6583: }

    6585: /*@
    6586:   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a `DM`

    6588:   Input Parameters:
    6589: + dm               - The empty `DM` object, usually from `DMCreate()` and `DMSetDimension()`
    6590: . depth            - The depth of the DAG
    6591: . numPoints        - Array of size $ depth + 1 $ containing the number of points at each `depth`
    6592: . coneSize         - The cone size of each point
    6593: . cones            - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
    6594: . coneOrientations - The orientation of each cone point
    6595: - vertexCoords     - An array of $ numPoints[0] \times spacedim $ numbers representing the coordinates of each vertex, with `spacedim` the value set via `DMSetCoordinateDim()`

    6597:   Output Parameter:
    6598: . dm - The `DM`

    6600:   Level: advanced

    6602:   Note:
    6603:   Two triangles sharing a face would have input
    6604: .vb
    6605:   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
    6606:   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
    6607:  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
    6608: .ve
    6609:   which would result in the DMPlex
    6610: .vb
    6611:         4
    6612:       / | \
    6613:      /  |  \
    6614:     /   |   \
    6615:    2  0 | 1  5
    6616:     \   |   /
    6617:      \  |  /
    6618:       \ | /
    6619:         3
    6620: .ve
    6621:   Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`

    6623:   Developer Note:
    6624:   This does not create anything so should not have create in the name.

    6626: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
    6627: @*/
    6628: PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
    6629: {
    6630:   Vec          coordinates;
    6631:   PetscSection coordSection;
    6632:   PetscScalar *coords;
    6633:   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;

    6635:   PetscFunctionBegin;
    6636:   PetscCall(DMGetDimension(dm, &dim));
    6637:   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
    6638:   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
    6639:   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
    6640:   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
    6641:   for (p = pStart; p < pEnd; ++p) {
    6642:     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
    6643:     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
    6644:   }
    6645:   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
    6646:   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
    6647:   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
    6648:     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
    6649:     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
    6650:   }
    6651:   PetscCall(DMPlexSymmetrize(dm));
    6652:   PetscCall(DMPlexStratify(dm));
    6653:   /* Build coordinates */
    6654:   PetscCall(DMGetCoordinateSection(dm, &coordSection));
    6655:   PetscCall(PetscSectionSetNumFields(coordSection, 1));
    6656:   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
    6657:   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
    6658:   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
    6659:     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
    6660:     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
    6661:   }
    6662:   PetscCall(PetscSectionSetUp(coordSection));
    6663:   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
    6664:   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
    6665:   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    6666:   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
    6667:   PetscCall(VecSetBlockSize(coordinates, PetscMax(dimEmbed, 1)));
    6668:   PetscCall(VecSetType(coordinates, VECSTANDARD));
    6669:   if (vertexCoords) {
    6670:     PetscCall(VecGetArray(coordinates, &coords));
    6671:     for (v = 0; v < numPoints[0]; ++v) {
    6672:       PetscInt off;

    6674:       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
    6675:       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
    6676:     }
    6677:   }
    6678:   PetscCall(VecRestoreArray(coordinates, &coords));
    6679:   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
    6680:   PetscCall(VecDestroy(&coordinates));
    6681:   PetscFunctionReturn(PETSC_SUCCESS);
    6682: }

    6684: /*
    6685:   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.

    6687:   Collective

    6689: + comm        - The MPI communicator
    6690: . filename    - Name of the .dat file
    6691: - interpolate - Create faces and edges in the mesh

    6693:   Output Parameter:
    6694: . dm  - The `DM` object representing the mesh

    6696:   Level: beginner

    6698:   Note:
    6699:   The format is the simplest possible:
    6700: .vb
    6701:   dim Ne Nv Nc Nl
    6702:   v_1 v_2 ... v_Nc
    6703:   ...
    6704:   x y z marker_1 ... marker_Nl
    6705: .ve

    6707:   Developer Note:
    6708:   Should use a `PetscViewer` not a filename

    6710: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
    6711: */
    6712: static PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
    6713: {
    6714:   DMLabel      marker;
    6715:   PetscViewer  viewer;
    6716:   Vec          coordinates;
    6717:   PetscSection coordSection;
    6718:   PetscScalar *coords;
    6719:   char         line[PETSC_MAX_PATH_LEN];
    6720:   PetscInt     cdim, coordSize, v, c, d;
    6721:   PetscMPIInt  rank;
    6722:   int          snum, dim, Nv, Nc, Ncn, Nl;

    6724:   PetscFunctionBegin;
    6725:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
    6726:   PetscCall(PetscViewerCreate(comm, &viewer));
    6727:   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
    6728:   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
    6729:   PetscCall(PetscViewerFileSetName(viewer, filename));
    6730:   if (rank == 0) {
    6731:     PetscCall(PetscViewerRead(viewer, line, 5, NULL, PETSC_STRING));
    6732:     snum = sscanf(line, "%d %d %d %d %d", &dim, &Nc, &Nv, &Ncn, &Nl);
    6733:     PetscCheck(snum == 5, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
    6734:   } else {
    6735:     Nc = Nv = Ncn = Nl = 0;
    6736:   }
    6737:   PetscCallMPI(MPI_Bcast(&dim, 1, MPI_INT, 0, comm));
    6738:   cdim = dim;
    6739:   PetscCall(DMCreate(comm, dm));
    6740:   PetscCall(DMSetType(*dm, DMPLEX));
    6741:   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
    6742:   PetscCall(DMSetDimension(*dm, dim));
    6743:   PetscCall(DMSetCoordinateDim(*dm, cdim));
    6744:   /* Read topology */
    6745:   if (rank == 0) {
    6746:     char     format[PETSC_MAX_PATH_LEN];
    6747:     PetscInt cone[8];
    6748:     int      vbuf[8], v;

    6750:     for (c = 0; c < Ncn; ++c) {
    6751:       format[c * 3 + 0] = '%';
    6752:       format[c * 3 + 1] = 'd';
    6753:       format[c * 3 + 2] = ' ';
    6754:     }
    6755:     format[Ncn * 3 - 1] = '\0';
    6756:     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
    6757:     PetscCall(DMSetUp(*dm));
    6758:     for (c = 0; c < Nc; ++c) {
    6759:       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
    6760:       switch (Ncn) {
    6761:       case 2:
    6762:         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
    6763:         break;
    6764:       case 3:
    6765:         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
    6766:         break;
    6767:       case 4:
    6768:         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
    6769:         break;
    6770:       case 6:
    6771:         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
    6772:         break;
    6773:       case 8:
    6774:         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
    6775:         break;
    6776:       default:
    6777:         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
    6778:       }
    6779:       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
    6780:       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
    6781:       /* Hexahedra are inverted */
    6782:       if (Ncn == 8) {
    6783:         PetscInt tmp = cone[1];
    6784:         cone[1]      = cone[3];
    6785:         cone[3]      = tmp;
    6786:       }
    6787:       PetscCall(DMPlexSetCone(*dm, c, cone));
    6788:     }
    6789:   }
    6790:   PetscCall(DMPlexSymmetrize(*dm));
    6791:   PetscCall(DMPlexStratify(*dm));
    6792:   /* Read coordinates */
    6793:   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
    6794:   PetscCall(PetscSectionSetNumFields(coordSection, 1));
    6795:   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
    6796:   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
    6797:   for (v = Nc; v < Nc + Nv; ++v) {
    6798:     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
    6799:     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
    6800:   }
    6801:   PetscCall(PetscSectionSetUp(coordSection));
    6802:   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
    6803:   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
    6804:   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    6805:   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
    6806:   PetscCall(VecSetBlockSize(coordinates, cdim));
    6807:   PetscCall(VecSetType(coordinates, VECSTANDARD));
    6808:   PetscCall(VecGetArray(coordinates, &coords));
    6809:   if (rank == 0) {
    6810:     char   format[PETSC_MAX_PATH_LEN];
    6811:     double x[3];
    6812:     int    l, val[3];

    6814:     if (Nl) {
    6815:       for (l = 0; l < Nl; ++l) {
    6816:         format[l * 3 + 0] = '%';
    6817:         format[l * 3 + 1] = 'd';
    6818:         format[l * 3 + 2] = ' ';
    6819:       }
    6820:       format[Nl * 3 - 1] = '\0';
    6821:       PetscCall(DMCreateLabel(*dm, "marker"));
    6822:       PetscCall(DMGetLabel(*dm, "marker", &marker));
    6823:     }
    6824:     for (v = 0; v < Nv; ++v) {
    6825:       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
    6826:       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
    6827:       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
    6828:       switch (Nl) {
    6829:       case 0:
    6830:         snum = 0;
    6831:         break;
    6832:       case 1:
    6833:         snum = sscanf(line, format, &val[0]);
    6834:         break;
    6835:       case 2:
    6836:         snum = sscanf(line, format, &val[0], &val[1]);
    6837:         break;
    6838:       case 3:
    6839:         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
    6840:         break;
    6841:       default:
    6842:         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
    6843:       }
    6844:       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
    6845:       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
    6846:       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
    6847:     }
    6848:   }
    6849:   PetscCall(VecRestoreArray(coordinates, &coords));
    6850:   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
    6851:   PetscCall(VecDestroy(&coordinates));
    6852:   PetscCall(PetscViewerDestroy(&viewer));
    6853:   if (interpolate) {
    6854:     DM      idm;
    6855:     DMLabel bdlabel;

    6857:     PetscCall(DMPlexInterpolate(*dm, &idm));
    6858:     PetscCall(DMDestroy(dm));
    6859:     *dm = idm;

    6861:     if (!Nl) {
    6862:       PetscCall(DMCreateLabel(*dm, "marker"));
    6863:       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
    6864:       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
    6865:       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
    6866:     }
    6867:   }
    6868:   PetscFunctionReturn(PETSC_SUCCESS);
    6869: }

    6871: /*
    6872:   DMPlexCreateSTLFromFile - Create a `DMPLEX` mesh from an STL file.

    6874:   Collective

    6876: + comm        - The MPI communicator
    6877: . filename    - Name of the .dat file
    6878: - interpolate - Create faces and edges in the mesh

    6880:   Output Parameter:
    6881: . dm  - The `DM` object representing the mesh

    6883:   Level: beginner

    6885:   Notes:
    6886:   The binary format is:
    6887: .vb
    6888:   UINT8[80]    - Header               - 80 bytes
    6889:   UINT32       - Number of triangles  - 04 bytes
    6890:   foreach triangle                    - 50 bytes
    6891:     REAL32[3] - Normal vector         - 12 bytes
    6892:     REAL32[3] - Vertex 1              - 12 bytes
    6893:     REAL32[3] - Vertex 2              - 12 bytes
    6894:     REAL32[3] - Vertex 3              - 12 bytes
    6895:     UINT16    - Attribute byte count  - 02 bytes
    6896:   end
    6897: .ve

    6899:   The format is here <https://en.wikipedia.org/wiki/STL_(file_format)>

    6901: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
    6902: */
    6903: static PetscErrorCode DMPlexCreateSTLFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
    6904: {
    6905:   PetscViewer  viewer;
    6906:   Vec          coordinates;
    6907:   PetscSection coordSection;
    6908:   PetscScalar *coords;
    6909:   PetscInt     coordSize;
    6910:   char         line[PETSC_MAX_PATH_LEN];
    6911:   PetscBT      bt;
    6912:   float       *trialCoords = NULL;
    6913:   PetscInt    *cells       = NULL;
    6914:   PetscBool    isascii;
    6915:   PetscMPIInt  rank;
    6916:   PetscInt     Nc, Nv, nv = 0;

    6918:   PetscFunctionBegin;
    6919:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
    6920:   PetscCall(PetscViewerCreate(comm, &viewer));
    6921:   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERBINARY));
    6922:   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
    6923:   PetscCall(PetscViewerFileSetName(viewer, filename));
    6924:   if (rank == 0) {
    6925:     int num;

    6927:     PetscCall(PetscViewerRead(viewer, line, 5, NULL, PETSC_CHAR));
    6928:     line[5] = 0;
    6929:     PetscCall(PetscStrncmp(line, "solid", 5, &isascii));
    6930:     PetscCheck(!isascii, comm, PETSC_ERR_SUP, "No support for ASCII yet");
    6931:     PetscCall(PetscViewerRead(viewer, line, 75, NULL, PETSC_CHAR));
    6932:     line[75] = 0;
    6933:     PetscCall(PetscViewerRead(viewer, &num, 1, NULL, PETSC_INT32));
    6934:     PetscCall(PetscByteSwap(&num, PETSC_INT32, 1));
    6935:     Nc = num;
    6936:     // Read facet coordinates
    6937:     PetscCall(PetscMalloc1(Nc * 9, &trialCoords));
    6938:     for (PetscInt c = 0; c < Nc; ++c) {
    6939:       double    normal[3];
    6940:       short int dummy;

    6942:       PetscCall(PetscViewerRead(viewer, normal, 3, NULL, PETSC_FLOAT));
    6943:       PetscCall(PetscViewerRead(viewer, &trialCoords[c * 9 + 0], 9, NULL, PETSC_FLOAT));
    6944:       PetscCall(PetscByteSwap(&trialCoords[c * 9 + 0], PETSC_FLOAT, 9));
    6945:       PetscCall(PetscViewerRead(viewer, &dummy, 1, NULL, PETSC_SHORT));
    6946:     }
    6947:     PetscCall(PetscMalloc1(Nc * 3, &cells));
    6948:     // Find unique vertices
    6949:     PetscCall(PetscBTCreate(Nc * 3, &bt));
    6950:     PetscCall(PetscBTMemzero(Nc * 3, bt));
    6951:     PetscCall(PetscBTSet(bt, 0));
    6952:     PetscCall(PetscBTSet(bt, 1));
    6953:     PetscCall(PetscBTSet(bt, 2));
    6954:     Nv       = 0;
    6955:     cells[0] = Nc + Nv++;
    6956:     cells[1] = Nc + Nv++;
    6957:     cells[2] = Nc + Nv++;
    6958:     for (PetscInt v = 3; v < Nc * 3; ++v) {
    6959:       PetscBool match = PETSC_FALSE;

    6961:       for (PetscInt w = 0, x = 0; w < v; ++w) {
    6962:         if (PetscBTLookup(bt, w)) {
    6963:           if (trialCoords[v * 3 + 0] == trialCoords[w * 3 + 0] && trialCoords[v * 3 + 1] == trialCoords[w * 3 + 1] && trialCoords[v * 3 + 2] == trialCoords[w * 3 + 2]) {
    6964:             match    = PETSC_TRUE;
    6965:             cells[v] = Nc + x;
    6966:             break;
    6967:           }
    6968:           ++x;
    6969:         }
    6970:       }
    6971:       if (!match) {
    6972:         PetscCall(PetscBTSet(bt, v));
    6973:         cells[v] = Nc + Nv++;
    6974:       }
    6975:     }
    6976:   } else {
    6977:     Nc = Nv = 0;
    6978:   }
    6979:   for (PetscInt v = 3; v < Nc * 3; ++v) PetscCheck(cells[v] < Nc + Nv, comm, PETSC_ERR_PLIB, "Invalid cells[%" PetscInt_FMT "]: %" PetscInt_FMT, v, cells[v]);
    6980:   PetscCall(DMCreate(comm, dm));
    6981:   PetscCall(DMSetType(*dm, DMPLEX));
    6982:   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
    6983:   PetscCall(DMSetDimension(*dm, 2));
    6984:   PetscCall(DMSetCoordinateDim(*dm, 3));
    6985:   for (PetscInt c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, 3));
    6986:   PetscCall(DMSetUp(*dm));
    6987:   for (PetscInt c = 0; c < Nc; ++c) PetscCall(DMPlexSetCone(*dm, c, &cells[c * 3]));
    6988:   PetscCall(PetscFree(cells));
    6989:   PetscCall(DMPlexSymmetrize(*dm));
    6990:   PetscCall(DMPlexStratify(*dm));

    6992:   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
    6993:   PetscCall(PetscSectionSetNumFields(coordSection, 1));
    6994:   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
    6995:   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
    6996:   for (PetscInt v = Nc; v < Nc + Nv; ++v) {
    6997:     PetscCall(PetscSectionSetDof(coordSection, v, 3));
    6998:     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
    6999:   }
    7000:   PetscCall(PetscSectionSetUp(coordSection));
    7001:   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
    7002:   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
    7003:   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    7004:   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
    7005:   PetscCall(VecSetBlockSize(coordinates, 3));
    7006:   PetscCall(VecSetType(coordinates, VECSTANDARD));
    7007:   PetscCall(VecGetArray(coordinates, &coords));
    7008:   for (PetscInt tv = 0; tv < Nc * 3; ++tv) {
    7009:     if (PetscBTLookup(bt, tv)) {
    7010:       for (PetscInt d = 0; d < 3; ++d) coords[nv * 3 + d] = trialCoords[tv * 3 + d];
    7011:       ++nv;
    7012:     }
    7013:   }
    7014:   PetscCheck(nv == Nv, comm, PETSC_ERR_PLIB, "Number of vertices copied %" PetscInt_FMT " != %" PetscInt_FMT " number of mesh vertices", nv, Nv);
    7015:   PetscCall(PetscFree(trialCoords));
    7016:   PetscCall(PetscBTDestroy(&bt));
    7017:   PetscCall(VecRestoreArray(coordinates, &coords));
    7018:   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
    7019:   PetscCall(VecDestroy(&coordinates));
    7020:   PetscCall(PetscViewerDestroy(&viewer));
    7021:   if (interpolate) {
    7022:     DM idm;

    7024:     PetscCall(DMPlexInterpolate(*dm, &idm));
    7025:     PetscCall(DMDestroy(dm));
    7026:     *dm = idm;
    7027:   }
    7028:   PetscFunctionReturn(PETSC_SUCCESS);
    7029: }

    7031: /*
    7032:   DMPlexCreateShapefile - Create a `DMPLEX` mesh from the ESRI Shapefile format.

    7034:   Collective

    7036: + comm   - The MPI communicator
    7037: - viewer - `PetscViewer` for the .shp file

    7039:   Output Parameter:
    7040: . dm - The `DM` object representing the mesh

    7042:   Level: beginner

    7044:   Note:
    7045:   The format is specified at [Wikipedia](https://en.wikipedia.org/wiki/Shapefile) and [ESRI](https://www.esri.com/content/dam/esrisites/sitecore-archive/Files/Pdfs/library/whitepapers/pdfs/shapefile.pdf).

    7047: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
    7048: */
    7049: static PetscErrorCode DMPlexCreateShapefile(MPI_Comm comm, PetscViewer viewer, DM *dm)
    7050: {
    7051:   Vec          coordinates;
    7052:   PetscSection coordSection;
    7053:   PetscScalar *coords;
    7054:   PetscInt     cdim   = 2, coordSize;
    7055:   int          dim    = 1, Nv, Nc, vOff;
    7056:   double      *points = NULL;
    7057:   double       mbr[4], zb[2], mb[2];
    7058:   PetscMPIInt  rank;

    7060:   PetscFunctionBegin;
    7061:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
    7062:   if (rank == 0) {
    7063:     PetscInt cnt;
    7064:     int      magic, dummy[5], len, version, shtype, totlen = 0;

    7066:     // Read header
    7067:     PetscCall(PetscViewerBinaryRead(viewer, &magic, 1, &cnt, PETSC_INT32));
    7068:     PetscCheck(cnt == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read bytes 0-3");
    7069:     PetscCheck(magic == 0x0000270a, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: invalid magic number %X", magic);
    7070:     PetscCall(PetscViewerBinaryRead(viewer, dummy, 5, &cnt, PETSC_INT32));
    7071:     PetscCheck(cnt == 5, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read bytes 4-23");
    7072:     PetscCall(PetscViewerBinaryRead(viewer, &len, 1, &cnt, PETSC_INT32));
    7073:     PetscCheck(cnt == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read bytes 24-27");
    7074:     PetscCall(PetscViewerBinaryRead(viewer, &version, 1, &cnt, PETSC_INT32));
    7075:     PetscCall(PetscByteSwap(&version, PETSC_INT32, 1));
    7076:     PetscCheck(cnt == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read bytes 28-31");
    7077:     PetscCall(PetscInfo(viewer, "Shapefile: version %d file length %d\n", version, len));
    7078:     PetscCall(PetscViewerBinaryRead(viewer, &shtype, 1, &cnt, PETSC_INT32));
    7079:     PetscCall(PetscByteSwap(&shtype, PETSC_INT32, 1));
    7080:     PetscCheck(cnt == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read bytes 32-35");
    7081:     PetscCall(PetscInfo(viewer, "Shapefile: shape type %d\n", shtype));
    7082:     PetscCall(PetscViewerBinaryRead(viewer, mbr, 4, &cnt, PETSC_DOUBLE));
    7083:     PetscCall(PetscByteSwap(mbr, PETSC_DOUBLE, 4));
    7084:     PetscCheck(cnt == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read bytes 36-67");
    7085:     PetscCall(PetscInfo(viewer, "Shapefile: minimum bounding rectangle (%g, %g) -- (%g, %g)\n", mbr[0], mbr[1], mbr[2], mbr[3]));
    7086:     PetscCall(PetscViewerBinaryRead(viewer, zb, 2, &cnt, PETSC_DOUBLE));
    7087:     PetscCall(PetscByteSwap(zb, PETSC_DOUBLE, 2));
    7088:     PetscCheck(cnt == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read bytes 68-83");
    7089:     PetscCall(PetscInfo(viewer, "Shapefile: Z bounds (%g) -- (%g)\n", zb[0], zb[1]));
    7090:     PetscCall(PetscViewerBinaryRead(viewer, mb, 2, &cnt, PETSC_DOUBLE));
    7091:     PetscCall(PetscByteSwap(mb, PETSC_DOUBLE, 2));
    7092:     PetscCheck(cnt == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read bytes 84-99");
    7093:     PetscCall(PetscInfo(viewer, "Shapefile: M bounds (%g) -- (%g)\n", mb[0], mb[1]));
    7094:     totlen += 100;
    7095:     {
    7096:       int    rnum, rlen, rshtype, rnpart, rnp;
    7097:       double rmbr[4];
    7098:       int   *partOffsets;

    7100:       // Read record header
    7101:       PetscCall(PetscViewerBinaryRead(viewer, &rnum, 1, &cnt, PETSC_INT32));
    7102:       PetscCheck(cnt == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read record header bytes 0-3");
    7103:       PetscCall(PetscViewerBinaryRead(viewer, &rlen, 1, &cnt, PETSC_INT32));
    7104:       PetscCheck(cnt == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read record header bytes 4-7");
    7105:       PetscCall(PetscInfo(viewer, "Shapefile: record %d length %d\n", rnum, rlen));
    7106:       totlen += 8;
    7107:       // Read record
    7108:       PetscCall(PetscViewerBinaryRead(viewer, &rshtype, 1, &cnt, PETSC_INT32));
    7109:       PetscCall(PetscByteSwap(&rshtype, PETSC_INT32, 1));
    7110:       PetscCheck(cnt == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read record bytes 0-3");
    7111:       PetscCall(PetscInfo(viewer, "Shapefile: record shape type %d\n", rshtype));
    7112:       totlen += 4;
    7113:       switch (rshtype) {
    7114:       case 5: // Polygon
    7115:         PetscCall(PetscViewerBinaryRead(viewer, rmbr, 4, &cnt, PETSC_DOUBLE));
    7116:         PetscCall(PetscByteSwap(rmbr, PETSC_DOUBLE, 4));
    7117:         PetscCheck(cnt == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read record bytes 4-35");
    7118:         PetscCall(PetscInfo(viewer, "Shapefile: record minimum bounding rectangle (%g, %g) -- (%g, %g)\n", rmbr[0], rmbr[1], rmbr[2], rmbr[3]));
    7119:         PetscCall(PetscViewerBinaryRead(viewer, &rnpart, 1, &cnt, PETSC_INT32));
    7120:         PetscCall(PetscByteSwap(&rnpart, PETSC_INT32, 1));
    7121:         PetscCheck(cnt == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read record bytes 36-39");
    7122:         PetscCall(PetscInfo(viewer, "Shapefile: record shape number of parts %d\n", rnpart));
    7123:         PetscCall(PetscViewerBinaryRead(viewer, &rnp, 1, &cnt, PETSC_INT32));
    7124:         PetscCall(PetscByteSwap(&rnp, PETSC_INT32, 1));
    7125:         PetscCheck(cnt == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read record bytes 36-39");
    7126:         PetscCall(PetscInfo(viewer, "Shapefile: record shape number of points %d\n", rnp));
    7127:         totlen += 40;
    7128:         PetscCall(PetscMalloc1(rnpart, &partOffsets));
    7129:         PetscCall(PetscViewerBinaryRead(viewer, partOffsets, rnpart, &cnt, PETSC_INT32));
    7130:         PetscCall(PetscByteSwap(partOffsets, PETSC_INT32, rnpart));
    7131:         PetscCheck(cnt == rnpart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read record part offsets");
    7132:         totlen += 4 * rnpart;
    7133:         PetscCall(PetscMalloc1(rnp * 2, &points));
    7134:         PetscCall(PetscViewerBinaryRead(viewer, points, rnp * 2, &cnt, PETSC_DOUBLE));
    7135:         PetscCall(PetscByteSwap(points, PETSC_DOUBLE, rnp * 2));
    7136:         PetscCheck(cnt == rnp * 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Shapefile file: could not read record points");
    7137:         totlen += 8 * rnp * 2;
    7138:         PetscCheck(totlen == 2 * len, PETSC_COMM_SELF, PETSC_ERR_SUP, "Unable to parse Shapefile file: only support a single object: %d != %d", totlen, 2 * len);
    7139:         // Only get the last polygon now
    7140:         vOff = partOffsets[rnpart - 1];
    7141:         Nv   = rnp - vOff;
    7142:         Nc   = Nv - 1;
    7143:         PetscCall(PetscInfo(viewer, "Shapefile: record first polygon size %d totlen %d\n", Nv, totlen));
    7144:         PetscCall(PetscFree(partOffsets));
    7145:         break;
    7146:       default:
    7147:         PetscCheck(0, PETSC_COMM_SELF, PETSC_ERR_SUP, "Only support polygons right now");
    7148:       }
    7149:     }
    7150:   } else {
    7151:     Nc = Nv = vOff = 0;
    7152:   }
    7153:   PetscCallMPI(MPI_Bcast(&dim, 1, MPI_INT, 0, comm));
    7154:   PetscCall(DMCreate(comm, dm));
    7155:   PetscCall(DMSetType(*dm, DMPLEX));
    7156:   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
    7157:   PetscCall(DMSetDimension(*dm, dim));
    7158:   PetscCall(DMSetCoordinateDim(*dm, cdim));
    7159:   // Topology of a circle
    7160:   if (rank == 0) {
    7161:     for (PetscInt c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, 2));
    7162:     PetscCall(DMSetUp(*dm));
    7163:     for (PetscInt c = 0; c < Nc; ++c) {
    7164:       PetscInt cone[2] = {Nc + c, Nc + c + 1};

    7166:       PetscCall(DMPlexSetCone(*dm, c, cone));
    7167:     }
    7168:   }
    7169:   PetscCall(DMPlexSymmetrize(*dm));
    7170:   PetscCall(DMPlexStratify(*dm));
    7171:   // Set coordinates
    7172:   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
    7173:   PetscCall(PetscSectionSetNumFields(coordSection, 1));
    7174:   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
    7175:   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
    7176:   for (PetscInt v = Nc; v < Nc + Nv; ++v) {
    7177:     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
    7178:     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
    7179:   }
    7180:   PetscCall(PetscSectionSetUp(coordSection));
    7181:   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
    7182:   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
    7183:   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
    7184:   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
    7185:   PetscCall(VecSetBlockSize(coordinates, cdim));
    7186:   PetscCall(VecSetType(coordinates, VECSTANDARD));
    7187:   PetscCall(VecGetArray(coordinates, &coords));
    7188:   for (PetscInt v = 0; v < Nv; ++v) {
    7189:     coords[v * cdim + 0] = points[(v + vOff) * cdim + 0] - mbr[0];
    7190:     coords[v * cdim + 1] = points[(v + vOff) * cdim + 1] - mbr[1];
    7191:   }
    7192:   PetscCall(PetscFree(points));
    7193:   PetscCall(VecRestoreArray(coordinates, &coords));
    7194:   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
    7195:   PetscCall(VecDestroy(&coordinates));
    7196:   PetscFunctionReturn(PETSC_SUCCESS);
    7197: }

    7199: /*
    7200:   DMPlexCreateShapefileFromFile - Create a `DMPLEX` mesh from the ESRI Shapefile format.

    7202:   Collective

    7204: + comm     - The MPI communicator
    7205: . filename - Name of the .shp file

    7207:   Output Parameter:
    7208: . dm - The `DM` object representing the mesh

    7210:   Level: beginner

    7212:   Note:
    7213:   The format is specified at [Wikipedia](https://en.wikipedia.org/wiki/Shapefile) and [ESRI](https://www.esri.com/content/dam/esrisites/sitecore-archive/Files/Pdfs/library/whitepapers/pdfs/shapefile.pdf).

    7215: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
    7216: */
    7217: static PetscErrorCode DMPlexCreateShapefileFromFile(MPI_Comm comm, const char filename[], DM *dm)
    7218: {
    7219:   PetscViewer viewer;

    7221:   PetscFunctionBegin;
    7222:   PetscCall(PetscViewerCreate(comm, &viewer));
    7223:   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERBINARY));
    7224:   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
    7225:   PetscCall(PetscViewerFileSetName(viewer, filename));
    7226:   PetscCall(DMPlexCreateShapefile(comm, viewer, dm));
    7227:   PetscCall(PetscViewerDestroy(&viewer));
    7228:   PetscFunctionReturn(PETSC_SUCCESS);
    7229: }

    7231: /*@
    7232:   DMPlexCreateFromFile - This takes a filename and produces a `DM`

    7234:   Collective

    7236:   Input Parameters:
    7237: + comm        - The communicator
    7238: . filename    - A file name
    7239: . plexname    - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
    7240: - interpolate - Flag to create intermediate mesh pieces (edges, faces)

    7242:   Output Parameter:
    7243: . dm - The `DM`

    7245:   Options Database Key:
    7246: . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5

    7248:   Use `-dm_plex_create_ prefix` to pass options to the internal `PetscViewer`, e.g. `-dm_plex_create_viewer_hdf5_collective`

    7250:   Level: beginner

    7252:   Notes:
    7253:   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
    7254:   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
    7255:   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
    7256:   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
    7257:   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.

    7259: .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
    7260: @*/
    7261: PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
    7262: {
    7263:   const char  extGmsh[]      = ".msh";
    7264:   const char  extGmsh2[]     = ".msh2";
    7265:   const char  extGmsh4[]     = ".msh4";
    7266:   const char  extCGNS[]      = ".cgns";
    7267:   const char  extExodus[]    = ".exo";
    7268:   const char  extExodus_e[]  = ".e";
    7269:   const char  extGenesis[]   = ".gen";
    7270:   const char  extFluent[]    = ".cas";
    7271:   const char  extHDF5[]      = ".h5";
    7272:   const char  extXDMFHDF5[]  = ".xdmf.h5";
    7273:   const char  extPLY[]       = ".ply";
    7274:   const char  extEGADSlite[] = ".egadslite";
    7275:   const char  extEGADS[]     = ".egads";
    7276:   const char  extIGES[]      = ".igs";
    7277:   const char  extIGES2[]     = ".iges";
    7278:   const char  extSTEP[]      = ".stp";
    7279:   const char  extSTEP2[]     = ".step";
    7280:   const char  extBREP[]      = ".brep";
    7281:   const char  extCV[]        = ".dat";
    7282:   const char  extSTL[]       = ".stl";
    7283:   const char  extSHP[]       = ".shp";
    7284:   size_t      len;
    7285:   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isPLY, isEGADSlite, isEGADS, isIGES, isIGES2, isSTEP, isSTEP2, isBREP, isCV, isSTL, isSHP, isXDMFHDF5;
    7286:   PetscMPIInt rank;

    7288:   PetscFunctionBegin;
    7289:   PetscAssertPointer(filename, 2);
    7290:   if (plexname) PetscAssertPointer(plexname, 3);
    7291:   PetscAssertPointer(dm, 5);
    7292:   PetscCall(DMInitializePackage());
    7293:   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
    7294:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
    7295:   PetscCall(PetscStrlen(filename, &len));
    7296:   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");

    7298: #define CheckExtension(extension__, is_extension__) \
    7299:   do { \
    7300:     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
    7301:     /* don't count the null-terminator at the end */ \
    7302:     const size_t ext_len = sizeof(extension__) - 1; \
    7303:     if (len < ext_len) { \
    7304:       is_extension__ = PETSC_FALSE; \
    7305:     } else { \
    7306:       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
    7307:     } \
    7308:   } while (0)

    7310:   CheckExtension(extGmsh, isGmsh);
    7311:   CheckExtension(extGmsh2, isGmsh2);
    7312:   CheckExtension(extGmsh4, isGmsh4);
    7313:   CheckExtension(extCGNS, isCGNS);
    7314:   CheckExtension(extExodus, isExodus);
    7315:   if (!isExodus) CheckExtension(extExodus_e, isExodus);
    7316:   CheckExtension(extGenesis, isGenesis);
    7317:   CheckExtension(extFluent, isFluent);
    7318:   CheckExtension(extHDF5, isHDF5);
    7319:   CheckExtension(extPLY, isPLY);
    7320:   CheckExtension(extEGADSlite, isEGADSlite);
    7321:   CheckExtension(extEGADS, isEGADS);
    7322:   CheckExtension(extIGES, isIGES);
    7323:   CheckExtension(extIGES2, isIGES2);
    7324:   CheckExtension(extSTEP, isSTEP);
    7325:   CheckExtension(extSTEP2, isSTEP2);
    7326:   CheckExtension(extBREP, isBREP);
    7327:   CheckExtension(extCV, isCV);
    7328:   CheckExtension(extSTL, isSTL);
    7329:   CheckExtension(extSHP, isSHP);
    7330:   CheckExtension(extXDMFHDF5, isXDMFHDF5);

    7332: #undef CheckExtension

    7334:   if (isGmsh || isGmsh2 || isGmsh4) {
    7335:     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
    7336:   } else if (isCGNS) {
    7337:     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
    7338:   } else if (isExodus || isGenesis) {
    7339:     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
    7340:   } else if (isFluent) {
    7341:     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
    7342:   } else if (isHDF5) {
    7343:     PetscViewer viewer;

    7345:     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
    7346:     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &isXDMFHDF5, NULL));
    7347:     PetscCall(PetscViewerCreate(comm, &viewer));
    7348:     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
    7349:     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
    7350:     PetscCall(PetscViewerSetFromOptions(viewer));
    7351:     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
    7352:     PetscCall(PetscViewerFileSetName(viewer, filename));

    7354:     PetscCall(DMCreate(comm, dm));
    7355:     PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
    7356:     PetscCall(DMSetType(*dm, DMPLEX));
    7357:     if (isXDMFHDF5) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
    7358:     PetscCall(DMLoad(*dm, viewer));
    7359:     if (isXDMFHDF5) PetscCall(PetscViewerPopFormat(viewer));
    7360:     PetscCall(PetscViewerDestroy(&viewer));

    7362:     if (interpolate) {
    7363:       DM idm;

    7365:       PetscCall(DMPlexInterpolate(*dm, &idm));
    7366:       PetscCall(DMDestroy(dm));
    7367:       *dm = idm;
    7368:     }
    7369:   } else if (isPLY) {
    7370:     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
    7371:   } else if (isEGADSlite || isEGADS || isIGES || isIGES2 || isSTEP || isSTEP2 || isBREP) {
    7372:     PetscCall(DMPlexCreateGeomFromFile(comm, filename, dm, isEGADSlite));

    7374:     if (!interpolate) {
    7375:       DM udm;

    7377:       PetscCall(DMPlexUninterpolate(*dm, &udm));
    7378:       PetscCall(DMDestroy(dm));
    7379:       *dm = udm;
    7380:     }
    7381:   } else if (isCV) {
    7382:     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
    7383:   } else if (isSTL) {
    7384:     PetscCall(DMPlexCreateSTLFromFile(comm, filename, interpolate, dm));
    7385:   } else if (isSHP) {
    7386:     PetscCall(DMPlexCreateShapefileFromFile(comm, filename, dm));
    7387:   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
    7388:   PetscCall(PetscStrlen(plexname, &len));
    7389:   if (len) PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
    7390:   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
    7391:   PetscFunctionReturn(PETSC_SUCCESS);
    7392: }

    7394: /*@
    7395:   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.

    7397:   Input Parameters:
    7398: + tr     - The `DMPlexTransform`
    7399: - prefix - An options prefix, or NULL

    7401:   Output Parameter:
    7402: . dm - The `DM`

    7404:   Level: beginner

    7406:   Notes:
    7407:   An emphemeral mesh is one that is not stored concretely, as in the default `DMPLEX` implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.

    7409: .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
    7410: @*/
    7411: PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, const char prefix[], DM *dm)
    7412: {
    7413:   DM           bdm, bcdm, cdm;
    7414:   Vec          coordinates, coordinatesNew;
    7415:   PetscSection cs;
    7416:   PetscInt     cdim, Nl;

    7418:   PetscFunctionBegin;
    7419:   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
    7420:   PetscCall(DMSetType(*dm, DMPLEX));
    7421:   ((DM_Plex *)(*dm)->data)->interpolated = DMPLEX_INTERPOLATED_FULL;
    7422:   // Handle coordinates
    7423:   PetscCall(DMPlexTransformGetDM(tr, &bdm));
    7424:   PetscCall(DMPlexTransformSetDimensions(tr, bdm, *dm));
    7425:   PetscCall(DMGetCoordinateDim(*dm, &cdim));
    7426:   PetscCall(DMGetCoordinateDM(bdm, &bcdm));
    7427:   PetscCall(DMGetCoordinateDM(*dm, &cdm));
    7428:   PetscCall(DMCopyDisc(bcdm, cdm));
    7429:   PetscCall(DMGetLocalSection(cdm, &cs));
    7430:   PetscCall(PetscSectionSetNumFields(cs, 1));
    7431:   PetscCall(PetscSectionSetFieldComponents(cs, 0, cdim));
    7432:   PetscCall(DMGetCoordinatesLocal(bdm, &coordinates));
    7433:   PetscCall(VecDuplicate(coordinates, &coordinatesNew));
    7434:   PetscCall(VecCopy(coordinates, coordinatesNew));
    7435:   PetscCall(DMSetCoordinatesLocal(*dm, coordinatesNew));
    7436:   PetscCall(VecDestroy(&coordinatesNew));

    7438:   PetscCall(PetscObjectReference((PetscObject)tr));
    7439:   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
    7440:   ((DM_Plex *)(*dm)->data)->tr = tr;
    7441:   PetscCall(DMPlexDistributeSetDefault(*dm, PETSC_FALSE));
    7442:   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)*dm, prefix));
    7443:   PetscCall(DMSetFromOptions(*dm));

    7445:   PetscCall(DMGetNumLabels(bdm, &Nl));
    7446:   for (PetscInt l = 0; l < Nl; ++l) {
    7447:     DMLabel     label, labelNew;
    7448:     const char *lname;
    7449:     PetscBool   isDepth, isCellType;

    7451:     PetscCall(DMGetLabelName(bdm, l, &lname));
    7452:     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
    7453:     if (isDepth) continue;
    7454:     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
    7455:     if (isCellType) continue;
    7456:     PetscCall(DMCreateLabel(*dm, lname));
    7457:     PetscCall(DMGetLabel(bdm, lname, &label));
    7458:     PetscCall(DMGetLabel(*dm, lname, &labelNew));
    7459:     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
    7460:     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
    7461:     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
    7462:     PetscCall(DMLabelSetUp(labelNew));
    7463:   }
    7464:   PetscFunctionReturn(PETSC_SUCCESS);
    7465: }