Actual source code: mg.c

      2: /*
      3:     Defines the multigrid preconditioner interface.
      4: */
      5: #include <petsc/private/pcmgimpl.h>
      6: #include <petsc/private/kspimpl.h>
      7: #include <petscdm.h>
      8: PETSC_INTERN PetscErrorCode PCPreSolveChangeRHS(PC,PetscBool*);

     10: /*
     11:    Contains the list of registered coarse space construction routines
     12: */
     13: PetscFunctionList PCMGCoarseList = NULL;

     15: PetscErrorCode PCMGMCycle_Private(PC pc,PC_MG_Levels **mglevelsin,PetscBool transpose,PetscBool matapp,PCRichardsonConvergedReason *reason)
     16: {
     17:   PC_MG          *mg = (PC_MG*)pc->data;
     18:   PC_MG_Levels   *mgc,*mglevels = *mglevelsin;
     20:   PetscInt       cycles = (mglevels->level == 1) ? 1 : (PetscInt) mglevels->cycles;

     23:   if (mglevels->eventsmoothsolve) {PetscLogEventBegin(mglevels->eventsmoothsolve,0,0,0,0);}
     24:   if (!transpose) {
     25:     if (matapp) {
     26:       KSPMatSolve(mglevels->smoothd,mglevels->B,mglevels->X);  /* pre-smooth */
     27:       KSPCheckSolve(mglevels->smoothd,pc,NULL);
     28:     } else {
     29:       KSPSolve(mglevels->smoothd,mglevels->b,mglevels->x);  /* pre-smooth */
     30:       KSPCheckSolve(mglevels->smoothd,pc,mglevels->x);
     31:     }
     32:   } else {
     33:     if (matapp) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_SUP,"Not supported");
     34:     KSPSolveTranspose(mglevels->smoothu,mglevels->b,mglevels->x); /* transpose of post-smooth */
     35:     KSPCheckSolve(mglevels->smoothu,pc,mglevels->x);
     36:   }
     37:   if (mglevels->eventsmoothsolve) {PetscLogEventEnd(mglevels->eventsmoothsolve,0,0,0,0);}
     38:   if (mglevels->level) {  /* not the coarsest grid */
     39:     if (mglevels->eventresidual) {PetscLogEventBegin(mglevels->eventresidual,0,0,0,0);}
     40:     if (matapp && !mglevels->R) {
     41:       MatDuplicate(mglevels->B,MAT_DO_NOT_COPY_VALUES,&mglevels->R);
     42:     }
     43:     if (!transpose) {
     44:       if (matapp) { (*mglevels->matresidual)(mglevels->A,mglevels->B,mglevels->X,mglevels->R); }
     45:       else { (*mglevels->residual)(mglevels->A,mglevels->b,mglevels->x,mglevels->r); }
     46:     } else {
     47:       if (matapp) { (*mglevels->matresidualtranspose)(mglevels->A,mglevels->B,mglevels->X,mglevels->R); }
     48:       else { (*mglevels->residualtranspose)(mglevels->A,mglevels->b,mglevels->x,mglevels->r); }
     49:     }
     50:     if (mglevels->eventresidual) {PetscLogEventEnd(mglevels->eventresidual,0,0,0,0);}

     52:     /* if on finest level and have convergence criteria set */
     53:     if (mglevels->level == mglevels->levels-1 && mg->ttol && reason) {
     54:       PetscReal rnorm;
     55:       VecNorm(mglevels->r,NORM_2,&rnorm);
     56:       if (rnorm <= mg->ttol) {
     57:         if (rnorm < mg->abstol) {
     58:           *reason = PCRICHARDSON_CONVERGED_ATOL;
     59:           PetscInfo2(pc,"Linear solver has converged. Residual norm %g is less than absolute tolerance %g\n",(double)rnorm,(double)mg->abstol);
     60:         } else {
     61:           *reason = PCRICHARDSON_CONVERGED_RTOL;
     62:           PetscInfo2(pc,"Linear solver has converged. Residual norm %g is less than relative tolerance times initial residual norm %g\n",(double)rnorm,(double)mg->ttol);
     63:         }
     64:         return(0);
     65:       }
     66:     }

     68:     mgc = *(mglevelsin - 1);
     69:     if (mglevels->eventinterprestrict) {PetscLogEventBegin(mglevels->eventinterprestrict,0,0,0,0);}
     70:     if (!transpose) {
     71:       if (matapp) { MatMatRestrict(mglevels->restrct,mglevels->R,&mgc->B); }
     72:       else { MatRestrict(mglevels->restrct,mglevels->r,mgc->b); }
     73:     } else {
     74:       if (matapp) { MatMatRestrict(mglevels->interpolate,mglevels->R,&mgc->B); }
     75:       else { MatRestrict(mglevels->interpolate,mglevels->r,mgc->b); }
     76:     }
     77:     if (mglevels->eventinterprestrict) {PetscLogEventEnd(mglevels->eventinterprestrict,0,0,0,0);}
     78:     if (matapp) {
     79:       if (!mgc->X) {
     80:         MatDuplicate(mgc->B,MAT_DO_NOT_COPY_VALUES,&mgc->X);
     81:       } else {
     82:         MatZeroEntries(mgc->X);
     83:       }
     84:     } else {
     85:       VecZeroEntries(mgc->x);
     86:     }
     87:     while (cycles--) {
     88:       PCMGMCycle_Private(pc,mglevelsin-1,transpose,matapp,reason);
     89:     }
     90:     if (mglevels->eventinterprestrict) {PetscLogEventBegin(mglevels->eventinterprestrict,0,0,0,0);}
     91:     if (!transpose) {
     92:       if (matapp) { MatMatInterpolateAdd(mglevels->interpolate,mgc->X,mglevels->X,&mglevels->X); }
     93:       else { MatInterpolateAdd(mglevels->interpolate,mgc->x,mglevels->x,mglevels->x); }
     94:     } else {
     95:       MatInterpolateAdd(mglevels->restrct,mgc->x,mglevels->x,mglevels->x);
     96:     }
     97:     if (mglevels->eventinterprestrict) {PetscLogEventEnd(mglevels->eventinterprestrict,0,0,0,0);}
     98:     if (mglevels->eventsmoothsolve) {PetscLogEventBegin(mglevels->eventsmoothsolve,0,0,0,0);}
     99:     if (!transpose) {
    100:       if (matapp) {
    101:         KSPMatSolve(mglevels->smoothu,mglevels->B,mglevels->X);    /* post smooth */
    102:         KSPCheckSolve(mglevels->smoothu,pc,NULL);
    103:       } else {
    104:         KSPSolve(mglevels->smoothu,mglevels->b,mglevels->x);    /* post smooth */
    105:         KSPCheckSolve(mglevels->smoothu,pc,mglevels->x);
    106:       }
    107:     } else {
    108:       if (matapp) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_SUP,"Not supported");
    109:       KSPSolveTranspose(mglevels->smoothd,mglevels->b,mglevels->x);    /* post smooth */
    110:       KSPCheckSolve(mglevels->smoothd,pc,mglevels->x);
    111:     }
    112:     if (mglevels->cr) {
    113:       if (matapp) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_SUP,"Not supported");
    114:       /* TODO Turn on copy and turn off noisy if we have an exact solution
    115:       VecCopy(mglevels->x, mglevels->crx);
    116:       VecCopy(mglevels->b, mglevels->crb); */
    117:       KSPSetNoisy_Private(mglevels->crx);
    118:       KSPSolve(mglevels->cr,mglevels->crb,mglevels->crx);    /* compatible relaxation */
    119:       KSPCheckSolve(mglevels->cr,pc,mglevels->crx);
    120:     }
    121:     if (mglevels->eventsmoothsolve) {PetscLogEventEnd(mglevels->eventsmoothsolve,0,0,0,0);}
    122:   }
    123:   return(0);
    124: }

    126: static PetscErrorCode PCApplyRichardson_MG(PC pc,Vec b,Vec x,Vec w,PetscReal rtol,PetscReal abstol, PetscReal dtol,PetscInt its,PetscBool zeroguess,PetscInt *outits,PCRichardsonConvergedReason *reason)
    127: {
    128:   PC_MG          *mg        = (PC_MG*)pc->data;
    129:   PC_MG_Levels   **mglevels = mg->levels;
    131:   PC             tpc;
    132:   PetscBool      changeu,changed;
    133:   PetscInt       levels = mglevels[0]->levels,i;

    136:   /* When the DM is supplying the matrix then it will not exist until here */
    137:   for (i=0; i<levels; i++) {
    138:     if (!mglevels[i]->A) {
    139:       KSPGetOperators(mglevels[i]->smoothu,&mglevels[i]->A,NULL);
    140:       PetscObjectReference((PetscObject)mglevels[i]->A);
    141:     }
    142:   }

    144:   KSPGetPC(mglevels[levels-1]->smoothd,&tpc);
    145:   PCPreSolveChangeRHS(tpc,&changed);
    146:   KSPGetPC(mglevels[levels-1]->smoothu,&tpc);
    147:   PCPreSolveChangeRHS(tpc,&changeu);
    148:   if (!changed && !changeu) {
    149:     VecDestroy(&mglevels[levels-1]->b);
    150:     mglevels[levels-1]->b = b;
    151:   } else { /* if the smoother changes the rhs during PreSolve, we cannot use the input vector */
    152:     if (!mglevels[levels-1]->b) {
    153:       Vec *vec;

    155:       KSPCreateVecs(mglevels[levels-1]->smoothd,1,&vec,0,NULL);
    156:       mglevels[levels-1]->b = *vec;
    157:       PetscFree(vec);
    158:     }
    159:     VecCopy(b,mglevels[levels-1]->b);
    160:   }
    161:   mglevels[levels-1]->x = x;

    163:   mg->rtol   = rtol;
    164:   mg->abstol = abstol;
    165:   mg->dtol   = dtol;
    166:   if (rtol) {
    167:     /* compute initial residual norm for relative convergence test */
    168:     PetscReal rnorm;
    169:     if (zeroguess) {
    170:       VecNorm(b,NORM_2,&rnorm);
    171:     } else {
    172:       (*mglevels[levels-1]->residual)(mglevels[levels-1]->A,b,x,w);
    173:       VecNorm(w,NORM_2,&rnorm);
    174:     }
    175:     mg->ttol = PetscMax(rtol*rnorm,abstol);
    176:   } else if (abstol) mg->ttol = abstol;
    177:   else mg->ttol = 0.0;

    179:   /* since smoother is applied to full system, not just residual we need to make sure that smoothers don't
    180:      stop prematurely due to small residual */
    181:   for (i=1; i<levels; i++) {
    182:     KSPSetTolerances(mglevels[i]->smoothu,0,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT);
    183:     if (mglevels[i]->smoothu != mglevels[i]->smoothd) {
    184:       /* For Richardson the initial guess is nonzero since it is solving in each cycle the original system not just applying as a preconditioner */
    185:       KSPSetInitialGuessNonzero(mglevels[i]->smoothd,PETSC_TRUE);
    186:       KSPSetTolerances(mglevels[i]->smoothd,0,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT);
    187:     }
    188:   }

    190:   *reason = (PCRichardsonConvergedReason)0;
    191:   for (i=0; i<its; i++) {
    192:     PCMGMCycle_Private(pc,mglevels+levels-1,PETSC_FALSE,PETSC_FALSE,reason);
    193:     if (*reason) break;
    194:   }
    195:   if (!*reason) *reason = PCRICHARDSON_CONVERGED_ITS;
    196:   *outits = i;
    197:   if (!changed && !changeu) mglevels[levels-1]->b = NULL;
    198:   return(0);
    199: }

    201: PetscErrorCode PCReset_MG(PC pc)
    202: {
    203:   PC_MG          *mg        = (PC_MG*)pc->data;
    204:   PC_MG_Levels   **mglevels = mg->levels;
    206:   PetscInt       i,c,n;

    209:   if (mglevels) {
    210:     n = mglevels[0]->levels;
    211:     for (i=0; i<n-1; i++) {
    212:       VecDestroy(&mglevels[i+1]->r);
    213:       VecDestroy(&mglevels[i]->b);
    214:       VecDestroy(&mglevels[i]->x);
    215:       MatDestroy(&mglevels[i+1]->R);
    216:       MatDestroy(&mglevels[i]->B);
    217:       MatDestroy(&mglevels[i]->X);
    218:       VecDestroy(&mglevels[i]->crx);
    219:       VecDestroy(&mglevels[i]->crb);
    220:       MatDestroy(&mglevels[i+1]->restrct);
    221:       MatDestroy(&mglevels[i+1]->interpolate);
    222:       MatDestroy(&mglevels[i+1]->inject);
    223:       VecDestroy(&mglevels[i+1]->rscale);
    224:     }
    225:     VecDestroy(&mglevels[n-1]->crx);
    226:     VecDestroy(&mglevels[n-1]->crb);
    227:     /* this is not null only if the smoother on the finest level
    228:        changes the rhs during PreSolve */
    229:     VecDestroy(&mglevels[n-1]->b);
    230:     MatDestroy(&mglevels[n-1]->B);

    232:     for (i=0; i<n; i++) {
    233:       if (mglevels[i]->coarseSpace) for (c = 0; c < mg->Nc; ++c) {VecDestroy(&mglevels[i]->coarseSpace[c]);}
    234:       PetscFree(mglevels[i]->coarseSpace);
    235:       mglevels[i]->coarseSpace = NULL;
    236:       MatDestroy(&mglevels[i]->A);
    237:       if (mglevels[i]->smoothd != mglevels[i]->smoothu) {
    238:         KSPReset(mglevels[i]->smoothd);
    239:       }
    240:       KSPReset(mglevels[i]->smoothu);
    241:       if (mglevels[i]->cr) {KSPReset(mglevels[i]->cr);}
    242:     }
    243:     mg->Nc = 0;
    244:   }
    245:   return(0);
    246: }

    248: /* Implementing CR

    250: We only want to make corrections that ``do not change'' the coarse solution. What we mean by not changing is that if I prolong my coarse solution to the fine grid and then inject that fine solution back to the coarse grid, I get the same answer. Injection is what Brannick calls R. We want the complementary projector to Inj, which we will call S, after Brannick, so that Inj S = 0. Now the orthogonal projector onto the range of Inj^T is

    252:   Inj^T (Inj Inj^T)^{-1} Inj

    254: and if Inj is a VecScatter, as it is now in PETSc, we have

    256:   Inj^T Inj

    258: and

    260:   S = I - Inj^T Inj

    262: since

    264:   Inj S = Inj - (Inj Inj^T) Inj = 0.

    266: Brannick suggests

    268:   A \to S^T A S  \qquad\mathrm{and}\qquad M \to S^T M S

    270: but I do not think his :math:`S^T S = I` is correct. Our S is an orthogonal projector, so :math:`S^T S = S^2 = S`. We will use

    272:   M^{-1} A \to S M^{-1} A S

    274: In fact, since it is somewhat hard in PETSc to do the symmetric application, we will just apply S on the left.

    276:   Check: || Inj P - I ||_F < tol
    277:   Check: In general, Inj Inj^T = I
    278: */

    280: typedef struct {
    281:   PC       mg;  /* The PCMG object */
    282:   PetscInt l;   /* The multigrid level for this solver */
    283:   Mat      Inj; /* The injection matrix */
    284:   Mat      S;   /* I - Inj^T Inj */
    285: } CRContext;

    287: static PetscErrorCode CRSetup_Private(PC pc)
    288: {
    289:   CRContext     *ctx;
    290:   Mat            It;

    294:   PCShellGetContext(pc, (void **) &ctx);
    295:   PCMGGetInjection(ctx->mg, ctx->l, &It);
    296:   if (!It) SETERRQ(PetscObjectComm((PetscObject) pc), PETSC_ERR_ARG_WRONGSTATE, "CR requires that injection be defined for this PCMG");
    297:   MatCreateTranspose(It, &ctx->Inj);
    298:   MatCreateNormal(ctx->Inj, &ctx->S);
    299:   MatScale(ctx->S, -1.0);
    300:   MatShift(ctx->S,  1.0);
    301:   return(0);
    302: }

    304: static PetscErrorCode CRApply_Private(PC pc, Vec x, Vec y)
    305: {
    306:   CRContext     *ctx;

    310:   PCShellGetContext(pc, (void **) &ctx);
    311:   MatMult(ctx->S, x, y);
    312:   return(0);
    313: }

    315: static PetscErrorCode CRDestroy_Private(PC pc)
    316: {
    317:   CRContext     *ctx;

    321:   PCShellGetContext(pc, (void **) &ctx);
    322:   MatDestroy(&ctx->Inj);
    323:   MatDestroy(&ctx->S);
    324:   PetscFree(ctx);
    325:   PCShellSetContext(pc, NULL);
    326:   return(0);
    327: }

    329: static PetscErrorCode CreateCR_Private(PC pc, PetscInt l, PC *cr)
    330: {
    331:   CRContext     *ctx;

    335:   PCCreate(PetscObjectComm((PetscObject) pc), cr);
    336:   PetscObjectSetName((PetscObject) *cr, "S (complementary projector to injection)");
    337:   PetscCalloc1(1, &ctx);
    338:   ctx->mg = pc;
    339:   ctx->l  = l;
    340:   PCSetType(*cr, PCSHELL);
    341:   PCShellSetContext(*cr, ctx);
    342:   PCShellSetApply(*cr, CRApply_Private);
    343:   PCShellSetSetUp(*cr, CRSetup_Private);
    344:   PCShellSetDestroy(*cr, CRDestroy_Private);
    345:   return(0);
    346: }

    348: PetscErrorCode PCMGSetLevels_MG(PC pc,PetscInt levels,MPI_Comm *comms)
    349: {
    351:   PC_MG          *mg        = (PC_MG*)pc->data;
    352:   MPI_Comm       comm;
    353:   PC_MG_Levels   **mglevels = mg->levels;
    354:   PCMGType       mgtype     = mg->am;
    355:   PetscInt       mgctype    = (PetscInt) PC_MG_CYCLE_V;
    356:   PetscInt       i;
    357:   PetscMPIInt    size;
    358:   const char     *prefix;
    359:   PC             ipc;
    360:   PetscInt       n;

    365:   if (mg->nlevels == levels) return(0);
    366:   PetscObjectGetComm((PetscObject)pc,&comm);
    367:   if (mglevels) {
    368:     mgctype = mglevels[0]->cycles;
    369:     /* changing the number of levels so free up the previous stuff */
    370:     PCReset_MG(pc);
    371:     n    = mglevels[0]->levels;
    372:     for (i=0; i<n; i++) {
    373:       if (mglevels[i]->smoothd != mglevels[i]->smoothu) {
    374:         KSPDestroy(&mglevels[i]->smoothd);
    375:       }
    376:       KSPDestroy(&mglevels[i]->smoothu);
    377:       KSPDestroy(&mglevels[i]->cr);
    378:       PetscFree(mglevels[i]);
    379:     }
    380:     PetscFree(mg->levels);
    381:   }

    383:   mg->nlevels = levels;

    385:   PetscMalloc1(levels,&mglevels);
    386:   PetscLogObjectMemory((PetscObject)pc,levels*(sizeof(PC_MG*)));

    388:   PCGetOptionsPrefix(pc,&prefix);

    390:   mg->stageApply = 0;
    391:   for (i=0; i<levels; i++) {
    392:     PetscNewLog(pc,&mglevels[i]);

    394:     mglevels[i]->level               = i;
    395:     mglevels[i]->levels              = levels;
    396:     mglevels[i]->cycles              = mgctype;
    397:     mg->default_smoothu              = 2;
    398:     mg->default_smoothd              = 2;
    399:     mglevels[i]->eventsmoothsetup    = 0;
    400:     mglevels[i]->eventsmoothsolve    = 0;
    401:     mglevels[i]->eventresidual       = 0;
    402:     mglevels[i]->eventinterprestrict = 0;

    404:     if (comms) comm = comms[i];
    405:     if (comm != MPI_COMM_NULL) {
    406:       KSPCreate(comm,&mglevels[i]->smoothd);
    407:       KSPSetErrorIfNotConverged(mglevels[i]->smoothd,pc->erroriffailure);
    408:       PetscObjectIncrementTabLevel((PetscObject)mglevels[i]->smoothd,(PetscObject)pc,levels-i);
    409:       KSPSetOptionsPrefix(mglevels[i]->smoothd,prefix);
    410:       PetscObjectComposedDataSetInt((PetscObject) mglevels[i]->smoothd, PetscMGLevelId, mglevels[i]->level);
    411:       if (i || levels == 1) {
    412:         char tprefix[128];

    414:         KSPSetType(mglevels[i]->smoothd,KSPCHEBYSHEV);
    415:         KSPSetConvergenceTest(mglevels[i]->smoothd,KSPConvergedSkip,NULL,NULL);
    416:         KSPSetNormType(mglevels[i]->smoothd,KSP_NORM_NONE);
    417:         KSPGetPC(mglevels[i]->smoothd,&ipc);
    418:         PCSetType(ipc,PCSOR);
    419:         KSPSetTolerances(mglevels[i]->smoothd,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT, mg->default_smoothd);

    421:         PetscSNPrintf(tprefix,128,"mg_levels_%d_",(int)i);
    422:         KSPAppendOptionsPrefix(mglevels[i]->smoothd,tprefix);
    423:       } else {
    424:         KSPAppendOptionsPrefix(mglevels[0]->smoothd,"mg_coarse_");

    426:         /* coarse solve is (redundant) LU by default; set shifttype NONZERO to avoid annoying zero-pivot in LU preconditioner */
    427:         KSPSetType(mglevels[0]->smoothd,KSPPREONLY);
    428:         KSPGetPC(mglevels[0]->smoothd,&ipc);
    429:         MPI_Comm_size(comm,&size);
    430:         if (size > 1) {
    431:           PCSetType(ipc,PCREDUNDANT);
    432:         } else {
    433:           PCSetType(ipc,PCLU);
    434:         }
    435:         PCFactorSetShiftType(ipc,MAT_SHIFT_INBLOCKS);
    436:       }
    437:       PetscLogObjectParent((PetscObject)pc,(PetscObject)mglevels[i]->smoothd);
    438:     }
    439:     mglevels[i]->smoothu = mglevels[i]->smoothd;
    440:     mg->rtol             = 0.0;
    441:     mg->abstol           = 0.0;
    442:     mg->dtol             = 0.0;
    443:     mg->ttol             = 0.0;
    444:     mg->cyclesperpcapply = 1;
    445:   }
    446:   mg->levels = mglevels;
    447:   PCMGSetType(pc,mgtype);
    448:   return(0);
    449: }

    451: /*@C
    452:    PCMGSetLevels - Sets the number of levels to use with MG.
    453:    Must be called before any other MG routine.

    455:    Logically Collective on PC

    457:    Input Parameters:
    458: +  pc - the preconditioner context
    459: .  levels - the number of levels
    460: -  comms - optional communicators for each level; this is to allow solving the coarser problems
    461:            on smaller sets of processes. For processes that are not included in the computation
    462:            you must pass MPI_COMM_NULL. Use comms = NULL to specify that all processes
    463:            should participate in each level of problem.

    465:    Level: intermediate

    467:    Notes:
    468:      If the number of levels is one then the multigrid uses the -mg_levels prefix
    469:      for setting the level options rather than the -mg_coarse prefix.

    471:      You can free the information in comms after this routine is called.

    473:      The array of MPI communicators must contain MPI_COMM_NULL for those ranks that at each level
    474:      are not participating in the coarser solve. For example, with 2 levels and 1 and 2 ranks on
    475:      the two levels, rank 0 in the original communicator will pass in an array of 2 communicators
    476:      of size 2 and 1, while rank 1 in the original communicator will pass in array of 2 communicators
    477:      the first of size 2 and the second of value MPI_COMM_NULL since the rank 1 does not participate
    478:      in the coarse grid solve.

    480:      Since each coarser level may have a new MPI_Comm with fewer ranks than the previous, one
    481:      must take special care in providing the restriction and interpolation operation. We recommend
    482:      providing these as two step operations; first perform a standard restriction or interpolation on
    483:      the full number of ranks for that level and then use an MPI call to copy the resulting vector
    484:      array entries (after calls to VecGetArray()) to the smaller or larger number of ranks, note in both
    485:      cases the MPI calls must be made on the larger of the two communicators. Traditional MPI send and
    486:      recieves or MPI_AlltoAllv() could be used to do the reshuffling of the vector entries.

    488:    Fortran Notes:
    489:      Use comms = PETSC_NULL_MPI_COMM as the equivalent of NULL in the C interface. Note PETSC_NULL_MPI_COMM
    490:      is not MPI_COMM_NULL. It is more like PETSC_NULL_INTEGER, PETSC_NULL_REAL etc.

    492: .seealso: PCMGSetType(), PCMGGetLevels()
    493: @*/
    494: PetscErrorCode PCMGSetLevels(PC pc,PetscInt levels,MPI_Comm *comms)
    495: {

    501:   PetscTryMethod(pc,"PCMGSetLevels_C",(PC,PetscInt,MPI_Comm*),(pc,levels,comms));
    502:   return(0);
    503: }

    506: PetscErrorCode PCDestroy_MG(PC pc)
    507: {
    509:   PC_MG          *mg        = (PC_MG*)pc->data;
    510:   PC_MG_Levels   **mglevels = mg->levels;
    511:   PetscInt       i,n;

    514:   PCReset_MG(pc);
    515:   if (mglevels) {
    516:     n = mglevels[0]->levels;
    517:     for (i=0; i<n; i++) {
    518:       if (mglevels[i]->smoothd != mglevels[i]->smoothu) {
    519:         KSPDestroy(&mglevels[i]->smoothd);
    520:       }
    521:       KSPDestroy(&mglevels[i]->smoothu);
    522:       KSPDestroy(&mglevels[i]->cr);
    523:       PetscFree(mglevels[i]);
    524:     }
    525:     PetscFree(mg->levels);
    526:   }
    527:   PetscFree(pc->data);
    528:   PetscObjectComposeFunction((PetscObject)pc,"PCGetInterpolations_C",NULL);
    529:   PetscObjectComposeFunction((PetscObject)pc,"PCGetCoarseOperators_C",NULL);
    530:   return(0);
    531: }

    534: /*
    535:    PCApply_MG - Runs either an additive, multiplicative, Kaskadic
    536:              or full cycle of multigrid.

    538:   Note:
    539:   A simple wrapper which calls PCMGMCycle(),PCMGACycle(), or PCMGFCycle().
    540: */
    541: static PetscErrorCode PCApply_MG_Internal(PC pc,Vec b,Vec x,Mat B,Mat X,PetscBool transpose)
    542: {
    543:   PC_MG          *mg        = (PC_MG*)pc->data;
    544:   PC_MG_Levels   **mglevels = mg->levels;
    546:   PC             tpc;
    547:   PetscInt       levels = mglevels[0]->levels,i;
    548:   PetscBool      changeu,changed,matapp;

    551:   matapp = (PetscBool)(B && X);
    552:   if (mg->stageApply) {PetscLogStagePush(mg->stageApply);}
    553:   /* When the DM is supplying the matrix then it will not exist until here */
    554:   for (i=0; i<levels; i++) {
    555:     if (!mglevels[i]->A) {
    556:       KSPGetOperators(mglevels[i]->smoothu,&mglevels[i]->A,NULL);
    557:       PetscObjectReference((PetscObject)mglevels[i]->A);
    558:     }
    559:   }

    561:   KSPGetPC(mglevels[levels-1]->smoothd,&tpc);
    562:   PCPreSolveChangeRHS(tpc,&changed);
    563:   KSPGetPC(mglevels[levels-1]->smoothu,&tpc);
    564:   PCPreSolveChangeRHS(tpc,&changeu);
    565:   if (!changeu && !changed) {
    566:     if (matapp) {
    567:       MatDestroy(&mglevels[levels-1]->B);
    568:       mglevels[levels-1]->B = B;
    569:     } else {
    570:       VecDestroy(&mglevels[levels-1]->b);
    571:       mglevels[levels-1]->b = b;
    572:     }
    573:   } else { /* if the smoother changes the rhs during PreSolve, we cannot use the input vector */
    574:     if (matapp) {
    575:       if (mglevels[levels-1]->B) {
    576:         PetscInt  N1,N2;
    577:         PetscBool flg;

    579:         MatGetSize(mglevels[levels-1]->B,NULL,&N1);
    580:         MatGetSize(B,NULL,&N2);
    581:         PetscObjectTypeCompare((PetscObject)mglevels[levels-1]->B,((PetscObject)B)->type_name,&flg);
    582:         if (N1 != N2 || !flg) {
    583:           MatDestroy(&mglevels[levels-1]->B);
    584:         }
    585:       }
    586:       if (!mglevels[levels-1]->B) {
    587:         MatDuplicate(B,MAT_COPY_VALUES,&mglevels[levels-1]->B);
    588:       } else {
    589:         MatCopy(B,mglevels[levels-1]->B,SAME_NONZERO_PATTERN);
    590:       }
    591:     } else {
    592:       if (!mglevels[levels-1]->b) {
    593:         Vec *vec;

    595:         KSPCreateVecs(mglevels[levels-1]->smoothd,1,&vec,0,NULL);
    596:         mglevels[levels-1]->b = *vec;
    597:         PetscFree(vec);
    598:       }
    599:       VecCopy(b,mglevels[levels-1]->b);
    600:     }
    601:   }
    602:   if (matapp) { mglevels[levels-1]->X = X; }
    603:   else { mglevels[levels-1]->x = x; }

    605:   /* If coarser Xs are present, it means we have already block applied the PC at least once
    606:      Reset operators if sizes/type do no match */
    607:   if (matapp && levels > 1 && mglevels[levels-2]->X) {
    608:     PetscInt  Xc,Bc;
    609:     PetscBool flg;

    611:     MatGetSize(mglevels[levels-2]->X,NULL,&Xc);
    612:     MatGetSize(mglevels[levels-1]->B,NULL,&Bc);
    613:     PetscObjectTypeCompare((PetscObject)mglevels[levels-2]->X,((PetscObject)mglevels[levels-1]->X)->type_name,&flg);
    614:     if (Xc != Bc || !flg) {
    615:       MatDestroy(&mglevels[levels-1]->R);
    616:       for (i=0;i<levels-1;i++) {
    617:         MatDestroy(&mglevels[i]->R);
    618:         MatDestroy(&mglevels[i]->B);
    619:         MatDestroy(&mglevels[i]->X);
    620:       }
    621:     }
    622:   }

    624:   if (mg->am == PC_MG_MULTIPLICATIVE) {
    625:     if (matapp) { MatZeroEntries(X); }
    626:     else { VecZeroEntries(x); }
    627:     for (i=0; i<mg->cyclesperpcapply; i++) {
    628:       PCMGMCycle_Private(pc,mglevels+levels-1,transpose,matapp,NULL);
    629:     }
    630:   } else if (mg->am == PC_MG_ADDITIVE) {
    631:     PCMGACycle_Private(pc,mglevels,transpose,matapp);
    632:   } else if (mg->am == PC_MG_KASKADE) {
    633:     PCMGKCycle_Private(pc,mglevels,transpose,matapp);
    634:   } else {
    635:     PCMGFCycle_Private(pc,mglevels,transpose,matapp);
    636:   }
    637:   if (mg->stageApply) {PetscLogStagePop();}
    638:   if (!changeu && !changed) {
    639:     if (matapp) { mglevels[levels-1]->B = NULL; }
    640:     else { mglevels[levels-1]->b = NULL; }
    641:   }
    642:   return(0);
    643: }

    645: static PetscErrorCode PCApply_MG(PC pc,Vec b,Vec x)
    646: {

    650:   PCApply_MG_Internal(pc,b,x,NULL,NULL,PETSC_FALSE);
    651:   return(0);
    652: }

    654: static PetscErrorCode PCApplyTranspose_MG(PC pc,Vec b,Vec x)
    655: {

    659:   PCApply_MG_Internal(pc,b,x,NULL,NULL,PETSC_TRUE);
    660:   return(0);
    661: }

    663: static PetscErrorCode PCMatApply_MG(PC pc,Mat b,Mat x)
    664: {

    668:   PCApply_MG_Internal(pc,NULL,NULL,b,x,PETSC_FALSE);
    669:   return(0);
    670: }

    672: PetscErrorCode PCSetFromOptions_MG(PetscOptionItems *PetscOptionsObject,PC pc)
    673: {
    674:   PetscErrorCode   ierr;
    675:   PetscInt         levels,cycles;
    676:   PetscBool        flg, flg2;
    677:   PC_MG            *mg = (PC_MG*)pc->data;
    678:   PC_MG_Levels     **mglevels;
    679:   PCMGType         mgtype;
    680:   PCMGCycleType    mgctype;
    681:   PCMGGalerkinType gtype;

    684:   levels = PetscMax(mg->nlevels,1);
    685:   PetscOptionsHead(PetscOptionsObject,"Multigrid options");
    686:   PetscOptionsInt("-pc_mg_levels","Number of Levels","PCMGSetLevels",levels,&levels,&flg);
    687:   if (!flg && !mg->levels && pc->dm) {
    688:     DMGetRefineLevel(pc->dm,&levels);
    689:     levels++;
    690:     mg->usedmfornumberoflevels = PETSC_TRUE;
    691:   }
    692:   PCMGSetLevels(pc,levels,NULL);
    693:   mglevels = mg->levels;

    695:   mgctype = (PCMGCycleType) mglevels[0]->cycles;
    696:   PetscOptionsEnum("-pc_mg_cycle_type","V cycle or for W-cycle","PCMGSetCycleType",PCMGCycleTypes,(PetscEnum)mgctype,(PetscEnum*)&mgctype,&flg);
    697:   if (flg) {
    698:     PCMGSetCycleType(pc,mgctype);
    699:   }
    700:   gtype = mg->galerkin;
    701:   PetscOptionsEnum("-pc_mg_galerkin","Use Galerkin process to compute coarser operators","PCMGSetGalerkin",PCMGGalerkinTypes,(PetscEnum)gtype,(PetscEnum*)&gtype,&flg);
    702:   if (flg) {
    703:     PCMGSetGalerkin(pc,gtype);
    704:   }
    705:   flg2 = PETSC_FALSE;
    706:   PetscOptionsBool("-pc_mg_adapt_interp","Adapt interpolation using some coarse space","PCMGSetAdaptInterpolation",PETSC_FALSE,&flg2,&flg);
    707:   if (flg) {PCMGSetAdaptInterpolation(pc, flg2);}
    708:   PetscOptionsInt("-pc_mg_adapt_interp_n","Size of the coarse space for adaptive interpolation","PCMGSetCoarseSpace",mg->Nc,&mg->Nc,&flg);
    709:   PetscOptionsEnum("-pc_mg_adapt_interp_coarse_space","Type of coarse space: polynomial, harmonic, eigenvector, generalized_eigenvector","PCMGSetAdaptCoarseSpaceType",PCMGCoarseSpaceTypes,(PetscEnum)mg->coarseSpaceType,(PetscEnum*)&mg->coarseSpaceType,&flg);
    710:   PetscOptionsBool("-pc_mg_mesp_monitor","Monitor the multilevel eigensolver","PCMGSetAdaptInterpolation",PETSC_FALSE,&mg->mespMonitor,&flg);
    711:   flg2 = PETSC_FALSE;
    712:   PetscOptionsBool("-pc_mg_adapt_cr","Monitor coarse space quality using Compatible Relaxation (CR)","PCMGSetAdaptCR",PETSC_FALSE,&flg2,&flg);
    713:   if (flg) {PCMGSetAdaptCR(pc, flg2);}
    714:   flg = PETSC_FALSE;
    715:   PetscOptionsBool("-pc_mg_distinct_smoothup","Create separate smoothup KSP and append the prefix _up","PCMGSetDistinctSmoothUp",PETSC_FALSE,&flg,NULL);
    716:   if (flg) {
    717:     PCMGSetDistinctSmoothUp(pc);
    718:   }
    719:   mgtype = mg->am;
    720:   PetscOptionsEnum("-pc_mg_type","Multigrid type","PCMGSetType",PCMGTypes,(PetscEnum)mgtype,(PetscEnum*)&mgtype,&flg);
    721:   if (flg) {
    722:     PCMGSetType(pc,mgtype);
    723:   }
    724:   if (mg->am == PC_MG_MULTIPLICATIVE) {
    725:     PetscOptionsInt("-pc_mg_multiplicative_cycles","Number of cycles for each preconditioner step","PCMGMultiplicativeSetCycles",mg->cyclesperpcapply,&cycles,&flg);
    726:     if (flg) {
    727:       PCMGMultiplicativeSetCycles(pc,cycles);
    728:     }
    729:   }
    730:   flg  = PETSC_FALSE;
    731:   PetscOptionsBool("-pc_mg_log","Log times for each multigrid level","None",flg,&flg,NULL);
    732:   if (flg) {
    733:     PetscInt i;
    734:     char     eventname[128];

    736:     levels = mglevels[0]->levels;
    737:     for (i=0; i<levels; i++) {
    738:       sprintf(eventname,"MGSetup Level %d",(int)i);
    739:       PetscLogEventRegister(eventname,((PetscObject)pc)->classid,&mglevels[i]->eventsmoothsetup);
    740:       sprintf(eventname,"MGSmooth Level %d",(int)i);
    741:       PetscLogEventRegister(eventname,((PetscObject)pc)->classid,&mglevels[i]->eventsmoothsolve);
    742:       if (i) {
    743:         sprintf(eventname,"MGResid Level %d",(int)i);
    744:         PetscLogEventRegister(eventname,((PetscObject)pc)->classid,&mglevels[i]->eventresidual);
    745:         sprintf(eventname,"MGInterp Level %d",(int)i);
    746:         PetscLogEventRegister(eventname,((PetscObject)pc)->classid,&mglevels[i]->eventinterprestrict);
    747:       }
    748:     }

    750: #if defined(PETSC_USE_LOG)
    751:     {
    752:       const char    *sname = "MG Apply";
    753:       PetscStageLog stageLog;
    754:       PetscInt      st;

    756:       PetscLogGetStageLog(&stageLog);
    757:       for (st = 0; st < stageLog->numStages; ++st) {
    758:         PetscBool same;

    760:         PetscStrcmp(stageLog->stageInfo[st].name, sname, &same);
    761:         if (same) mg->stageApply = st;
    762:       }
    763:       if (!mg->stageApply) {
    764:         PetscLogStageRegister(sname, &mg->stageApply);
    765:       }
    766:     }
    767: #endif
    768:   }
    769:   PetscOptionsTail();
    770:   /* Check option consistency */
    771:   PCMGGetGalerkin(pc, &gtype);
    772:   PCMGGetAdaptInterpolation(pc, &flg);
    773:   if (flg && (gtype >= PC_MG_GALERKIN_NONE)) SETERRQ(PetscObjectComm((PetscObject) pc), PETSC_ERR_ARG_INCOMP, "Must use Galerkin coarse operators when adapting the interpolator");
    774:   return(0);
    775: }

    777: const char *const PCMGTypes[] = {"MULTIPLICATIVE","ADDITIVE","FULL","KASKADE","PCMGType","PC_MG",NULL};
    778: const char *const PCMGCycleTypes[] = {"invalid","v","w","PCMGCycleType","PC_MG_CYCLE",NULL};
    779: const char *const PCMGGalerkinTypes[] = {"both","pmat","mat","none","external","PCMGGalerkinType","PC_MG_GALERKIN",NULL};
    780: const char *const PCMGCoarseSpaceTypes[] = {"polynomial","harmonic","eigenvector","generalized_eigenvector","PCMGCoarseSpaceType","PCMG_POLYNOMIAL",NULL};

    782: #include <petscdraw.h>
    783: PetscErrorCode PCView_MG(PC pc,PetscViewer viewer)
    784: {
    785:   PC_MG          *mg        = (PC_MG*)pc->data;
    786:   PC_MG_Levels   **mglevels = mg->levels;
    788:   PetscInt       levels = mglevels ? mglevels[0]->levels : 0,i;
    789:   PetscBool      iascii,isbinary,isdraw;

    792:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
    793:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERBINARY,&isbinary);
    794:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERDRAW,&isdraw);
    795:   if (iascii) {
    796:     const char *cyclename = levels ? (mglevels[0]->cycles == PC_MG_CYCLE_V ? "v" : "w") : "unknown";
    797:     PetscViewerASCIIPrintf(viewer,"  type is %s, levels=%D cycles=%s\n", PCMGTypes[mg->am],levels,cyclename);
    798:     if (mg->am == PC_MG_MULTIPLICATIVE) {
    799:       PetscViewerASCIIPrintf(viewer,"    Cycles per PCApply=%d\n",mg->cyclesperpcapply);
    800:     }
    801:     if (mg->galerkin == PC_MG_GALERKIN_BOTH) {
    802:       PetscViewerASCIIPrintf(viewer,"    Using Galerkin computed coarse grid matrices\n");
    803:     } else if (mg->galerkin == PC_MG_GALERKIN_PMAT) {
    804:       PetscViewerASCIIPrintf(viewer,"    Using Galerkin computed coarse grid matrices for pmat\n");
    805:     } else if (mg->galerkin == PC_MG_GALERKIN_MAT) {
    806:       PetscViewerASCIIPrintf(viewer,"    Using Galerkin computed coarse grid matrices for mat\n");
    807:     } else if (mg->galerkin == PC_MG_GALERKIN_EXTERNAL) {
    808:       PetscViewerASCIIPrintf(viewer,"    Using externally compute Galerkin coarse grid matrices\n");
    809:     } else {
    810:       PetscViewerASCIIPrintf(viewer,"    Not using Galerkin computed coarse grid matrices\n");
    811:     }
    812:     if (mg->view){
    813:       (*mg->view)(pc,viewer);
    814:     }
    815:     for (i=0; i<levels; i++) {
    816:       if (!i) {
    817:         PetscViewerASCIIPrintf(viewer,"Coarse grid solver -- level -------------------------------\n",i);
    818:       } else {
    819:         PetscViewerASCIIPrintf(viewer,"Down solver (pre-smoother) on level %D -------------------------------\n",i);
    820:       }
    821:       PetscViewerASCIIPushTab(viewer);
    822:       KSPView(mglevels[i]->smoothd,viewer);
    823:       PetscViewerASCIIPopTab(viewer);
    824:       if (i && mglevels[i]->smoothd == mglevels[i]->smoothu) {
    825:         PetscViewerASCIIPrintf(viewer,"Up solver (post-smoother) same as down solver (pre-smoother)\n");
    826:       } else if (i) {
    827:         PetscViewerASCIIPrintf(viewer,"Up solver (post-smoother) on level %D -------------------------------\n",i);
    828:         PetscViewerASCIIPushTab(viewer);
    829:         KSPView(mglevels[i]->smoothu,viewer);
    830:         PetscViewerASCIIPopTab(viewer);
    831:       }
    832:       if (i && mglevels[i]->cr) {
    833:         PetscViewerASCIIPrintf(viewer,"CR solver on level %D -------------------------------\n",i);
    834:         PetscViewerASCIIPushTab(viewer);
    835:         KSPView(mglevels[i]->cr,viewer);
    836:         PetscViewerASCIIPopTab(viewer);
    837:       }
    838:     }
    839:   } else if (isbinary) {
    840:     for (i=levels-1; i>=0; i--) {
    841:       KSPView(mglevels[i]->smoothd,viewer);
    842:       if (i && mglevels[i]->smoothd != mglevels[i]->smoothu) {
    843:         KSPView(mglevels[i]->smoothu,viewer);
    844:       }
    845:     }
    846:   } else if (isdraw) {
    847:     PetscDraw draw;
    848:     PetscReal x,w,y,bottom,th;
    849:     PetscViewerDrawGetDraw(viewer,0,&draw);
    850:     PetscDrawGetCurrentPoint(draw,&x,&y);
    851:     PetscDrawStringGetSize(draw,NULL,&th);
    852:     bottom = y - th;
    853:     for (i=levels-1; i>=0; i--) {
    854:       if (!mglevels[i]->smoothu || (mglevels[i]->smoothu == mglevels[i]->smoothd)) {
    855:         PetscDrawPushCurrentPoint(draw,x,bottom);
    856:         KSPView(mglevels[i]->smoothd,viewer);
    857:         PetscDrawPopCurrentPoint(draw);
    858:       } else {
    859:         w    = 0.5*PetscMin(1.0-x,x);
    860:         PetscDrawPushCurrentPoint(draw,x+w,bottom);
    861:         KSPView(mglevels[i]->smoothd,viewer);
    862:         PetscDrawPopCurrentPoint(draw);
    863:         PetscDrawPushCurrentPoint(draw,x-w,bottom);
    864:         KSPView(mglevels[i]->smoothu,viewer);
    865:         PetscDrawPopCurrentPoint(draw);
    866:       }
    867:       PetscDrawGetBoundingBox(draw,NULL,&bottom,NULL,NULL);
    868:       bottom -= th;
    869:     }
    870:   }
    871:   return(0);
    872: }

    874: #include <petsc/private/kspimpl.h>

    876: /*
    877:     Calls setup for the KSP on each level
    878: */
    879: PetscErrorCode PCSetUp_MG(PC pc)
    880: {
    881:   PC_MG          *mg        = (PC_MG*)pc->data;
    882:   PC_MG_Levels   **mglevels = mg->levels;
    884:   PetscInt       i,n;
    885:   PC             cpc;
    886:   PetscBool      dump = PETSC_FALSE,opsset,use_amat,missinginterpolate = PETSC_FALSE;
    887:   Mat            dA,dB;
    888:   Vec            tvec;
    889:   DM             *dms;
    890:   PetscViewer    viewer = NULL;
    891:   PetscBool      dAeqdB = PETSC_FALSE, needRestricts = PETSC_FALSE, doCR = PETSC_FALSE;

    894:   if (!mglevels) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must set MG levels with PCMGSetLevels() before setting up");
    895:   n = mglevels[0]->levels;
    896:   /* FIX: Move this to PCSetFromOptions_MG? */
    897:   if (mg->usedmfornumberoflevels) {
    898:     PetscInt levels;
    899:     DMGetRefineLevel(pc->dm,&levels);
    900:     levels++;
    901:     if (levels > n) { /* the problem is now being solved on a finer grid */
    902:       PCMGSetLevels(pc,levels,NULL);
    903:       n        = levels;
    904:       PCSetFromOptions(pc); /* it is bad to call this here, but otherwise will never be called for the new hierarchy */
    905:       mglevels = mg->levels;
    906:     }
    907:   }
    908:   KSPGetPC(mglevels[0]->smoothd,&cpc);

    911:   /* If user did not provide fine grid operators OR operator was not updated since last global KSPSetOperators() */
    912:   /* so use those from global PC */
    913:   /* Is this what we always want? What if user wants to keep old one? */
    914:   KSPGetOperatorsSet(mglevels[n-1]->smoothd,NULL,&opsset);
    915:   if (opsset) {
    916:     Mat mmat;
    917:     KSPGetOperators(mglevels[n-1]->smoothd,NULL,&mmat);
    918:     if (mmat == pc->pmat) opsset = PETSC_FALSE;
    919:   }

    921:   /* Create CR solvers */
    922:   PCMGGetAdaptCR(pc, &doCR);
    923:   if (doCR) {
    924:     const char *prefix;

    926:     PCGetOptionsPrefix(pc, &prefix);
    927:     for (i = 1; i < n; ++i) {
    928:       PC   ipc, cr;
    929:       char crprefix[128];

    931:       KSPCreate(PetscObjectComm((PetscObject) pc), &mglevels[i]->cr);
    932:       KSPSetErrorIfNotConverged(mglevels[i]->cr, PETSC_FALSE);
    933:       PetscObjectIncrementTabLevel((PetscObject) mglevels[i]->cr, (PetscObject) pc, n-i);
    934:       KSPSetOptionsPrefix(mglevels[i]->cr, prefix);
    935:       PetscObjectComposedDataSetInt((PetscObject) mglevels[i]->cr, PetscMGLevelId, mglevels[i]->level);
    936:       KSPSetType(mglevels[i]->cr, KSPCHEBYSHEV);
    937:       KSPSetConvergenceTest(mglevels[i]->cr, KSPConvergedSkip, NULL, NULL);
    938:       KSPSetNormType(mglevels[i]->cr, KSP_NORM_PRECONDITIONED);
    939:       KSPGetPC(mglevels[i]->cr, &ipc);

    941:       PCSetType(ipc, PCCOMPOSITE);
    942:       PCCompositeSetType(ipc, PC_COMPOSITE_MULTIPLICATIVE);
    943:       PCCompositeAddPCType(ipc, PCSOR);
    944:       CreateCR_Private(pc, i, &cr);
    945:       PCCompositeAddPC(ipc, cr);
    946:       PCDestroy(&cr);

    948:       KSPSetTolerances(mglevels[i]->cr, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT, mg->default_smoothd);
    949:       KSPSetInitialGuessNonzero(mglevels[i]->cr, PETSC_TRUE);
    950:       PetscSNPrintf(crprefix, 128, "mg_levels_%d_cr_", (int) i);
    951:       KSPAppendOptionsPrefix(mglevels[i]->cr, crprefix);
    952:       PetscLogObjectParent((PetscObject) pc, (PetscObject) mglevels[i]->cr);
    953:     }
    954:   }

    956:   if (!opsset) {
    957:     PCGetUseAmat(pc,&use_amat);
    958:     if (use_amat) {
    959:       PetscInfo(pc,"Using outer operators to define finest grid operator \n  because PCMGGetSmoother(pc,nlevels-1,&ksp);KSPSetOperators(ksp,...); was not called.\n");
    960:       KSPSetOperators(mglevels[n-1]->smoothd,pc->mat,pc->pmat);
    961:     } else {
    962:       PetscInfo(pc,"Using matrix (pmat) operators to define finest grid operator \n  because PCMGGetSmoother(pc,nlevels-1,&ksp);KSPSetOperators(ksp,...); was not called.\n");
    963:       KSPSetOperators(mglevels[n-1]->smoothd,pc->pmat,pc->pmat);
    964:     }
    965:   }

    967:   for (i=n-1; i>0; i--) {
    968:     if (!(mglevels[i]->interpolate || mglevels[i]->restrct)) {
    969:       missinginterpolate = PETSC_TRUE;
    970:       continue;
    971:     }
    972:   }

    974:   KSPGetOperators(mglevels[n-1]->smoothd,&dA,&dB);
    975:   if (dA == dB) dAeqdB = PETSC_TRUE;
    976:   if ((mg->galerkin == PC_MG_GALERKIN_NONE) || (((mg->galerkin == PC_MG_GALERKIN_PMAT) || (mg->galerkin == PC_MG_GALERKIN_MAT)) && !dAeqdB)) {
    977:     needRestricts = PETSC_TRUE;  /* user must compute either mat, pmat, or both so must restrict x to coarser levels */
    978:   }

    981:   /*
    982:    Skipping if user has provided all interpolation/restriction needed (since DM might not be able to produce them (when coming from SNES/TS)
    983:    Skipping for galerkin==2 (externally managed hierarchy such as ML and GAMG). Cleaner logic here would be great. Wrap ML/GAMG as DMs?
    984:   */
    985:   if (missinginterpolate && pc->dm && mg->galerkin != PC_MG_GALERKIN_EXTERNAL && !pc->setupcalled) {
    986:         /* construct the interpolation from the DMs */
    987:     Mat p;
    988:     Vec rscale;
    989:     PetscMalloc1(n,&dms);
    990:     dms[n-1] = pc->dm;
    991:     /* Separately create them so we do not get DMKSP interference between levels */
    992:     for (i=n-2; i>-1; i--) {DMCoarsen(dms[i+1],MPI_COMM_NULL,&dms[i]);}
    993:         /*
    994:            Force the mat type of coarse level operator to be AIJ because usually we want to use LU for coarse level.
    995:            Notice that it can be overwritten by -mat_type because KSPSetUp() reads command line options.
    996:            But it is safe to use -dm_mat_type.

    998:            The mat type should not be hardcoded like this, we need to find a better way.
    999:     DMSetMatType(dms[0],MATAIJ);
    1000:     */
    1001:     for (i=n-2; i>-1; i--) {
    1002:       DMKSP     kdm;
    1003:       PetscBool dmhasrestrict, dmhasinject;
    1004:       KSPSetDM(mglevels[i]->smoothd,dms[i]);
    1005:       if (!needRestricts) {KSPSetDMActive(mglevels[i]->smoothd,PETSC_FALSE);}
    1006:       if (mglevels[i]->smoothd != mglevels[i]->smoothu) {
    1007:         KSPSetDM(mglevels[i]->smoothu,dms[i]);
    1008:         if (!needRestricts) {KSPSetDMActive(mglevels[i]->smoothu,PETSC_FALSE);}
    1009:       }
    1010:       if (mglevels[i]->cr) {
    1011:         KSPSetDM(mglevels[i]->cr,dms[i]);
    1012:         if (!needRestricts) {KSPSetDMActive(mglevels[i]->cr,PETSC_FALSE);}
    1013:       }
    1014:       DMGetDMKSPWrite(dms[i],&kdm);
    1015:       /* Ugly hack so that the next KSPSetUp() will use the RHS that we set. A better fix is to change dmActive to take
    1016:        * a bitwise OR of computing the matrix, RHS, and initial iterate. */
    1017:       kdm->ops->computerhs = NULL;
    1018:       kdm->rhsctx          = NULL;
    1019:       if (!mglevels[i+1]->interpolate) {
    1020:         DMCreateInterpolation(dms[i],dms[i+1],&p,&rscale);
    1021:         PCMGSetInterpolation(pc,i+1,p);
    1022:         if (rscale) {PCMGSetRScale(pc,i+1,rscale);}
    1023:         VecDestroy(&rscale);
    1024:         MatDestroy(&p);
    1025:       }
    1026:       DMHasCreateRestriction(dms[i],&dmhasrestrict);
    1027:       if (dmhasrestrict && !mglevels[i+1]->restrct){
    1028:         DMCreateRestriction(dms[i],dms[i+1],&p);
    1029:         PCMGSetRestriction(pc,i+1,p);
    1030:         MatDestroy(&p);
    1031:       }
    1032:       DMHasCreateInjection(dms[i],&dmhasinject);
    1033:       if (dmhasinject && !mglevels[i+1]->inject){
    1034:         DMCreateInjection(dms[i],dms[i+1],&p);
    1035:         PCMGSetInjection(pc,i+1,p);
    1036:         MatDestroy(&p);
    1037:       }
    1038:     }

    1040:     for (i=n-2; i>-1; i--) {DMDestroy(&dms[i]);}
    1041:     PetscFree(dms);
    1042:   }

    1044:   if (pc->dm && !pc->setupcalled) {
    1045:     /* finest smoother also gets DM but it is not active, independent of whether galerkin==PC_MG_GALERKIN_EXTERNAL */
    1046:     KSPSetDM(mglevels[n-1]->smoothd,pc->dm);
    1047:     KSPSetDMActive(mglevels[n-1]->smoothd,PETSC_FALSE);
    1048:     if (mglevels[n-1]->smoothd != mglevels[n-1]->smoothu) {
    1049:       KSPSetDM(mglevels[n-1]->smoothu,pc->dm);
    1050:       KSPSetDMActive(mglevels[n-1]->smoothu,PETSC_FALSE);
    1051:     }
    1052:     if (mglevels[n-1]->cr) {
    1053:       KSPSetDM(mglevels[n-1]->cr,pc->dm);
    1054:       KSPSetDMActive(mglevels[n-1]->cr,PETSC_FALSE);
    1055:     }
    1056:   }

    1058:   if (mg->galerkin < PC_MG_GALERKIN_NONE) {
    1059:     Mat       A,B;
    1060:     PetscBool doA = PETSC_FALSE,doB = PETSC_FALSE;
    1061:     MatReuse  reuse = MAT_INITIAL_MATRIX;

    1063:     if ((mg->galerkin == PC_MG_GALERKIN_PMAT) || (mg->galerkin == PC_MG_GALERKIN_BOTH)) doB = PETSC_TRUE;
    1064:     if ((mg->galerkin == PC_MG_GALERKIN_MAT) || ((mg->galerkin == PC_MG_GALERKIN_BOTH) && (dA != dB))) doA = PETSC_TRUE;
    1065:     if (pc->setupcalled) reuse = MAT_REUSE_MATRIX;
    1066:     for (i=n-2; i>-1; i--) {
    1067:       if (!mglevels[i+1]->restrct && !mglevels[i+1]->interpolate) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must provide interpolation or restriction for each MG level except level 0");
    1068:       if (!mglevels[i+1]->interpolate) {
    1069:         PCMGSetInterpolation(pc,i+1,mglevels[i+1]->restrct);
    1070:       }
    1071:       if (!mglevels[i+1]->restrct) {
    1072:         PCMGSetRestriction(pc,i+1,mglevels[i+1]->interpolate);
    1073:       }
    1074:       if (reuse == MAT_REUSE_MATRIX) {
    1075:         KSPGetOperators(mglevels[i]->smoothd,&A,&B);
    1076:       }
    1077:       if (doA) {
    1078:         MatGalerkin(mglevels[i+1]->restrct,dA,mglevels[i+1]->interpolate,reuse,1.0,&A);
    1079:       }
    1080:       if (doB) {
    1081:         MatGalerkin(mglevels[i+1]->restrct,dB,mglevels[i+1]->interpolate,reuse,1.0,&B);
    1082:       }
    1083:       /* the management of the PetscObjectReference() and PetscObjecDereference() below is rather delicate */
    1084:       if (!doA && dAeqdB) {
    1085:         if (reuse == MAT_INITIAL_MATRIX) {PetscObjectReference((PetscObject)B);}
    1086:         A = B;
    1087:       } else if (!doA && reuse == MAT_INITIAL_MATRIX) {
    1088:         KSPGetOperators(mglevels[i]->smoothd,&A,NULL);
    1089:         PetscObjectReference((PetscObject)A);
    1090:       }
    1091:       if (!doB && dAeqdB) {
    1092:         if (reuse == MAT_INITIAL_MATRIX) {PetscObjectReference((PetscObject)A);}
    1093:         B = A;
    1094:       } else if (!doB && reuse == MAT_INITIAL_MATRIX) {
    1095:         KSPGetOperators(mglevels[i]->smoothd,NULL,&B);
    1096:         PetscObjectReference((PetscObject)B);
    1097:       }
    1098:       if (reuse == MAT_INITIAL_MATRIX) {
    1099:         KSPSetOperators(mglevels[i]->smoothd,A,B);
    1100:         PetscObjectDereference((PetscObject)A);
    1101:         PetscObjectDereference((PetscObject)B);
    1102:       }
    1103:       dA = A;
    1104:       dB = B;
    1105:     }
    1106:   }

    1109:   /* Adapt interpolation matrices */
    1110:   if (mg->adaptInterpolation) {
    1111:     mg->Nc = mg->Nc < 0 ? 6 : mg->Nc; /* Default to 6 modes */
    1112:     for (i = 0; i < n; ++i) {
    1113:       PCMGComputeCoarseSpace_Internal(pc, i, mg->coarseSpaceType, mg->Nc, !i ? NULL : mglevels[i-1]->coarseSpace, &mglevels[i]->coarseSpace);
    1114:       if (i) {PCMGAdaptInterpolator_Internal(pc, i, mglevels[i-1]->smoothu, mglevels[i]->smoothu, mg->Nc, mglevels[i-1]->coarseSpace, mglevels[i]->coarseSpace);}
    1115:     }
    1116:     for (i = n-2; i > -1; --i) {
    1117:       PCMGRecomputeLevelOperators_Internal(pc, i);
    1118:     }
    1119:   }

    1121:   if (needRestricts && pc->dm) {
    1122:     for (i=n-2; i>=0; i--) {
    1123:       DM  dmfine,dmcoarse;
    1124:       Mat Restrict,Inject;
    1125:       Vec rscale;
    1126:       KSPGetDM(mglevels[i+1]->smoothd,&dmfine);
    1127:       KSPGetDM(mglevels[i]->smoothd,&dmcoarse);
    1128:       PCMGGetRestriction(pc,i+1,&Restrict);
    1129:       PCMGGetRScale(pc,i+1,&rscale);
    1130:       PCMGGetInjection(pc,i+1,&Inject);
    1131:       DMRestrict(dmfine,Restrict,rscale,Inject,dmcoarse);
    1132:     }
    1133:   }

    1135:   if (!pc->setupcalled) {
    1136:     for (i=0; i<n; i++) {
    1137:       KSPSetFromOptions(mglevels[i]->smoothd);
    1138:     }
    1139:     for (i=1; i<n; i++) {
    1140:       if (mglevels[i]->smoothu && (mglevels[i]->smoothu != mglevels[i]->smoothd)) {
    1141:         KSPSetFromOptions(mglevels[i]->smoothu);
    1142:       }
    1143:       if (mglevels[i]->cr) {
    1144:         KSPSetFromOptions(mglevels[i]->cr);
    1145:       }
    1146:     }
    1147:     /* insure that if either interpolation or restriction is set the other other one is set */
    1148:     for (i=1; i<n; i++) {
    1149:       PCMGGetInterpolation(pc,i,NULL);
    1150:       PCMGGetRestriction(pc,i,NULL);
    1151:     }
    1152:     for (i=0; i<n-1; i++) {
    1153:       if (!mglevels[i]->b) {
    1154:         Vec *vec;
    1155:         KSPCreateVecs(mglevels[i]->smoothd,1,&vec,0,NULL);
    1156:         PCMGSetRhs(pc,i,*vec);
    1157:         VecDestroy(vec);
    1158:         PetscFree(vec);
    1159:       }
    1160:       if (!mglevels[i]->r && i) {
    1161:         VecDuplicate(mglevels[i]->b,&tvec);
    1162:         PCMGSetR(pc,i,tvec);
    1163:         VecDestroy(&tvec);
    1164:       }
    1165:       if (!mglevels[i]->x) {
    1166:         VecDuplicate(mglevels[i]->b,&tvec);
    1167:         PCMGSetX(pc,i,tvec);
    1168:         VecDestroy(&tvec);
    1169:       }
    1170:       if (doCR) {
    1171:         VecDuplicate(mglevels[i]->b,&mglevels[i]->crx);
    1172:         VecDuplicate(mglevels[i]->b,&mglevels[i]->crb);
    1173:         VecZeroEntries(mglevels[i]->crb);
    1174:       }
    1175:     }
    1176:     if (n != 1 && !mglevels[n-1]->r) {
    1177:       /* PCMGSetR() on the finest level if user did not supply it */
    1178:       Vec *vec;
    1179:       KSPCreateVecs(mglevels[n-1]->smoothd,1,&vec,0,NULL);
    1180:       PCMGSetR(pc,n-1,*vec);
    1181:       VecDestroy(vec);
    1182:       PetscFree(vec);
    1183:     }
    1184:     if (doCR) {
    1185:       VecDuplicate(mglevels[n-1]->r, &mglevels[n-1]->crx);
    1186:       VecDuplicate(mglevels[n-1]->r, &mglevels[n-1]->crb);
    1187:       VecZeroEntries(mglevels[n-1]->crb);
    1188:     }
    1189:   }

    1191:   if (pc->dm) {
    1192:     /* need to tell all the coarser levels to rebuild the matrix using the DM for that level */
    1193:     for (i=0; i<n-1; i++) {
    1194:       if (mglevels[i]->smoothd->setupstage != KSP_SETUP_NEW) mglevels[i]->smoothd->setupstage = KSP_SETUP_NEWMATRIX;
    1195:     }
    1196:   }

    1198:   for (i=1; i<n; i++) {
    1199:     if (mglevels[i]->smoothu == mglevels[i]->smoothd || mg->am == PC_MG_FULL || mg->am == PC_MG_KASKADE || mg->cyclesperpcapply > 1){
    1200:       /* if doing only down then initial guess is zero */
    1201:       KSPSetInitialGuessNonzero(mglevels[i]->smoothd,PETSC_TRUE);
    1202:     }
    1203:     if (mglevels[i]->cr) {KSPSetInitialGuessNonzero(mglevels[i]->cr,PETSC_TRUE);}
    1204:     if (mglevels[i]->eventsmoothsetup) {PetscLogEventBegin(mglevels[i]->eventsmoothsetup,0,0,0,0);}
    1205:     KSPSetUp(mglevels[i]->smoothd);
    1206:     if (mglevels[i]->smoothd->reason == KSP_DIVERGED_PC_FAILED) {
    1207:       pc->failedreason = PC_SUBPC_ERROR;
    1208:     }
    1209:     if (mglevels[i]->eventsmoothsetup) {PetscLogEventEnd(mglevels[i]->eventsmoothsetup,0,0,0,0);}
    1210:     if (!mglevels[i]->residual) {
    1211:       Mat mat;
    1212:       KSPGetOperators(mglevels[i]->smoothd,&mat,NULL);
    1213:       PCMGSetResidual(pc,i,PCMGResidualDefault,mat);
    1214:     }
    1215:     if (!mglevels[i]->residualtranspose) {
    1216:       Mat mat;
    1217:       KSPGetOperators(mglevels[i]->smoothd,&mat,NULL);
    1218:       PCMGSetResidualTranspose(pc,i,PCMGResidualTransposeDefault,mat);
    1219:     }
    1220:   }
    1221:   for (i=1; i<n; i++) {
    1222:     if (mglevels[i]->smoothu && mglevels[i]->smoothu != mglevels[i]->smoothd) {
    1223:       Mat downmat,downpmat;

    1225:       /* check if operators have been set for up, if not use down operators to set them */
    1226:       KSPGetOperatorsSet(mglevels[i]->smoothu,&opsset,NULL);
    1227:       if (!opsset) {
    1228:         KSPGetOperators(mglevels[i]->smoothd,&downmat,&downpmat);
    1229:         KSPSetOperators(mglevels[i]->smoothu,downmat,downpmat);
    1230:       }

    1232:       KSPSetInitialGuessNonzero(mglevels[i]->smoothu,PETSC_TRUE);
    1233:       if (mglevels[i]->eventsmoothsetup) {PetscLogEventBegin(mglevels[i]->eventsmoothsetup,0,0,0,0);}
    1234:       KSPSetUp(mglevels[i]->smoothu);
    1235:       if (mglevels[i]->smoothu->reason == KSP_DIVERGED_PC_FAILED) {
    1236:         pc->failedreason = PC_SUBPC_ERROR;
    1237:       }
    1238:       if (mglevels[i]->eventsmoothsetup) {PetscLogEventEnd(mglevels[i]->eventsmoothsetup,0,0,0,0);}
    1239:     }
    1240:     if (mglevels[i]->cr) {
    1241:       Mat downmat,downpmat;

    1243:       /* check if operators have been set for up, if not use down operators to set them */
    1244:       KSPGetOperatorsSet(mglevels[i]->cr,&opsset,NULL);
    1245:       if (!opsset) {
    1246:         KSPGetOperators(mglevels[i]->smoothd,&downmat,&downpmat);
    1247:         KSPSetOperators(mglevels[i]->cr,downmat,downpmat);
    1248:       }

    1250:       KSPSetInitialGuessNonzero(mglevels[i]->cr,PETSC_TRUE);
    1251:       if (mglevels[i]->eventsmoothsetup) {PetscLogEventBegin(mglevels[i]->eventsmoothsetup,0,0,0,0);}
    1252:       KSPSetUp(mglevels[i]->cr);
    1253:       if (mglevels[i]->cr->reason == KSP_DIVERGED_PC_FAILED) {
    1254:         pc->failedreason = PC_SUBPC_ERROR;
    1255:       }
    1256:       if (mglevels[i]->eventsmoothsetup) {PetscLogEventEnd(mglevels[i]->eventsmoothsetup,0,0,0,0);}
    1257:     }
    1258:   }

    1260:   if (mglevels[0]->eventsmoothsetup) {PetscLogEventBegin(mglevels[0]->eventsmoothsetup,0,0,0,0);}
    1261:   KSPSetUp(mglevels[0]->smoothd);
    1262:   if (mglevels[0]->smoothd->reason == KSP_DIVERGED_PC_FAILED) {
    1263:     pc->failedreason = PC_SUBPC_ERROR;
    1264:   }
    1265:   if (mglevels[0]->eventsmoothsetup) {PetscLogEventEnd(mglevels[0]->eventsmoothsetup,0,0,0,0);}

    1267:   /*
    1268:      Dump the interpolation/restriction matrices plus the
    1269:    Jacobian/stiffness on each level. This allows MATLAB users to
    1270:    easily check if the Galerkin condition A_c = R A_f R^T is satisfied.

    1272:    Only support one or the other at the same time.
    1273:   */
    1274: #if defined(PETSC_USE_SOCKET_VIEWER)
    1275:   PetscOptionsGetBool(((PetscObject)pc)->options,((PetscObject)pc)->prefix,"-pc_mg_dump_matlab",&dump,NULL);
    1276:   if (dump) viewer = PETSC_VIEWER_SOCKET_(PetscObjectComm((PetscObject)pc));
    1277:   dump = PETSC_FALSE;
    1278: #endif
    1279:   PetscOptionsGetBool(((PetscObject)pc)->options,((PetscObject)pc)->prefix,"-pc_mg_dump_binary",&dump,NULL);
    1280:   if (dump) viewer = PETSC_VIEWER_BINARY_(PetscObjectComm((PetscObject)pc));

    1282:   if (viewer) {
    1283:     for (i=1; i<n; i++) {
    1284:       MatView(mglevels[i]->restrct,viewer);
    1285:     }
    1286:     for (i=0; i<n; i++) {
    1287:       KSPGetPC(mglevels[i]->smoothd,&pc);
    1288:       MatView(pc->mat,viewer);
    1289:     }
    1290:   }
    1291:   return(0);
    1292: }

    1294: /* -------------------------------------------------------------------------------------*/

    1296: PetscErrorCode PCMGGetLevels_MG(PC pc, PetscInt *levels)
    1297: {
    1298:   PC_MG *mg = (PC_MG *) pc->data;

    1301:   *levels = mg->nlevels;
    1302:   return(0);
    1303: }

    1305: /*@
    1306:    PCMGGetLevels - Gets the number of levels to use with MG.

    1308:    Not Collective

    1310:    Input Parameter:
    1311: .  pc - the preconditioner context

    1313:    Output parameter:
    1314: .  levels - the number of levels

    1316:    Level: advanced

    1318: .seealso: PCMGSetLevels()
    1319: @*/
    1320: PetscErrorCode PCMGGetLevels(PC pc,PetscInt *levels)
    1321: {

    1327:   *levels = 0;
    1328:   PetscTryMethod(pc,"PCMGGetLevels_C",(PC,PetscInt*),(pc,levels));
    1329:   return(0);
    1330: }

    1332: /*@
    1333:    PCMGSetType - Determines the form of multigrid to use:
    1334:    multiplicative, additive, full, or the Kaskade algorithm.

    1336:    Logically Collective on PC

    1338:    Input Parameters:
    1339: +  pc - the preconditioner context
    1340: -  form - multigrid form, one of PC_MG_MULTIPLICATIVE, PC_MG_ADDITIVE,
    1341:    PC_MG_FULL, PC_MG_KASKADE

    1343:    Options Database Key:
    1344: .  -pc_mg_type <form> - Sets <form>, one of multiplicative,
    1345:    additive, full, kaskade

    1347:    Level: advanced

    1349: .seealso: PCMGSetLevels()
    1350: @*/
    1351: PetscErrorCode  PCMGSetType(PC pc,PCMGType form)
    1352: {
    1353:   PC_MG *mg = (PC_MG*)pc->data;

    1358:   mg->am = form;
    1359:   if (form == PC_MG_MULTIPLICATIVE) pc->ops->applyrichardson = PCApplyRichardson_MG;
    1360:   else pc->ops->applyrichardson = NULL;
    1361:   return(0);
    1362: }

    1364: /*@
    1365:    PCMGGetType - Determines the form of multigrid to use:
    1366:    multiplicative, additive, full, or the Kaskade algorithm.

    1368:    Logically Collective on PC

    1370:    Input Parameter:
    1371: .  pc - the preconditioner context

    1373:    Output Parameter:
    1374: .  type - one of PC_MG_MULTIPLICATIVE, PC_MG_ADDITIVE,PC_MG_FULL, PC_MG_KASKADE

    1377:    Level: advanced

    1379: .seealso: PCMGSetLevels()
    1380: @*/
    1381: PetscErrorCode  PCMGGetType(PC pc,PCMGType *type)
    1382: {
    1383:   PC_MG *mg = (PC_MG*)pc->data;

    1387:   *type = mg->am;
    1388:   return(0);
    1389: }

    1391: /*@
    1392:    PCMGSetCycleType - Sets the type cycles to use.  Use PCMGSetCycleTypeOnLevel() for more
    1393:    complicated cycling.

    1395:    Logically Collective on PC

    1397:    Input Parameters:
    1398: +  pc - the multigrid context
    1399: -  n - either PC_MG_CYCLE_V or PC_MG_CYCLE_W

    1401:    Options Database Key:
    1402: .  -pc_mg_cycle_type <v,w> - provide the cycle desired

    1404:    Level: advanced

    1406: .seealso: PCMGSetCycleTypeOnLevel()
    1407: @*/
    1408: PetscErrorCode  PCMGSetCycleType(PC pc,PCMGCycleType n)
    1409: {
    1410:   PC_MG        *mg        = (PC_MG*)pc->data;
    1411:   PC_MG_Levels **mglevels = mg->levels;
    1412:   PetscInt     i,levels;

    1417:   if (!mglevels) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ORDER,"Must set MG levels with PCMGSetLevels() before calling");
    1418:   levels = mglevels[0]->levels;
    1419:   for (i=0; i<levels; i++) mglevels[i]->cycles = n;
    1420:   return(0);
    1421: }

    1423: /*@
    1424:    PCMGMultiplicativeSetCycles - Sets the number of cycles to use for each preconditioner step
    1425:          of multigrid when PCMGType of PC_MG_MULTIPLICATIVE is used

    1427:    Logically Collective on PC

    1429:    Input Parameters:
    1430: +  pc - the multigrid context
    1431: -  n - number of cycles (default is 1)

    1433:    Options Database Key:
    1434: .  -pc_mg_multiplicative_cycles n

    1436:    Level: advanced

    1438:    Notes:
    1439:     This is not associated with setting a v or w cycle, that is set with PCMGSetCycleType()

    1441: .seealso: PCMGSetCycleTypeOnLevel(), PCMGSetCycleType()
    1442: @*/
    1443: PetscErrorCode  PCMGMultiplicativeSetCycles(PC pc,PetscInt n)
    1444: {
    1445:   PC_MG        *mg        = (PC_MG*)pc->data;

    1450:   mg->cyclesperpcapply = n;
    1451:   return(0);
    1452: }

    1454: PetscErrorCode PCMGSetGalerkin_MG(PC pc,PCMGGalerkinType use)
    1455: {
    1456:   PC_MG *mg = (PC_MG*)pc->data;

    1459:   mg->galerkin = use;
    1460:   return(0);
    1461: }

    1463: /*@
    1464:    PCMGSetGalerkin - Causes the coarser grid matrices to be computed from the
    1465:       finest grid via the Galerkin process: A_i-1 = r_i * A_i * p_i

    1467:    Logically Collective on PC

    1469:    Input Parameters:
    1470: +  pc - the multigrid context
    1471: -  use - one of PC_MG_GALERKIN_BOTH,PC_MG_GALERKIN_PMAT,PC_MG_GALERKIN_MAT, or PC_MG_GALERKIN_NONE

    1473:    Options Database Key:
    1474: .  -pc_mg_galerkin <both,pmat,mat,none>

    1476:    Level: intermediate

    1478:    Notes:
    1479:     Some codes that use PCMG such as PCGAMG use Galerkin internally while constructing the hierarchy and thus do not
    1480:      use the PCMG construction of the coarser grids.

    1482: .seealso: PCMGGetGalerkin(), PCMGGalerkinType

    1484: @*/
    1485: PetscErrorCode PCMGSetGalerkin(PC pc,PCMGGalerkinType use)
    1486: {

    1491:   PetscTryMethod(pc,"PCMGSetGalerkin_C",(PC,PCMGGalerkinType),(pc,use));
    1492:   return(0);
    1493: }

    1495: /*@
    1496:    PCMGGetGalerkin - Checks if Galerkin multigrid is being used, i.e.
    1497:       A_i-1 = r_i * A_i * p_i

    1499:    Not Collective

    1501:    Input Parameter:
    1502: .  pc - the multigrid context

    1504:    Output Parameter:
    1505: .  galerkin - one of PC_MG_GALERKIN_BOTH,PC_MG_GALERKIN_PMAT,PC_MG_GALERKIN_MAT, PC_MG_GALERKIN_NONE, or PC_MG_GALERKIN_EXTERNAL

    1507:    Level: intermediate

    1509: .seealso: PCMGSetGalerkin(), PCMGGalerkinType

    1511: @*/
    1512: PetscErrorCode  PCMGGetGalerkin(PC pc,PCMGGalerkinType  *galerkin)
    1513: {
    1514:   PC_MG *mg = (PC_MG*)pc->data;

    1518:   *galerkin = mg->galerkin;
    1519:   return(0);
    1520: }

    1522: PetscErrorCode PCMGSetAdaptInterpolation_MG(PC pc, PetscBool adapt)
    1523: {
    1524:   PC_MG *mg = (PC_MG *) pc->data;

    1527:   mg->adaptInterpolation = adapt;
    1528:   return(0);
    1529: }

    1531: PetscErrorCode PCMGGetAdaptInterpolation_MG(PC pc, PetscBool *adapt)
    1532: {
    1533:   PC_MG *mg = (PC_MG *) pc->data;

    1536:   *adapt = mg->adaptInterpolation;
    1537:   return(0);
    1538: }

    1540: PetscErrorCode PCMGSetAdaptCR_MG(PC pc, PetscBool cr)
    1541: {
    1542:   PC_MG *mg = (PC_MG *) pc->data;

    1545:   mg->compatibleRelaxation = cr;
    1546:   return(0);
    1547: }

    1549: PetscErrorCode PCMGGetAdaptCR_MG(PC pc, PetscBool *cr)
    1550: {
    1551:   PC_MG *mg = (PC_MG *) pc->data;

    1554:   *cr = mg->compatibleRelaxation;
    1555:   return(0);
    1556: }

    1558: /*@
    1559:   PCMGSetAdaptInterpolation - Adapt the interpolator based upon a vector space which should be accurately captured by the next coarser mesh, and thus accurately interpolated.

    1561:   Logically Collective on PC

    1563:   Input Parameters:
    1564: + pc    - the multigrid context
    1565: - adapt - flag for adaptation of the interpolator

    1567:   Options Database Keys:
    1568: + -pc_mg_adapt_interp                     - Turn on adaptation
    1569: . -pc_mg_adapt_interp_n <int>             - The number of modes to use, should be divisible by dimension
    1570: - -pc_mg_adapt_interp_coarse_space <type> - The type of coarse space: polynomial, harmonic, eigenvector, generalized_eigenvector

    1572:   Level: intermediate

    1574: .keywords: MG, set, Galerkin
    1575: .seealso: PCMGGetAdaptInterpolation(), PCMGSetGalerkin()
    1576: @*/
    1577: PetscErrorCode PCMGSetAdaptInterpolation(PC pc, PetscBool adapt)
    1578: {

    1583:   PetscTryMethod(pc,"PCMGSetAdaptInterpolation_C",(PC,PetscBool),(pc,adapt));
    1584:   return(0);
    1585: }

    1587: /*@
    1588:   PCMGGetAdaptInterpolation - Get the flag to adapt the interpolator based upon a vector space which should be accurately captured by the next coarser mesh, and thus accurately interpolated.

    1590:   Not collective

    1592:   Input Parameter:
    1593: . pc    - the multigrid context

    1595:   Output Parameter:
    1596: . adapt - flag for adaptation of the interpolator

    1598:   Level: intermediate

    1600: .keywords: MG, set, Galerkin
    1601: .seealso: PCMGSetAdaptInterpolation(), PCMGSetGalerkin()
    1602: @*/
    1603: PetscErrorCode PCMGGetAdaptInterpolation(PC pc, PetscBool *adapt)
    1604: {

    1610:   PetscUseMethod(pc,"PCMGGetAdaptInterpolation_C",(PC,PetscBool*),(pc,adapt));
    1611:   return(0);
    1612: }

    1614: /*@
    1615:   PCMGSetAdaptCR - Monitor the coarse space quality using an auxiliary solve with compatible relaxation.

    1617:   Logically Collective on PC

    1619:   Input Parameters:
    1620: + pc - the multigrid context
    1621: - cr - flag for compatible relaxation

    1623:   Options Database Keys:
    1624: . -pc_mg_adapt_cr - Turn on compatible relaxation

    1626:   Level: intermediate

    1628: .keywords: MG, set, Galerkin
    1629: .seealso: PCMGGetAdaptCR(), PCMGSetAdaptInterpolation(), PCMGSetGalerkin()
    1630: @*/
    1631: PetscErrorCode PCMGSetAdaptCR(PC pc, PetscBool cr)
    1632: {

    1637:   PetscTryMethod(pc,"PCMGSetAdaptCR_C",(PC,PetscBool),(pc,cr));
    1638:   return(0);
    1639: }

    1641: /*@
    1642:   PCMGGetAdaptCR - Get the flag to monitor coarse space quality using an auxiliary solve with compatible relaxation.

    1644:   Not collective

    1646:   Input Parameter:
    1647: . pc    - the multigrid context

    1649:   Output Parameter:
    1650: . cr - flag for compatible relaxaion

    1652:   Level: intermediate

    1654: .keywords: MG, set, Galerkin
    1655: .seealso: PCMGSetAdaptCR(), PCMGGetAdaptInterpolation(), PCMGSetGalerkin()
    1656: @*/
    1657: PetscErrorCode PCMGGetAdaptCR(PC pc, PetscBool *cr)
    1658: {

    1664:   PetscUseMethod(pc,"PCMGGetAdaptCR_C",(PC,PetscBool*),(pc,cr));
    1665:   return(0);
    1666: }

    1668: /*@
    1669:    PCMGSetNumberSmooth - Sets the number of pre and post-smoothing steps to use
    1670:    on all levels.  Use PCMGDistinctSmoothUp() to create separate up and down smoothers if you want different numbers of
    1671:    pre- and post-smoothing steps.

    1673:    Logically Collective on PC

    1675:    Input Parameters:
    1676: +  mg - the multigrid context
    1677: -  n - the number of smoothing steps

    1679:    Options Database Key:
    1680: .  -mg_levels_ksp_max_it <n> - Sets number of pre and post-smoothing steps

    1682:    Level: advanced

    1684:    Notes:
    1685:     this does not set a value on the coarsest grid, since we assume that
    1686:     there is no separate smooth up on the coarsest grid.

    1688: .seealso: PCMGSetDistinctSmoothUp()
    1689: @*/
    1690: PetscErrorCode  PCMGSetNumberSmooth(PC pc,PetscInt n)
    1691: {
    1692:   PC_MG          *mg        = (PC_MG*)pc->data;
    1693:   PC_MG_Levels   **mglevels = mg->levels;
    1695:   PetscInt       i,levels;

    1700:   if (!mglevels) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ORDER,"Must set MG levels with PCMGSetLevels() before calling");
    1701:   levels = mglevels[0]->levels;

    1703:   for (i=1; i<levels; i++) {
    1704:     KSPSetTolerances(mglevels[i]->smoothu,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,n);
    1705:     KSPSetTolerances(mglevels[i]->smoothd,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,n);
    1706:     mg->default_smoothu = n;
    1707:     mg->default_smoothd = n;
    1708:   }
    1709:   return(0);
    1710: }

    1712: /*@
    1713:    PCMGSetDistinctSmoothUp - sets the up (post) smoother to be a separate KSP from the down (pre) smoother on all levels
    1714:        and adds the suffix _up to the options name

    1716:    Logically Collective on PC

    1718:    Input Parameters:
    1719: .  pc - the preconditioner context

    1721:    Options Database Key:
    1722: .  -pc_mg_distinct_smoothup

    1724:    Level: advanced

    1726:    Notes:
    1727:     this does not set a value on the coarsest grid, since we assume that
    1728:     there is no separate smooth up on the coarsest grid.

    1730: .seealso: PCMGSetNumberSmooth()
    1731: @*/
    1732: PetscErrorCode  PCMGSetDistinctSmoothUp(PC pc)
    1733: {
    1734:   PC_MG          *mg        = (PC_MG*)pc->data;
    1735:   PC_MG_Levels   **mglevels = mg->levels;
    1737:   PetscInt       i,levels;
    1738:   KSP            subksp;

    1742:   if (!mglevels) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ORDER,"Must set MG levels with PCMGSetLevels() before calling");
    1743:   levels = mglevels[0]->levels;

    1745:   for (i=1; i<levels; i++) {
    1746:     const char *prefix = NULL;
    1747:     /* make sure smoother up and down are different */
    1748:     PCMGGetSmootherUp(pc,i,&subksp);
    1749:     KSPGetOptionsPrefix(mglevels[i]->smoothd,&prefix);
    1750:     KSPSetOptionsPrefix(subksp,prefix);
    1751:     KSPAppendOptionsPrefix(subksp,"up_");
    1752:   }
    1753:   return(0);
    1754: }

    1756: /* No new matrices are created, and the coarse operator matrices are the references to the original ones */
    1757: PetscErrorCode  PCGetInterpolations_MG(PC pc,PetscInt *num_levels,Mat *interpolations[])
    1758: {
    1759:   PC_MG          *mg        = (PC_MG*)pc->data;
    1760:   PC_MG_Levels   **mglevels = mg->levels;
    1761:   Mat            *mat;
    1762:   PetscInt       l;

    1766:   if (!mglevels) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must set MG levels before calling");
    1767:   PetscMalloc1(mg->nlevels,&mat);
    1768:   for (l=1; l< mg->nlevels; l++) {
    1769:     mat[l-1] = mglevels[l]->interpolate;
    1770:     PetscObjectReference((PetscObject)mat[l-1]);
    1771:   }
    1772:   *num_levels = mg->nlevels;
    1773:   *interpolations = mat;
    1774:   return(0);
    1775: }

    1777: /* No new matrices are created, and the coarse operator matrices are the references to the original ones */
    1778: PetscErrorCode  PCGetCoarseOperators_MG(PC pc,PetscInt *num_levels,Mat *coarseOperators[])
    1779: {
    1780:   PC_MG          *mg        = (PC_MG*)pc->data;
    1781:   PC_MG_Levels   **mglevels = mg->levels;
    1782:   PetscInt       l;
    1783:   Mat            *mat;

    1787:   if (!mglevels) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must set MG levels before calling");
    1788:   PetscMalloc1(mg->nlevels,&mat);
    1789:   for (l=0; l<mg->nlevels-1; l++) {
    1790:     KSPGetOperators(mglevels[l]->smoothd,NULL,&(mat[l]));
    1791:     PetscObjectReference((PetscObject)mat[l]);
    1792:   }
    1793:   *num_levels = mg->nlevels;
    1794:   *coarseOperators = mat;
    1795:   return(0);
    1796: }

    1798: /*@C
    1799:   PCMGRegisterCoarseSpaceConstructor -  Adds a method to the PCMG package for coarse space construction.

    1801:   Not collective

    1803:   Input Parameters:
    1804: + name     - name of the constructor
    1805: - function - constructor routine

    1807:   Notes:
    1808:   Calling sequence for the routine:
    1809: $ my_csp(PC pc, PetscInt l, DM dm, KSP smooth, PetscInt Nc, const Vec initGuess[], Vec **coarseSp)
    1810: $   pc        - The PC object
    1811: $   l         - The multigrid level, 0 is the coarse level
    1812: $   dm        - The DM for this level
    1813: $   smooth    - The level smoother
    1814: $   Nc        - The size of the coarse space
    1815: $   initGuess - Basis for an initial guess for the space
    1816: $   coarseSp  - A basis for the computed coarse space

    1818:   Level: advanced

    1820: .seealso: PCMGGetCoarseSpaceConstructor(), PCRegister()
    1821: @*/
    1822: PetscErrorCode PCMGRegisterCoarseSpaceConstructor(const char name[], PetscErrorCode (*function)(PC, PetscInt, DM, KSP, PetscInt, const Vec[], Vec **))
    1823: {

    1827:   PCInitializePackage();
    1828:   PetscFunctionListAdd(&PCMGCoarseList,name,function);
    1829:   return(0);
    1830: }

    1832: /*@C
    1833:   PCMGGetCoarseSpaceConstructor -  Returns the given coarse space construction method.

    1835:   Not collective

    1837:   Input Parameter:
    1838: . name     - name of the constructor

    1840:   Output Parameter:
    1841: . function - constructor routine

    1843:   Notes:
    1844:   Calling sequence for the routine:
    1845: $ my_csp(PC pc, PetscInt l, DM dm, KSP smooth, PetscInt Nc, const Vec initGuess[], Vec **coarseSp)
    1846: $   pc        - The PC object
    1847: $   l         - The multigrid level, 0 is the coarse level
    1848: $   dm        - The DM for this level
    1849: $   smooth    - The level smoother
    1850: $   Nc        - The size of the coarse space
    1851: $   initGuess - Basis for an initial guess for the space
    1852: $   coarseSp  - A basis for the computed coarse space

    1854:   Level: advanced

    1856: .seealso: PCMGRegisterCoarseSpaceConstructor(), PCRegister()
    1857: @*/
    1858: PetscErrorCode PCMGGetCoarseSpaceConstructor(const char name[], PetscErrorCode (**function)(PC, PetscInt, DM, KSP, PetscInt, const Vec[], Vec **))
    1859: {

    1863:   PetscFunctionListFind(PCMGCoarseList,name,function);
    1864:   return(0);
    1865: }

    1867: /* ----------------------------------------------------------------------------------------*/

    1869: /*MC
    1870:    PCMG - Use multigrid preconditioning. This preconditioner requires you provide additional
    1871:     information about the coarser grid matrices and restriction/interpolation operators.

    1873:    Options Database Keys:
    1874: +  -pc_mg_levels <nlevels> - number of levels including finest
    1875: .  -pc_mg_cycle_type <v,w> - provide the cycle desired
    1876: .  -pc_mg_type <additive,multiplicative,full,kaskade> - multiplicative is the default
    1877: .  -pc_mg_log - log information about time spent on each level of the solver
    1878: .  -pc_mg_distinct_smoothup - configure up (after interpolation) and down (before restriction) smoothers separately (with different options prefixes)
    1879: .  -pc_mg_galerkin <both,pmat,mat,none> - use Galerkin process to compute coarser operators, i.e. Acoarse = R A R'
    1880: .  -pc_mg_multiplicative_cycles - number of cycles to use as the preconditioner (defaults to 1)
    1881: .  -pc_mg_dump_matlab - dumps the matrices for each level and the restriction/interpolation matrices
    1882:                         to the Socket viewer for reading from MATLAB.
    1883: -  -pc_mg_dump_binary - dumps the matrices for each level and the restriction/interpolation matrices
    1884:                         to the binary output file called binaryoutput

    1886:    Notes:
    1887:     If one uses a Krylov method such GMRES or CG as the smoother then one must use KSPFGMRES, KSPGCR, or KSPRICHARDSON as the outer Krylov method

    1889:        When run with a single level the smoother options are used on that level NOT the coarse grid solver options

    1891:        When run with KSPRICHARDSON the convergence test changes slightly if monitor is turned on. The iteration count may change slightly. This
    1892:        is because without monitoring the residual norm is computed WITHIN each multigrid cycle on the finest level after the pre-smoothing
    1893:        (because the residual has just been computed for the multigrid algorithm and is hence available for free) while with monitoring the
    1894:        residual is computed at the end of each cycle.

    1896:    Level: intermediate

    1898: .seealso:  PCCreate(), PCSetType(), PCType (for list of available types), PC, PCMGType, PCEXOTIC, PCGAMG, PCML, PCHYPRE
    1899:            PCMGSetLevels(), PCMGGetLevels(), PCMGSetType(), PCMGSetCycleType(),
    1900:            PCMGSetDistinctSmoothUp(), PCMGGetCoarseSolve(), PCMGSetResidual(), PCMGSetInterpolation(),
    1901:            PCMGSetRestriction(), PCMGGetSmoother(), PCMGGetSmootherUp(), PCMGGetSmootherDown(),
    1902:            PCMGSetCycleTypeOnLevel(), PCMGSetRhs(), PCMGSetX(), PCMGSetR()
    1903: M*/

    1905: PETSC_EXTERN PetscErrorCode PCCreate_MG(PC pc)
    1906: {
    1907:   PC_MG          *mg;

    1911:   PetscNewLog(pc,&mg);
    1912:   pc->data     = (void*)mg;
    1913:   mg->nlevels  = -1;
    1914:   mg->am       = PC_MG_MULTIPLICATIVE;
    1915:   mg->galerkin = PC_MG_GALERKIN_NONE;
    1916:   mg->adaptInterpolation = PETSC_FALSE;
    1917:   mg->Nc                 = -1;
    1918:   mg->eigenvalue         = -1;

    1920:   pc->useAmat = PETSC_TRUE;

    1922:   pc->ops->apply          = PCApply_MG;
    1923:   pc->ops->applytranspose = PCApplyTranspose_MG;
    1924:   pc->ops->matapply       = PCMatApply_MG;
    1925:   pc->ops->setup          = PCSetUp_MG;
    1926:   pc->ops->reset          = PCReset_MG;
    1927:   pc->ops->destroy        = PCDestroy_MG;
    1928:   pc->ops->setfromoptions = PCSetFromOptions_MG;
    1929:   pc->ops->view           = PCView_MG;

    1931:   PetscObjectComposedDataRegister(&mg->eigenvalue);
    1932:   PetscObjectComposeFunction((PetscObject)pc,"PCMGSetGalerkin_C",PCMGSetGalerkin_MG);
    1933:   PetscObjectComposeFunction((PetscObject)pc,"PCMGGetLevels_C",PCMGGetLevels_MG);
    1934:   PetscObjectComposeFunction((PetscObject)pc,"PCMGSetLevels_C",PCMGSetLevels_MG);
    1935:   PetscObjectComposeFunction((PetscObject)pc,"PCGetInterpolations_C",PCGetInterpolations_MG);
    1936:   PetscObjectComposeFunction((PetscObject)pc,"PCGetCoarseOperators_C",PCGetCoarseOperators_MG);
    1937:   PetscObjectComposeFunction((PetscObject)pc,"PCMGSetAdaptInterpolation_C",PCMGSetAdaptInterpolation_MG);
    1938:   PetscObjectComposeFunction((PetscObject)pc,"PCMGGetAdaptInterpolation_C",PCMGGetAdaptInterpolation_MG);
    1939:   PetscObjectComposeFunction((PetscObject)pc,"PCMGSetAdaptCR_C",PCMGSetAdaptCR_MG);
    1940:   PetscObjectComposeFunction((PetscObject)pc,"PCMGGetAdaptCR_C",PCMGGetAdaptCR_MG);
    1941:   return(0);
    1942: }