Actual source code: trajmemory.c

      1: #include <petsc/private/tsimpl.h>
      2: #include <petscsys.h>
      3: #if defined(PETSC_HAVE_REVOLVE)
      4: #include <revolve_c.h>

      6: /* Limit Revolve to 32-bits */
      7: #define PETSC_REVOLVE_INT_MAX  2147483647

      9: typedef int PetscRevolveInt;

     11: PETSC_STATIC_INLINE PetscErrorCode PetscRevolveIntCast(PetscInt a,PetscRevolveInt *b)
     12: {
     14: #if defined(PETSC_USE_64BIT_INDICES)
     15:   *b = 0;
     16:   if ((a) > PETSC_REVOLVE_INT_MAX) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Parameter is too large for Revolve, which is restricted to 32 bit integers");
     17: #endif
     18:   *b = (PetscRevolveInt)(a);
     19:   return(0);
     20: }
     21: #endif

     23: PetscLogEvent TSTrajectory_DiskWrite, TSTrajectory_DiskRead;
     24: static PetscErrorCode TSTrajectorySet_Memory(TSTrajectory,TS,PetscInt,PetscReal,Vec);

     26: typedef enum {NONE,TWO_LEVEL_NOREVOLVE,TWO_LEVEL_REVOLVE,TWO_LEVEL_TWO_REVOLVE,REVOLVE_OFFLINE,REVOLVE_ONLINE,REVOLVE_MULTISTAGE} SchedulerType;

     28: typedef struct _StackElement {
     29:   PetscInt  stepnum;
     30:   Vec       X;
     31:   Vec       *Y;
     32:   PetscReal time;
     33:   PetscReal timeprev; /* for no solution_only mode */
     34:   PetscReal timenext; /* for solution_only mode */
     35: } *StackElement;

     37: #if defined(PETSC_HAVE_REVOLVE)
     38: typedef struct _RevolveCTX {
     39:   PetscBool       reverseonestep;
     40:   PetscRevolveInt where;
     41:   PetscRevolveInt snaps_in;
     42:   PetscRevolveInt stepsleft;
     43:   PetscRevolveInt check;
     44:   PetscRevolveInt oldcapo;
     45:   PetscRevolveInt capo;
     46:   PetscRevolveInt fine;
     47:   PetscRevolveInt info;
     48: } RevolveCTX;
     49: #endif

     51: typedef struct _Stack {
     52:   PetscInt      stacksize;
     53:   PetscInt      top;
     54:   StackElement  *container;
     55:   PetscInt      nallocated;
     56:   PetscInt      numY;
     57:   PetscBool     solution_only;
     58:   PetscBool     use_dram;
     59: } Stack;

     61: typedef struct _DiskStack {
     62:   PetscInt  stacksize;
     63:   PetscInt  top;
     64:   PetscInt  *container;
     65: } DiskStack;

     67: typedef struct _TJScheduler {
     68:   SchedulerType stype;
     69: #if defined(PETSC_HAVE_REVOLVE)
     70:   RevolveCTX    *rctx,*rctx2;
     71:   PetscBool     use_online;
     72:   PetscInt      store_stride;
     73: #endif
     74:   PetscBool     recompute;
     75:   PetscBool     skip_trajectory;
     76:   PetscBool     save_stack;
     77:   PetscInt      max_cps_ram;  /* maximum checkpoints in RAM */
     78:   PetscInt      max_cps_disk; /* maximum checkpoints on disk */
     79:   PetscInt      stride;
     80:   PetscInt      total_steps;  /* total number of steps */
     81:   Stack         stack;
     82:   DiskStack     diskstack;
     83:   PetscViewer   viewer;
     84: } TJScheduler;

     86: static PetscErrorCode TurnForwardWithStepsize(TS ts,PetscReal nextstepsize)
     87: {

     91:   /* reverse the direction */
     92:   TSSetTimeStep(ts,nextstepsize);
     93:   return(0);
     94: }

     96: static PetscErrorCode TurnForward(TS ts)
     97: {
     98:   PetscReal      stepsize;

    102:   /* reverse the direction */
    103:   TSGetTimeStep(ts,&stepsize);
    104:   TSSetTimeStep(ts,-stepsize);
    105:   return(0);
    106: }

    108: static PetscErrorCode TurnBackward(TS ts)
    109: {
    110:   PetscReal      stepsize;

    114:   if (!ts->trajectory->adjoint_solve_mode) return(0);
    115:   /* reverse the direction */
    116:   stepsize = ts->ptime_prev-ts->ptime;
    117:   TSSetTimeStep(ts,stepsize);
    118:   return(0);
    119: }

    121: static PetscErrorCode ElementCreate(TS ts,Stack *stack,StackElement *e)
    122: {
    123:   Vec            X;
    124:   Vec            *Y;

    128:   if (stack->top < stack->stacksize-1 && stack->container[stack->top+1]) {
    129:     *e = stack->container[stack->top+1];
    130:     return(0);
    131:   }
    132:   if (stack->use_dram) {
    133:     PetscMallocSetDRAM();
    134:   }
    135:   PetscNew(e);
    136:   TSGetSolution(ts,&X);
    137:   VecDuplicate(X,&(*e)->X);
    138:   if (stack->numY > 0 && !stack->solution_only) {
    139:     TSGetStages(ts,&stack->numY,&Y);
    140:     VecDuplicateVecs(Y[0],stack->numY,&(*e)->Y);
    141:   }
    142:   if (stack->use_dram) {
    143:     PetscMallocResetDRAM();
    144:   }
    145:   stack->nallocated++;
    146:   return(0);
    147: }

    149: static PetscErrorCode ElementSet(TS ts,Stack *stack,StackElement *e,PetscInt stepnum,PetscReal time,Vec X)
    150: {
    151:   Vec            *Y;
    152:   PetscInt       i;
    153:   PetscReal      timeprev;

    157:   VecCopy(X,(*e)->X);
    158:   if (stack->numY > 0 && !stack->solution_only) {
    159:     TSGetStages(ts,&stack->numY,&Y);
    160:     for (i=0;i<stack->numY;i++) {
    161:       VecCopy(Y[i],(*e)->Y[i]);
    162:     }
    163:   }
    164:   (*e)->stepnum = stepnum;
    165:   (*e)->time    = time;
    166:   /* for consistency */
    167:   if (stepnum == 0) {
    168:     (*e)->timeprev = (*e)->time - ts->time_step;
    169:   } else {
    170:     TSGetPrevTime(ts,&timeprev);
    171:     (*e)->timeprev = timeprev;
    172:   }
    173:   return(0);
    174: }

    176: static PetscErrorCode ElementDestroy(Stack *stack,StackElement e)
    177: {

    181:   if (stack->use_dram) {
    182:     PetscMallocSetDRAM();
    183:   }
    184:   VecDestroy(&e->X);
    185:   if (stack->numY > 0 && !stack->solution_only) {
    186:     VecDestroyVecs(stack->numY,&e->Y);
    187:   }
    188:   PetscFree(e);
    189:   if (stack->use_dram) {
    190:     PetscMallocResetDRAM();
    191:   }
    192:   stack->nallocated--;
    193:   return(0);
    194: }

    196: static PetscErrorCode StackResize(Stack *stack,PetscInt newsize)
    197: {
    198:   StackElement   *newcontainer;
    199:   PetscInt       i;

    203:   PetscCalloc1(newsize*sizeof(StackElement),&newcontainer);
    204:   for (i=0;i<stack->stacksize;i++) {
    205:     newcontainer[i] = stack->container[i];
    206:   }
    207:   PetscFree(stack->container);
    208:   stack->container = newcontainer;
    209:   stack->stacksize = newsize;
    210:   return(0);
    211: }

    213: static PetscErrorCode StackPush(Stack *stack,StackElement e)
    214: {
    216:   if (stack->top+1 >= stack->stacksize) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_MEMC,"Maximum stack size (%D) exceeded",stack->stacksize);
    217:   stack->container[++stack->top] = e;
    218:   return(0);
    219: }

    221: static PetscErrorCode StackPop(Stack *stack,StackElement *e)
    222: {
    224:   *e = NULL;
    225:   if (stack->top == -1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEMC,"Empty stack");
    226:   *e = stack->container[stack->top--];
    227:   return(0);
    228: }

    230: static PetscErrorCode StackTop(Stack *stack,StackElement *e)
    231: {
    233:   *e = stack->container[stack->top];
    234:   return(0);
    235: }

    237: static PetscErrorCode StackInit(Stack *stack,PetscInt size,PetscInt ny)
    238: {

    242:   stack->top  = -1;
    243:   stack->numY = ny;

    245:   if (!stack->container) {
    246:     PetscCalloc1(size,&stack->container);
    247:   }
    248:   return(0);
    249: }

    251: static PetscErrorCode StackDestroy(Stack *stack)
    252: {
    253:   PetscInt       i,n = stack->nallocated;

    257:   if (!stack->container) return(0);
    258:   if (stack->top+1 > stack->nallocated) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Stack size does not match element counter %D",stack->nallocated);
    259:   for (i=0; i<n; i++) {
    260:     ElementDestroy(stack,stack->container[i]);
    261:   }
    262:   PetscFree(stack->container);
    263:   return(0);
    264: }

    266: static PetscErrorCode StackFind(Stack *stack,StackElement *e,PetscInt index)
    267: {
    269:   *e = NULL;
    270:   if (index < 0 || index > stack->top) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Invalid index %D",index);
    271:   *e = stack->container[index];
    272:   return(0);
    273: }

    275: static PetscErrorCode WriteToDisk(PetscInt stepnum,PetscReal time,PetscReal timeprev,Vec X,Vec *Y,PetscInt numY,PetscBool solution_only,PetscViewer viewer)
    276: {
    277:   PetscInt       i;

    281:   PetscViewerBinaryWrite(viewer,&stepnum,1,PETSC_INT);
    282:   VecView(X,viewer);
    283:   for (i=0;!solution_only && i<numY;i++) {
    284:     VecView(Y[i],viewer);
    285:   }
    286:   PetscViewerBinaryWrite(viewer,&time,1,PETSC_REAL);
    287:   PetscViewerBinaryWrite(viewer,&timeprev,1,PETSC_REAL);
    288:   return(0);
    289: }

    291: static PetscErrorCode ReadFromDisk(PetscInt *stepnum,PetscReal *time,PetscReal *timeprev,Vec X,Vec *Y,PetscInt numY,PetscBool solution_only,PetscViewer viewer)
    292: {
    293:   PetscInt       i;

    297:   PetscViewerBinaryRead(viewer,stepnum,1,NULL,PETSC_INT);
    298:   VecLoad(X,viewer);
    299:   for (i=0;!solution_only && i<numY;i++) {
    300:     VecLoad(Y[i],viewer);
    301:   }
    302:   PetscViewerBinaryRead(viewer,time,1,NULL,PETSC_REAL);
    303:   PetscViewerBinaryRead(viewer,timeprev,1,NULL,PETSC_REAL);
    304:   return(0);
    305: }

    307: static PetscErrorCode StackDumpAll(TSTrajectory tj,TS ts,Stack *stack,PetscInt id)
    308: {
    309:   Vec            *Y;
    310:   PetscInt       i;
    311:   StackElement   e = NULL;
    312:   TJScheduler    *tjsch = (TJScheduler*)tj->data;
    313:   char           filename[PETSC_MAX_PATH_LEN];
    315:   MPI_Comm       comm;

    318:   PetscObjectGetComm((PetscObject)ts,&comm);
    319:   if (tj->monitor) {
    320:     PetscViewerASCIIPushTab(tj->monitor);
    321:     PetscViewerASCIIPrintf(tj->monitor,"Dump stack id %D to file\n",id);
    322:     PetscViewerASCIIPopTab(tj->monitor);
    323:   }
    324:   PetscSNPrintf(filename,sizeof(filename),"%s/TS-STACK%06d.bin",tj->dirname,id);
    325:   PetscViewerFileSetName(tjsch->viewer,filename);
    326:   PetscViewerSetUp(tjsch->viewer);
    327:   for (i=0;i<stack->stacksize;i++) {
    328:     e = stack->container[i];
    329:     PetscLogEventBegin(TSTrajectory_DiskWrite,tj,ts,0,0);
    330:     WriteToDisk(e->stepnum,e->time,e->timeprev,e->X,e->Y,stack->numY,stack->solution_only,tjsch->viewer);
    331:     PetscLogEventEnd(TSTrajectory_DiskWrite,tj,ts,0,0);
    332:     ts->trajectory->diskwrites++;
    333:   }
    334:   /* save the last step for restart, the last step is in memory when using single level schemes, but not necessarily the case for multi level schemes */
    335:   TSGetStages(ts,&stack->numY,&Y);
    336:   PetscLogEventBegin(TSTrajectory_DiskWrite,tj,ts,0,0);
    337:   WriteToDisk(ts->steps,ts->ptime,ts->ptime_prev,ts->vec_sol,Y,stack->numY,stack->solution_only,tjsch->viewer);
    338:   PetscLogEventEnd(TSTrajectory_DiskWrite,tj,ts,0,0);
    339:   ts->trajectory->diskwrites++;
    340:   for (i=0;i<stack->stacksize;i++) {
    341:     StackPop(stack,&e);
    342:   }
    343:   return(0);
    344: }

    346: static PetscErrorCode StackLoadAll(TSTrajectory tj,TS ts,Stack *stack,PetscInt id)
    347: {
    348:   Vec            *Y;
    349:   PetscInt       i;
    350:   StackElement   e;
    351:   PetscViewer    viewer;
    352:   char           filename[PETSC_MAX_PATH_LEN];

    356:   if (tj->monitor) {
    357:     PetscViewerASCIIAddTab(tj->monitor,((PetscObject)tj)->tablevel);
    358:     PetscViewerASCIIPrintf(tj->monitor,"Load stack from file\n");
    359:     PetscViewerASCIISubtractTab(tj->monitor,((PetscObject)tj)->tablevel);
    360:   }
    361:   PetscSNPrintf(filename,sizeof filename,"%s/TS-STACK%06d.bin",tj->dirname,id);
    362:   PetscViewerBinaryOpen(PetscObjectComm((PetscObject)tj),filename,FILE_MODE_READ,&viewer);
    363:   PetscViewerBinarySetSkipInfo(viewer,PETSC_TRUE);
    364:   PetscViewerPushFormat(viewer,PETSC_VIEWER_NATIVE);
    365:   for (i=0;i<stack->stacksize;i++) {
    366:     ElementCreate(ts,stack,&e);
    367:     StackPush(stack,e);
    368:     PetscLogEventBegin(TSTrajectory_DiskRead,tj,ts,0,0);
    369:     ReadFromDisk(&e->stepnum,&e->time,&e->timeprev,e->X,e->Y,stack->numY,stack->solution_only,viewer);
    370:     PetscLogEventEnd(TSTrajectory_DiskRead,tj,ts,0,0);
    371:     ts->trajectory->diskreads++;
    372:   }
    373:   /* load the last step into TS */
    374:   TSGetStages(ts,&stack->numY,&Y);
    375:   PetscLogEventBegin(TSTrajectory_DiskRead,tj,ts,0,0);
    376:   ReadFromDisk(&ts->steps,&ts->ptime,&ts->ptime_prev,ts->vec_sol,Y,stack->numY,stack->solution_only,viewer);
    377:   PetscLogEventEnd(TSTrajectory_DiskRead,tj,ts,0,0);
    378:   ts->trajectory->diskreads++;
    379:   TurnBackward(ts);
    380:   PetscViewerDestroy(&viewer);
    381:   return(0);
    382: }

    384: #if defined(PETSC_HAVE_REVOLVE)
    385: static PetscErrorCode StackLoadLast(TSTrajectory tj,TS ts,Stack *stack,PetscInt id)
    386: {
    387:   Vec            *Y;
    388:   PetscInt       size;
    389:   PetscViewer    viewer;
    390:   char           filename[PETSC_MAX_PATH_LEN];
    391: #if defined(PETSC_HAVE_MPIIO)
    392:   PetscBool      usempiio;
    393: #endif
    394:   int            fd;
    395:   off_t          off,offset;

    399:   if (tj->monitor) {
    400:     PetscViewerASCIIAddTab(tj->monitor,((PetscObject)tj)->tablevel);
    401:     PetscViewerASCIIPrintf(tj->monitor,"Load last stack element from file\n");
    402:     PetscViewerASCIISubtractTab(tj->monitor,((PetscObject)tj)->tablevel);
    403:   }
    404:   TSGetStages(ts,&stack->numY,&Y);
    405:   VecGetSize(Y[0],&size);
    406:   /* VecView writes to file two extra int's for class id and number of rows */
    407:   off  = -((stack->solution_only?0:stack->numY)+1)*(size*PETSC_BINARY_SCALAR_SIZE+2*PETSC_BINARY_INT_SIZE)-PETSC_BINARY_INT_SIZE-2*PETSC_BINARY_SCALAR_SIZE;

    409:   PetscSNPrintf(filename,sizeof filename,"%s/TS-STACK%06d.bin",tj->dirname,id);
    410:   PetscViewerBinaryOpen(PetscObjectComm((PetscObject)tj),filename,FILE_MODE_READ,&viewer);
    411:   PetscViewerBinarySetSkipInfo(viewer,PETSC_TRUE);
    412:   PetscViewerPushFormat(viewer,PETSC_VIEWER_NATIVE);
    413: #if defined(PETSC_HAVE_MPIIO)
    414:   PetscViewerBinaryGetUseMPIIO(viewer,&usempiio);
    415:   if (usempiio) {
    416:     PetscViewerBinaryGetMPIIODescriptor(viewer,(MPI_File*)&fd);
    417:     PetscBinarySynchronizedSeek(PetscObjectComm((PetscObject)tj),fd,off,PETSC_BINARY_SEEK_END,&offset);
    418:   } else {
    419: #endif
    420:     PetscViewerBinaryGetDescriptor(viewer,&fd);
    421:     PetscBinarySeek(fd,off,PETSC_BINARY_SEEK_END,&offset);
    422: #if defined(PETSC_HAVE_MPIIO)
    423:   }
    424: #endif
    425:   /* load the last step into TS */
    426:   PetscLogEventBegin(TSTrajectory_DiskRead,tj,ts,0,0);
    427:   ReadFromDisk(&ts->steps,&ts->ptime,&ts->ptime_prev,ts->vec_sol,Y,stack->numY,stack->solution_only,viewer);
    428:   PetscLogEventEnd(TSTrajectory_DiskRead,tj,ts,0,0);
    429:   ts->trajectory->diskreads++;
    430:   PetscViewerDestroy(&viewer);
    431:   TurnBackward(ts);
    432:   return(0);
    433: }
    434: #endif

    436: static PetscErrorCode DumpSingle(TSTrajectory tj,TS ts,Stack *stack,PetscInt id)
    437: {
    438:   Vec            *Y;
    439:   PetscInt       stepnum;
    440:   TJScheduler    *tjsch = (TJScheduler*)tj->data;
    441:   char           filename[PETSC_MAX_PATH_LEN];
    443:   MPI_Comm       comm;

    446:   PetscObjectGetComm((PetscObject)ts,&comm);
    447:   if (tj->monitor) {
    448:     PetscViewerASCIIAddTab(tj->monitor,((PetscObject)tj)->tablevel);
    449:     PetscViewerASCIIPrintf(tj->monitor,"Dump a single point from file\n");
    450:     PetscViewerASCIISubtractTab(tj->monitor,((PetscObject)tj)->tablevel);
    451:   }
    452:   TSGetStepNumber(ts,&stepnum);
    453:   PetscSNPrintf(filename,sizeof(filename),"%s/TS-CPS%06d.bin",tj->dirname,id);
    454:   PetscViewerFileSetName(tjsch->viewer,filename);
    455:   PetscViewerSetUp(tjsch->viewer);

    457:   TSGetStages(ts,&stack->numY,&Y);
    458:   PetscLogEventBegin(TSTrajectory_DiskWrite,tj,ts,0,0);
    459:   WriteToDisk(stepnum,ts->ptime,ts->ptime_prev,ts->vec_sol,Y,stack->numY,stack->solution_only,tjsch->viewer);
    460:   PetscLogEventEnd(TSTrajectory_DiskWrite,tj,ts,0,0);
    461:   ts->trajectory->diskwrites++;
    462:   return(0);
    463: }

    465: static PetscErrorCode LoadSingle(TSTrajectory tj,TS ts,Stack *stack,PetscInt id)
    466: {
    467:   Vec            *Y;
    468:   PetscViewer    viewer;
    469:   char           filename[PETSC_MAX_PATH_LEN];

    473:   if (tj->monitor) {
    474:     PetscViewerASCIIAddTab(tj->monitor,((PetscObject)tj)->tablevel);
    475:     PetscViewerASCIIPrintf(tj->monitor,"Load a single point from file\n");
    476:     PetscViewerASCIISubtractTab(tj->monitor,((PetscObject)tj)->tablevel);
    477:   }
    478:   PetscSNPrintf(filename,sizeof filename,"%s/TS-CPS%06d.bin",tj->dirname,id);
    479:   PetscViewerBinaryOpen(PetscObjectComm((PetscObject)tj),filename,FILE_MODE_READ,&viewer);
    480:   PetscViewerBinarySetSkipInfo(viewer,PETSC_TRUE);
    481:   PetscViewerPushFormat(viewer,PETSC_VIEWER_NATIVE);
    482:   TSGetStages(ts,&stack->numY,&Y);
    483:   PetscLogEventBegin(TSTrajectory_DiskRead,tj,ts,0,0);
    484:   ReadFromDisk(&ts->steps,&ts->ptime,&ts->ptime_prev,ts->vec_sol,Y,stack->numY,stack->solution_only,viewer);
    485:   PetscLogEventEnd(TSTrajectory_DiskRead,tj,ts,0,0);
    486:   ts->trajectory->diskreads++;
    487:   PetscViewerDestroy(&viewer);
    488:   return(0);
    489: }

    491: static PetscErrorCode UpdateTS(TS ts,Stack *stack,StackElement e, PetscBool adjoint_mode)
    492: {
    493:   Vec            *Y;
    494:   PetscInt       i;

    498:   VecCopy(e->X,ts->vec_sol);
    499:   if (!stack->solution_only && e->stepnum) {
    500:     TSGetStages(ts,&stack->numY,&Y);
    501:     for (i=0;i<stack->numY;i++) {
    502:       VecCopy(e->Y[i],Y[i]);
    503:     }
    504:   }
    505:   if (adjoint_mode) {
    506:     TSSetTimeStep(ts,e->timeprev-e->time); /* stepsize will be negative */
    507:   } else {
    508:     TSSetTimeStep(ts,e->time-e->timeprev); /* stepsize will be positive */
    509:   }
    510:   ts->ptime      = e->time;
    511:   ts->ptime_prev = e->timeprev;
    512:   return(0);
    513: }

    515: static PetscErrorCode ReCompute(TS ts,TJScheduler *tjsch,PetscInt stepnumbegin,PetscInt stepnumend)
    516: {
    517:   Stack          *stack = &tjsch->stack;
    518:   PetscInt       i;

    522:   tjsch->recompute = PETSC_TRUE; /* hints TSTrajectorySet() that it is in recompute mode */
    523:   TSSetStepNumber(ts,stepnumbegin);/* global step number */
    524:   for (i=stepnumbegin;i<stepnumend;i++) { /* assume fixed step size */
    525:     if (stack->solution_only && !tjsch->skip_trajectory) { /* revolve online need this */
    526:       /* don't use the public interface as it will update the TSHistory: this need a better fix */
    527:       TSTrajectorySet_Memory(ts->trajectory,ts,ts->steps,ts->ptime,ts->vec_sol);
    528:     }
    529:     TSMonitor(ts,ts->steps,ts->ptime,ts->vec_sol);
    530:     TSStep(ts);
    531:     if (!stack->solution_only && !tjsch->skip_trajectory) {
    532:       /* don't use the public interface as it will update the TSHistory: this need a better fix */
    533:       TSTrajectorySet_Memory(ts->trajectory,ts,ts->steps,ts->ptime,ts->vec_sol);
    534:     }
    535:     TSEventHandler(ts);
    536:     if (!ts->steprollback) {
    537:       TSPostStep(ts);
    538:     }
    539:   }
    540:   TurnBackward(ts);
    541:   ts->trajectory->recomps += stepnumend-stepnumbegin; /* recomputation counter */
    542:   TSSetStepNumber(ts,stepnumend);
    543:   tjsch->recompute = PETSC_FALSE; /* reset the flag for recompute mode */
    544:   return(0);
    545: }

    547: static PetscErrorCode TopLevelStore(TSTrajectory tj,TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscInt localstepnum,PetscInt laststridesize,PetscBool *done)
    548: {
    549:   Stack          *stack = &tjsch->stack;
    550:   DiskStack      *diskstack = &tjsch->diskstack;
    551:   PetscInt       stridenum;

    555:   *done = PETSC_FALSE;
    556:   stridenum    = stepnum/tjsch->stride;
    557:   /* make sure saved checkpoint id starts from 1
    558:      skip last stride when using stridenum+1
    559:      skip first stride when using stridenum */
    560:   if (stack->solution_only) {
    561:     if (tjsch->save_stack) {
    562:       if (localstepnum == tjsch->stride-1 && stepnum < tjsch->total_steps-laststridesize) { /* current step will be saved without going through stack */
    563:         StackDumpAll(tj,ts,stack,stridenum+1);
    564:         if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) diskstack->container[++diskstack->top] = stridenum+1;
    565:         *done = PETSC_TRUE;
    566:       }
    567:     } else {
    568:       if (localstepnum == 0 && stepnum < tjsch->total_steps-laststridesize) {
    569:         DumpSingle(tj,ts,stack,stridenum+1);
    570:         if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) diskstack->container[++diskstack->top] = stridenum+1;
    571:         *done = PETSC_TRUE;
    572:       }
    573:     }
    574:   } else {
    575:     if (tjsch->save_stack) {
    576:       if (localstepnum == 0 && stepnum < tjsch->total_steps && stepnum != 0) { /* skip the first stride */
    577:         StackDumpAll(tj,ts,stack,stridenum);
    578:         if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) diskstack->container[++diskstack->top] = stridenum;
    579:         *done = PETSC_TRUE;
    580:       }
    581:     } else {
    582:       if (localstepnum == 1 && stepnum < tjsch->total_steps-laststridesize) {
    583:         DumpSingle(tj,ts,stack,stridenum+1);
    584:         if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) diskstack->container[++diskstack->top] = stridenum+1;
    585:         *done = PETSC_TRUE;
    586:       }
    587:     }
    588:   }
    589:   return(0);
    590: }

    592: static PetscErrorCode SetTrajN(TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
    593: {
    594:   Stack          *stack = &tjsch->stack;
    595:   StackElement   e;

    599:   /* skip the last step */
    600:   if (ts->reason) { /* only affect the forward run */
    601:     /* update total_steps in the end of forward run */
    602:     if (stepnum != tjsch->total_steps) tjsch->total_steps = stepnum;
    603:     if (stack->solution_only) {
    604:       /* get rid of the solution at second last step */
    605:       StackPop(stack,&e);
    606:     }
    607:     return(0);
    608:   }
    609:   /*  do not save trajectory at the recompute stage for solution_only mode */
    610:   if (stack->solution_only && tjsch->recompute) return(0);
    611:   /* skip the first step for no_solution_only mode */
    612:   if (!stack->solution_only && stepnum == 0) return(0);

    614:   /* resize the stack */
    615:   if (stack->top+1 == stack->stacksize) {
    616:     StackResize(stack,2*stack->stacksize);
    617:   }
    618:   /* update timenext for the previous step; necessary for step adaptivity */
    619:   if (stack->top > -1) {
    620:     StackTop(stack,&e);
    621:     e->timenext = ts->ptime;
    622:   }
    623:   if (stepnum < stack->top) {
    624:     SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_MEMC,"Illegal modification of a non-top stack element");
    625:   }
    626:   ElementCreate(ts,stack,&e);
    627:   ElementSet(ts,stack,&e,stepnum,time,X);
    628:   StackPush(stack,e);
    629:   return(0);
    630: }

    632: static PetscErrorCode SetTrajN_2(TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
    633: {
    634:   Stack          *stack = &tjsch->stack;
    635:   StackElement   e;

    639:   if (stack->top+1 == stack->stacksize) {
    640:     StackResize(stack,2*stack->stacksize);
    641:   }
    642:   /* update timenext for the previous step; necessary for step adaptivity */
    643:   if (stack->top > -1) {
    644:     StackTop(stack,&e);
    645:     e->timenext = ts->ptime;
    646:   }
    647:   if (stepnum < stack->top) SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_MEMC,"Illegal modification of a non-top stack element");
    648:   ElementCreate(ts,stack,&e);
    649:   ElementSet(ts,stack,&e,stepnum,time,X);
    650:   StackPush(stack,e);
    651:   return(0);
    652: }

    654: static PetscErrorCode GetTrajN(TS ts,TJScheduler *tjsch,PetscInt stepnum)
    655: {
    656:   Stack          *stack = &tjsch->stack;
    657:   StackElement   e;
    658:   PetscInt       ns;

    662:   /* If TSTrajectoryGet() is called after TSAdjointSolve() converges (e.g. outside the while loop in TSAdjointSolve()), skip getting the checkpoint. */
    663:   if (ts->reason) return(0);
    664:   if (stepnum == tjsch->total_steps) {
    665:     TurnBackward(ts);
    666:     return(0);
    667:   }
    668:   /* restore a checkpoint */
    669:   StackTop(stack,&e);
    670:   UpdateTS(ts,stack,e,PETSC_TRUE);
    671:   TSGetStages(ts,&ns,NULL);
    672:   if (stack->solution_only && ns) { /* recompute one step */
    673:     TurnForwardWithStepsize(ts,e->timenext-e->time);
    674:     ReCompute(ts,tjsch,e->stepnum,stepnum);
    675:   }
    676:   StackPop(stack,&e);
    677:   return(0);
    678: }

    680: static PetscErrorCode GetTrajN_2(TS ts,TJScheduler *tjsch,PetscInt stepnum)
    681: {
    682:   Stack          *stack = &tjsch->stack;
    683:   StackElement   e = NULL;

    687:   StackFind(stack,&e,stepnum);
    688:   if (stepnum != e->stepnum) SETERRQ2(PetscObjectComm((PetscObject)ts),PETSC_ERR_PLIB,"Inconsistent steps! %D != %D",stepnum,e->stepnum);
    689:   UpdateTS(ts,stack,e,PETSC_FALSE);
    690:   return(0);
    691: }

    693: static PetscErrorCode SetTrajTLNR(TSTrajectory tj,TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
    694: {
    695:   Stack          *stack = &tjsch->stack;
    696:   PetscInt       localstepnum,laststridesize;
    697:   StackElement   e;
    698:   PetscBool      done;

    702:   if (!stack->solution_only && stepnum == 0) return(0);
    703:   if (stack->solution_only && stepnum == tjsch->total_steps) return(0);
    704:   if (tjsch->save_stack && tjsch->recompute) return(0);

    706:   localstepnum = stepnum%tjsch->stride;
    707:   /* (stridesize-1) checkpoints are saved in each stride; an extra point is added by StackDumpAll() */
    708:   laststridesize = tjsch->total_steps%tjsch->stride;
    709:   if (!laststridesize) laststridesize = tjsch->stride;

    711:   if (!tjsch->recompute) {
    712:     TopLevelStore(tj,ts,tjsch,stepnum,localstepnum,laststridesize,&done);
    713:     if (!tjsch->save_stack && stepnum < tjsch->total_steps-laststridesize) return(0);
    714:   }
    715:   if (!stack->solution_only && localstepnum == 0) return(0); /* skip last point in each stride at recompute stage or last stride */
    716:   if (stack->solution_only && localstepnum == tjsch->stride-1) return(0); /* skip last step in each stride at recompute stage or last stride */

    718:   ElementCreate(ts,stack,&e);
    719:   ElementSet(ts,stack,&e,stepnum,time,X);
    720:   StackPush(stack,e);
    721:   return(0);
    722: }

    724: static PetscErrorCode GetTrajTLNR(TSTrajectory tj,TS ts,TJScheduler *tjsch,PetscInt stepnum)
    725: {
    726:   Stack          *stack = &tjsch->stack;
    727:   PetscInt       id,localstepnum,laststridesize;
    728:   StackElement   e;

    732:   if (stepnum == tjsch->total_steps) {
    733:     TurnBackward(ts);
    734:     return(0);
    735:   }

    737:   localstepnum = stepnum%tjsch->stride;
    738:   laststridesize = tjsch->total_steps%tjsch->stride;
    739:   if (!laststridesize) laststridesize = tjsch->stride;
    740:   if (stack->solution_only) {
    741:     /* fill stack with info */
    742:     if (localstepnum == 0 && tjsch->total_steps-stepnum >= laststridesize) {
    743:       id = stepnum/tjsch->stride;
    744:       if (tjsch->save_stack) {
    745:         StackLoadAll(tj,ts,stack,id);
    746:         tjsch->skip_trajectory = PETSC_TRUE;
    747:         TurnForward(ts);
    748:         ReCompute(ts,tjsch,id*tjsch->stride-1,id*tjsch->stride);
    749:         tjsch->skip_trajectory = PETSC_FALSE;
    750:       } else {
    751:         LoadSingle(tj,ts,stack,id);
    752:         TurnForward(ts);
    753:         ReCompute(ts,tjsch,(id-1)*tjsch->stride,id*tjsch->stride);
    754:       }
    755:       return(0);
    756:     }
    757:     /* restore a checkpoint */
    758:     StackPop(stack,&e);
    759:     UpdateTS(ts,stack,e,PETSC_TRUE);
    760:     tjsch->skip_trajectory = PETSC_TRUE;
    761:     TurnForward(ts);
    762:     ReCompute(ts,tjsch,e->stepnum,stepnum);
    763:     tjsch->skip_trajectory = PETSC_FALSE;
    764:   } else {
    765:     /* fill stack with info */
    766:     if (localstepnum == 0 && tjsch->total_steps-stepnum >= laststridesize) {
    767:       id = stepnum/tjsch->stride;
    768:       if (tjsch->save_stack) {
    769:         StackLoadAll(tj,ts,stack,id);
    770:       } else {
    771:         LoadSingle(tj,ts,stack,id);
    772:         ElementCreate(ts,stack,&e);
    773:         ElementSet(ts,stack,&e,(id-1)*tjsch->stride+1,ts->ptime,ts->vec_sol);
    774:         StackPush(stack,e);
    775:         TurnForward(ts);
    776:         ReCompute(ts,tjsch,e->stepnum,id*tjsch->stride);
    777:       }
    778:       return(0);
    779:     }
    780:     /* restore a checkpoint */
    781:     StackPop(stack,&e);
    782:     UpdateTS(ts,stack,e,PETSC_TRUE);
    783:   }
    784:   return(0);
    785: }

    787: #if defined(PETSC_HAVE_REVOLVE)
    788: static PetscErrorCode printwhattodo(PetscViewer viewer,PetscRevolveInt whattodo,RevolveCTX *rctx,PetscRevolveInt shift)
    789: {

    793:   if (!viewer) return(0);

    795:   switch(whattodo) {
    796:     case 1:
    797:       PetscViewerASCIIPrintf(viewer,"Advance from %D to %D\n",rctx->oldcapo+shift,rctx->capo+shift);
    798:       break;
    799:     case 2:
    800:       PetscViewerASCIIPrintf(viewer,"Store in checkpoint number %D (located in RAM)\n",rctx->check);
    801:       break;
    802:     case 3:
    803:       PetscViewerASCIIPrintf(viewer,"First turn: Initialize adjoints and reverse first step\n");
    804:       break;
    805:     case 4:
    806:       PetscViewerASCIIPrintf(viewer,"Forward and reverse one step\n");
    807:       break;
    808:     case 5:
    809:       PetscViewerASCIIPrintf(viewer,"Restore in checkpoint number %D (located in RAM)\n",rctx->check);
    810:       break;
    811:     case 7:
    812:       PetscViewerASCIIPrintf(viewer,"Store in checkpoint number %D (located on disk)\n",rctx->check);
    813:       break;
    814:     case 8:
    815:       PetscViewerASCIIPrintf(viewer,"Restore in checkpoint number %D (located on disk)\n",rctx->check);
    816:       break;
    817:     case -1:
    818:       PetscViewerASCIIPrintf(viewer,"Error!");
    819:       break;
    820:   }
    821:   return(0);
    822: }

    824: static PetscErrorCode printwhattodo2(PetscViewer viewer,PetscRevolveInt whattodo,RevolveCTX *rctx,PetscRevolveInt shift)
    825: {

    829:   if (!viewer) return(0);

    831:   switch(whattodo) {
    832:     case 1:
    833:       PetscViewerASCIIPrintf(viewer,"[Top Level] Advance from stride %D to stride %D\n",rctx->oldcapo+shift,rctx->capo+shift);
    834:       break;
    835:     case 2:
    836:       PetscViewerASCIIPrintf(viewer,"[Top Level] Store in checkpoint number %D\n",rctx->check);
    837:       break;
    838:     case 3:
    839:       PetscViewerASCIIPrintf(viewer,"[Top Level] First turn: Initialize adjoints and reverse first stride\n");
    840:       break;
    841:     case 4:
    842:       PetscViewerASCIIPrintf(viewer,"[Top Level] Forward and reverse one stride\n");
    843:       break;
    844:     case 5:
    845:       PetscViewerASCIIPrintf(viewer,"[Top Level] Restore in checkpoint number %D\n",rctx->check);
    846:       break;
    847:     case 7:
    848:       PetscViewerASCIIPrintf(viewer,"[Top Level] Store in top-level checkpoint number %D\n",rctx->check);
    849:       break;
    850:     case 8:
    851:       PetscViewerASCIIPrintf(viewer,"[Top Level] Restore in top-level checkpoint number %D\n",rctx->check);
    852:       break;
    853:     case -1:
    854:       PetscViewerASCIIPrintf(viewer,"[Top Level] Error!");
    855:       break;
    856:   }
    857:   return(0);
    858: }

    860: static PetscErrorCode InitRevolve(PetscInt fine,PetscInt snaps,RevolveCTX *rctx)
    861: {
    862:   PetscRevolveInt rsnaps,rfine;
    863:   PetscErrorCode  ierr;

    866:   PetscRevolveIntCast(snaps,&rsnaps);
    867:   PetscRevolveIntCast(fine,&rfine);
    868:   revolve_reset();
    869:   revolve_create_offline(rfine,rsnaps);
    870:   rctx->snaps_in       = rsnaps;
    871:   rctx->fine           = rfine;
    872:   rctx->check          = 0;
    873:   rctx->capo           = 0;
    874:   rctx->reverseonestep = PETSC_FALSE;
    875:   /* check stepsleft? */
    876:   return(0);
    877: }

    879: static PetscErrorCode FastForwardRevolve(RevolveCTX *rctx)
    880: {
    881:   PetscRevolveInt whattodo;

    884:   whattodo = 0;
    885:   while (whattodo!=3) { /* we have to fast forward revolve to the beginning of the backward sweep due to unfriendly revolve interface */
    886:     whattodo = revolve_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where);
    887:   }
    888:   return(0);
    889: }

    891: static PetscErrorCode ApplyRevolve(PetscViewer viewer,SchedulerType stype,RevolveCTX *rctx,PetscRevolveInt total_steps,PetscRevolveInt stepnum,PetscRevolveInt localstepnum,PetscBool toplevel,PetscInt *store)
    892: {
    893:   PetscErrorCode  ierr;
    894:   PetscRevolveInt shift,whattodo;

    897:   *store = 0;
    898:   if (rctx->stepsleft > 0) { /* advance the solution without checkpointing anything as Revolve requires */
    899:     rctx->stepsleft--;
    900:     return(0);
    901:   }
    902:   /* let Revolve determine what to do next */
    903:   shift         = stepnum-localstepnum;
    904:   rctx->oldcapo = rctx->capo;
    905:   rctx->capo    = localstepnum;

    907:   if (!toplevel) whattodo = revolve_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where);
    908:   else whattodo = revolve2_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where);
    909:   if (stype == REVOLVE_ONLINE && whattodo == 8) whattodo = 5;
    910:   if (stype == REVOLVE_ONLINE && whattodo == 7) whattodo = 2;
    911:   if (!toplevel) {printwhattodo(viewer,whattodo,rctx,shift);}
    912:   else {printwhattodo2(viewer,whattodo,rctx,shift);}
    913:   if (whattodo == -1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Error in the Revolve library");
    914:   if (whattodo == 1) { /* advance some time steps */
    915:     if (stype == REVOLVE_ONLINE && rctx->capo >= total_steps-1) {
    916:       revolve_turn(total_steps,&rctx->capo,&rctx->fine);
    917:       if (!toplevel) {printwhattodo(viewer,whattodo,rctx,shift);}
    918:       else {printwhattodo2(viewer,whattodo,rctx,shift);}
    919:     }
    920:     rctx->stepsleft = rctx->capo-rctx->oldcapo-1;
    921:   }
    922:   if (whattodo == 3 || whattodo == 4) { /* ready for a reverse step */
    923:     rctx->reverseonestep = PETSC_TRUE;
    924:   }
    925:   if (whattodo == 5) { /* restore a checkpoint and ask Revolve what to do next */
    926:     rctx->oldcapo = rctx->capo;
    927:     if (!toplevel) whattodo = revolve_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where); /* must return 1 or 3 or 4*/
    928:     else whattodo = revolve2_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where);
    929:     if (!toplevel) {printwhattodo(viewer,whattodo,rctx,shift);}
    930:     else {printwhattodo2(viewer,whattodo,rctx,shift);}
    931:     if (whattodo == 3 || whattodo == 4) rctx->reverseonestep = PETSC_TRUE;
    932:     if (whattodo == 1) rctx->stepsleft = rctx->capo-rctx->oldcapo;
    933:   }
    934:   if (whattodo == 7) { /* save the checkpoint to disk */
    935:     *store = 2;
    936:     rctx->oldcapo = rctx->capo;
    937:     whattodo = revolve_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where); /* must return 1 */
    938:     printwhattodo(viewer,whattodo,rctx,shift);
    939:     rctx->stepsleft = rctx->capo-rctx->oldcapo-1;
    940:   }
    941:   if (whattodo == 2) { /* store a checkpoint to RAM and ask Revolve how many time steps to advance next */
    942:     *store = 1;
    943:     rctx->oldcapo = rctx->capo;
    944:     if (!toplevel) whattodo = revolve_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where); /* must return 1 */
    945:     else whattodo = revolve2_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where);
    946:     if (!toplevel) {printwhattodo(viewer,whattodo,rctx,shift);}
    947:     else {printwhattodo2(viewer,whattodo,rctx,shift);}
    948:     if (stype == REVOLVE_ONLINE && rctx->capo >= total_steps-1) {
    949:       revolve_turn(total_steps,&rctx->capo,&rctx->fine);
    950:       printwhattodo(viewer,whattodo,rctx,shift);
    951:     }
    952:     rctx->stepsleft = rctx->capo-rctx->oldcapo-1;
    953:   }
    954:   return(0);
    955: }

    957: static PetscErrorCode SetTrajROF(TSTrajectory tj,TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
    958: {
    959:   Stack           *stack = &tjsch->stack;
    960:   PetscInt        store;
    961:   StackElement    e;
    962:   PetscRevolveInt rtotal_steps,rstepnum;
    963:   PetscErrorCode  ierr;

    966:   if (!stack->solution_only && stepnum == 0) return(0);
    967:   if (stack->solution_only && stepnum == tjsch->total_steps) return(0);
    968:   PetscRevolveIntCast(tjsch->total_steps,&rtotal_steps);
    969:   PetscRevolveIntCast(stepnum,&rstepnum);
    970:   ApplyRevolve(tj->monitor,tjsch->stype,tjsch->rctx,rtotal_steps,rstepnum,rstepnum,PETSC_FALSE,&store);
    971:   if (store == 1) {
    972:     if (stepnum < stack->top) SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_MEMC,"Illegal modification of a non-top stack element");
    973:     ElementCreate(ts,stack,&e);
    974:     ElementSet(ts,stack,&e,stepnum,time,X);
    975:     StackPush(stack,e);
    976:   }
    977:   return(0);
    978: }

    980: static PetscErrorCode GetTrajROF(TSTrajectory tj,TS ts,TJScheduler *tjsch,PetscInt stepnum)
    981: {
    982:   Stack           *stack = &tjsch->stack;
    983:   PetscInt        store;
    984:   PetscRevolveInt whattodo,shift,rtotal_steps,rstepnum;
    985:   StackElement    e;
    986:   PetscErrorCode  ierr;

    989:   if (stepnum == 0 || stepnum == tjsch->total_steps) {
    990:     TurnBackward(ts);
    991:     tjsch->rctx->reverseonestep = PETSC_FALSE;
    992:     return(0);
    993:   }
    994:   /* restore a checkpoint */
    995:   StackTop(stack,&e);
    996:   UpdateTS(ts,stack,e,PETSC_TRUE);
    997:   PetscRevolveIntCast(tjsch->total_steps,&rtotal_steps);
    998:   PetscRevolveIntCast(stepnum,&rstepnum);
    999:   if (stack->solution_only) { /* start with restoring a checkpoint */
    1000:     tjsch->rctx->capo = rstepnum;
    1001:     tjsch->rctx->oldcapo = tjsch->rctx->capo;
    1002:     shift = 0;
    1003:     whattodo = revolve_action(&tjsch->rctx->check,&tjsch->rctx->capo,&tjsch->rctx->fine,tjsch->rctx->snaps_in,&tjsch->rctx->info,&tjsch->rctx->where);
    1004:     printwhattodo(tj->monitor,whattodo,tjsch->rctx,shift);
    1005:   } else { /* 2 revolve actions: restore a checkpoint and then advance */
    1006:     ApplyRevolve(tj->monitor,tjsch->stype,tjsch->rctx,rtotal_steps,rstepnum,rstepnum,PETSC_FALSE,&store);
    1007:     if (tj->monitor) {
    1008:       PetscViewerASCIIAddTab(tj->monitor,((PetscObject)tj)->tablevel);
    1009:       PetscViewerASCIIPrintf(tj->monitor,"Skip the step from %D to %D (stage values already checkpointed)\n",tjsch->rctx->oldcapo,tjsch->rctx->oldcapo+1);
    1010:       PetscViewerASCIISubtractTab(tj->monitor,((PetscObject)tj)->tablevel);
    1011:     }
    1012:     if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
    1013:   }
    1014:   if (stack->solution_only || (!stack->solution_only && e->stepnum < stepnum)) {
    1015:     TurnForward(ts);
    1016:     ReCompute(ts,tjsch,e->stepnum,stepnum);
    1017:   }
    1018:   if ((stack->solution_only && e->stepnum+1 == stepnum) || (!stack->solution_only && e->stepnum == stepnum)) {
    1019:     StackPop(stack,&e);
    1020:   }
    1021:   tjsch->rctx->reverseonestep = PETSC_FALSE;
    1022:   return(0);
    1023: }

    1025: static PetscErrorCode SetTrajRON(TSTrajectory tj,TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
    1026: {
    1027:   Stack           *stack = &tjsch->stack;
    1028:   Vec             *Y;
    1029:   PetscInt        i,store;
    1030:   PetscReal       timeprev;
    1031:   StackElement    e;
    1032:   RevolveCTX      *rctx = tjsch->rctx;
    1033:   PetscRevolveInt rtotal_steps,rstepnum;
    1034:   PetscErrorCode  ierr;

    1037:   if (!stack->solution_only && stepnum == 0) return(0);
    1038:   if (stack->solution_only && stepnum == tjsch->total_steps) return(0);
    1039:   PetscRevolveIntCast(tjsch->total_steps,&rtotal_steps);
    1040:   PetscRevolveIntCast(stepnum,&rstepnum);
    1041:   ApplyRevolve(tj->monitor,tjsch->stype,rctx,rtotal_steps,rstepnum,rstepnum,PETSC_FALSE,&store);
    1042:   if (store == 1) {
    1043:     if (rctx->check != stack->top+1) { /* overwrite some non-top checkpoint in the stack */
    1044:       StackFind(stack,&e,rctx->check);
    1045:       VecCopy(X,e->X);
    1046:       if (stack->numY > 0 && !stack->solution_only) {
    1047:         TSGetStages(ts,&stack->numY,&Y);
    1048:         for (i=0;i<stack->numY;i++) {
    1049:           VecCopy(Y[i],e->Y[i]);
    1050:         }
    1051:       }
    1052:       e->stepnum  = stepnum;
    1053:       e->time     = time;
    1054:       TSGetPrevTime(ts,&timeprev);
    1055:       e->timeprev = timeprev;
    1056:     } else {
    1057:       if (stepnum < stack->top) SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_MEMC,"Illegal modification of a non-top stack element");
    1058:       ElementCreate(ts,stack,&e);
    1059:       ElementSet(ts,stack,&e,stepnum,time,X);
    1060:       StackPush(stack,e);
    1061:     }
    1062:   }
    1063:   return(0);
    1064: }

    1066: static PetscErrorCode GetTrajRON(TSTrajectory tj,TS ts,TJScheduler *tjsch,PetscInt stepnum)
    1067: {
    1068:   Stack           *stack = &tjsch->stack;
    1069:   PetscRevolveInt whattodo,shift,rstepnum;
    1070:   StackElement    e;
    1071:   PetscErrorCode  ierr;

    1074:   if (stepnum == 0 || stepnum == tjsch->total_steps) {
    1075:     TurnBackward(ts);
    1076:     tjsch->rctx->reverseonestep = PETSC_FALSE;
    1077:     return(0);
    1078:   }
    1079:   PetscRevolveIntCast(stepnum,&rstepnum);
    1080:   tjsch->rctx->capo = rstepnum;
    1081:   tjsch->rctx->oldcapo = tjsch->rctx->capo;
    1082:   shift = 0;
    1083:   whattodo = revolve_action(&tjsch->rctx->check,&tjsch->rctx->capo,&tjsch->rctx->fine,tjsch->rctx->snaps_in,&tjsch->rctx->info,&tjsch->rctx->where); /* whattodo=restore */
    1084:   if (whattodo == 8) whattodo = 5;
    1085:   printwhattodo(tj->monitor,whattodo,tjsch->rctx,shift);
    1086:   /* restore a checkpoint */
    1087:   StackFind(stack,&e,tjsch->rctx->check);
    1088:   UpdateTS(ts,stack,e,PETSC_TRUE);
    1089:   if (!stack->solution_only) { /* whattodo must be 5 */
    1090:     /* ask Revolve what to do next */
    1091:     tjsch->rctx->oldcapo = tjsch->rctx->capo;
    1092:     whattodo = revolve_action(&tjsch->rctx->check,&tjsch->rctx->capo,&tjsch->rctx->fine,tjsch->rctx->snaps_in,&tjsch->rctx->info,&tjsch->rctx->where); /* must return 1 or 3 or 4*/
    1093:     printwhattodo(tj->monitor,whattodo,tjsch->rctx,shift);
    1094:     if (whattodo == 3 || whattodo == 4) tjsch->rctx->reverseonestep = PETSC_TRUE;
    1095:     if (whattodo == 1) tjsch->rctx->stepsleft = tjsch->rctx->capo-tjsch->rctx->oldcapo;
    1096:     if (tj->monitor) {
    1097:       PetscViewerASCIIAddTab(tj->monitor,((PetscObject)tj)->tablevel);
    1098:       PetscViewerASCIIPrintf(tj->monitor,"Skip the step from %D to %D (stage values already checkpointed)\n",tjsch->rctx->oldcapo,tjsch->rctx->oldcapo+1);
    1099:       PetscViewerASCIISubtractTab(tj->monitor,((PetscObject)tj)->tablevel);
    1100:     }
    1101:     if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
    1102:   }
    1103:   if (stack->solution_only || (!stack->solution_only && e->stepnum < stepnum)) {
    1104:     TurnForward(ts);
    1105:     ReCompute(ts,tjsch,e->stepnum,stepnum);
    1106:   }
    1107:   tjsch->rctx->reverseonestep = PETSC_FALSE;
    1108:   return(0);
    1109: }

    1111: static PetscErrorCode SetTrajTLR(TSTrajectory tj,TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
    1112: {
    1113:   Stack           *stack = &tjsch->stack;
    1114:   PetscInt        store,localstepnum,laststridesize;
    1115:   StackElement    e;
    1116:   PetscBool       done = PETSC_FALSE;
    1117:   PetscRevolveInt rtotal_steps,rstepnum,rlocalstepnum;

    1121:   if (!stack->solution_only && stepnum == 0) return(0);
    1122:   if (stack->solution_only && stepnum == tjsch->total_steps) return(0);

    1124:   localstepnum = stepnum%tjsch->stride;
    1125:   laststridesize = tjsch->total_steps%tjsch->stride;
    1126:   if (!laststridesize) laststridesize = tjsch->stride;

    1128:   if (!tjsch->recompute) {
    1129:     TopLevelStore(tj,ts,tjsch,stepnum,localstepnum,laststridesize,&done);
    1130:     /* revolve is needed for the last stride; different starting points for last stride between solutin_only and !solutin_only */
    1131:     if (!stack->solution_only && !tjsch->save_stack && stepnum <= tjsch->total_steps-laststridesize) return(0);
    1132:     if (stack->solution_only && !tjsch->save_stack && stepnum < tjsch->total_steps-laststridesize) return(0);
    1133:   }
    1134:   if (tjsch->save_stack && done) {
    1135:     InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
    1136:     return(0);
    1137:   }
    1138:   if (laststridesize < tjsch->stride) {
    1139:     if (stack->solution_only && stepnum == tjsch->total_steps-laststridesize && !tjsch->recompute) { /* step tjsch->total_steps-laststridesize-1 is skipped, but the next step is not */
    1140:       InitRevolve(laststridesize,tjsch->max_cps_ram,tjsch->rctx);
    1141:     }
    1142:     if (!stack->solution_only && stepnum == tjsch->total_steps-laststridesize+1 && !tjsch->recompute) { /* step tjsch->total_steps-laststridesize is skipped, but the next step is not */
    1143:       InitRevolve(laststridesize,tjsch->max_cps_ram,tjsch->rctx);
    1144:     }
    1145:   }
    1146:   PetscRevolveIntCast(tjsch->total_steps,&rtotal_steps);
    1147:   PetscRevolveIntCast(stepnum,&rstepnum);
    1148:   PetscRevolveIntCast(localstepnum,&rlocalstepnum);
    1149:   ApplyRevolve(tj->monitor,tjsch->stype,tjsch->rctx,rtotal_steps,rstepnum,rlocalstepnum,PETSC_FALSE,&store);
    1150:   if (store == 1) {
    1151:     if (localstepnum < stack->top) SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_MEMC,"Illegal modification of a non-top stack element");
    1152:     ElementCreate(ts,stack,&e);
    1153:     ElementSet(ts,stack,&e,stepnum,time,X);
    1154:     StackPush(stack,e);
    1155:   }
    1156:   return(0);
    1157: }

    1159: static PetscErrorCode GetTrajTLR(TSTrajectory tj,TS ts,TJScheduler *tjsch,PetscInt stepnum)
    1160: {
    1161:   Stack           *stack = &tjsch->stack;
    1162:   PetscRevolveInt whattodo,shift,rstepnum,rlocalstepnum,rtotal_steps;
    1163:   PetscInt        localstepnum,stridenum,laststridesize,store;
    1164:   StackElement    e;
    1165:   PetscErrorCode  ierr;

    1168:   localstepnum = stepnum%tjsch->stride;
    1169:   stridenum    = stepnum/tjsch->stride;
    1170:   if (stepnum == tjsch->total_steps) {
    1171:     TurnBackward(ts);
    1172:     tjsch->rctx->reverseonestep = PETSC_FALSE;
    1173:     return(0);
    1174:   }
    1175:   laststridesize = tjsch->total_steps%tjsch->stride;
    1176:   if (!laststridesize) laststridesize = tjsch->stride;
    1177:   PetscRevolveIntCast(tjsch->total_steps,&rtotal_steps);
    1178:   PetscRevolveIntCast(stepnum,&rstepnum);
    1179:   PetscRevolveIntCast(localstepnum,&rlocalstepnum);
    1180:   if (stack->solution_only) {
    1181:     /* fill stack */
    1182:     if (localstepnum == 0 && stepnum <= tjsch->total_steps-laststridesize) {
    1183:       if (tjsch->save_stack) {
    1184:         StackLoadAll(tj,ts,stack,stridenum);
    1185:         InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
    1186:         FastForwardRevolve(tjsch->rctx);
    1187:         tjsch->skip_trajectory = PETSC_TRUE;
    1188:         TurnForward(ts);
    1189:         ReCompute(ts,tjsch,stridenum*tjsch->stride-1,stridenum*tjsch->stride);
    1190:         tjsch->skip_trajectory = PETSC_FALSE;
    1191:       } else {
    1192:         LoadSingle(tj,ts,stack,stridenum);
    1193:         InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
    1194:         TurnForward(ts);
    1195:         ReCompute(ts,tjsch,(stridenum-1)*tjsch->stride,stridenum*tjsch->stride);
    1196:       }
    1197:       return(0);
    1198:     }
    1199:     /* restore a checkpoint */
    1200:     StackTop(stack,&e);
    1201:     UpdateTS(ts,stack,e,PETSC_TRUE);
    1202:     /* start with restoring a checkpoint */
    1203:     tjsch->rctx->capo = rstepnum;
    1204:     tjsch->rctx->oldcapo = tjsch->rctx->capo;
    1205:     shift = rstepnum-rlocalstepnum;
    1206:     whattodo = revolve_action(&tjsch->rctx->check,&tjsch->rctx->capo,&tjsch->rctx->fine,tjsch->rctx->snaps_in,&tjsch->rctx->info,&tjsch->rctx->where);
    1207:     printwhattodo(tj->monitor,whattodo,tjsch->rctx,shift);
    1208:     TurnForward(ts);
    1209:     ReCompute(ts,tjsch,e->stepnum,stepnum);
    1210:     if (e->stepnum+1 == stepnum) {
    1211:       StackPop(stack,&e);
    1212:     }
    1213:   } else {
    1214:     /* fill stack with info */
    1215:     if (localstepnum == 0 && tjsch->total_steps-stepnum >= laststridesize) {
    1216:       if (tjsch->save_stack) {
    1217:         StackLoadAll(tj,ts,stack,stridenum);
    1218:         InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
    1219:         FastForwardRevolve(tjsch->rctx);
    1220:       } else {
    1221:         PetscRevolveInt rnum;
    1222:         LoadSingle(tj,ts,stack,stridenum);
    1223:         InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
    1224:         PetscRevolveIntCast((stridenum-1)*tjsch->stride+1,&rnum);
    1225:         ApplyRevolve(tj->monitor,tjsch->stype,tjsch->rctx,rtotal_steps,rnum,1,PETSC_FALSE,&store);
    1226:         if (tj->monitor) {
    1227:           PetscViewerASCIIAddTab(tj->monitor,((PetscObject)tj)->tablevel);
    1228:           PetscViewerASCIIPrintf(tj->monitor,"Skip the step from %D to %D (stage values already checkpointed)\n",(stridenum-1)*tjsch->stride+tjsch->rctx->oldcapo,(stridenum-1)*tjsch->stride+tjsch->rctx->oldcapo+1);
    1229:           PetscViewerASCIISubtractTab(tj->monitor,((PetscObject)tj)->tablevel);
    1230:         }
    1231:         ElementCreate(ts,stack,&e);
    1232:         ElementSet(ts,stack,&e,(stridenum-1)*tjsch->stride+1,ts->ptime,ts->vec_sol);
    1233:         StackPush(stack,e);
    1234:         TurnForward(ts);
    1235:         ReCompute(ts,tjsch,e->stepnum,stridenum*tjsch->stride);
    1236:       }
    1237:       return(0);
    1238:     }
    1239:     /* restore a checkpoint */
    1240:     StackTop(stack,&e);
    1241:     UpdateTS(ts,stack,e,PETSC_TRUE);
    1242:     /* 2 revolve actions: restore a checkpoint and then advance */
    1243:     ApplyRevolve(tj->monitor,tjsch->stype,tjsch->rctx,rtotal_steps,rstepnum,rlocalstepnum,PETSC_FALSE,&store);
    1244:     if (tj->monitor) {
    1245:       PetscViewerASCIIAddTab(tj->monitor,((PetscObject)tj)->tablevel);
    1246:       PetscViewerASCIIPrintf(tj->monitor,"Skip the step from %D to %D (stage values already checkpointed)\n",stepnum-localstepnum+tjsch->rctx->oldcapo,stepnum-localstepnum+tjsch->rctx->oldcapo+1);
    1247:       PetscViewerASCIISubtractTab(tj->monitor,((PetscObject)tj)->tablevel);
    1248:     }
    1249:     if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
    1250:     if (e->stepnum < stepnum) {
    1251:       TurnForward(ts);
    1252:       ReCompute(ts,tjsch,e->stepnum,stepnum);
    1253:     }
    1254:     if (e->stepnum == stepnum) {
    1255:       StackPop(stack,&e);
    1256:     }
    1257:   }
    1258:   tjsch->rctx->reverseonestep = PETSC_FALSE;
    1259:   return(0);
    1260: }

    1262: static PetscErrorCode SetTrajTLTR(TSTrajectory tj,TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
    1263: {
    1264:   Stack           *stack = &tjsch->stack;
    1265:   PetscInt        store,localstepnum,stridenum,laststridesize;
    1266:   StackElement    e;
    1267:   PetscBool       done = PETSC_FALSE;
    1268:   PetscRevolveInt rlocalstepnum,rstepnum,rtotal_steps;
    1269:   PetscErrorCode  ierr;

    1272:   if (!stack->solution_only && stepnum == 0) return(0);
    1273:   if (stack->solution_only && stepnum == tjsch->total_steps) return(0);

    1275:   localstepnum = stepnum%tjsch->stride; /* index at the bottom level (inside a stride) */
    1276:   stridenum    = stepnum/tjsch->stride; /* index at the top level */
    1277:   laststridesize = tjsch->total_steps%tjsch->stride;
    1278:   if (!laststridesize) laststridesize = tjsch->stride;
    1279:   if (stack->solution_only && localstepnum == 0 && !tjsch->rctx2->reverseonestep) {
    1280:     PetscRevolveIntCast((tjsch->total_steps+tjsch->stride-1)/tjsch->stride,&rtotal_steps);
    1281:     PetscRevolveIntCast(stridenum,&rstepnum);
    1282:     ApplyRevolve(tj->monitor,tjsch->stype,tjsch->rctx2,rtotal_steps,rstepnum,rstepnum,PETSC_TRUE,&tjsch->store_stride);
    1283:     if (laststridesize < tjsch->stride && stepnum == tjsch->total_steps-laststridesize) {
    1284:       InitRevolve(laststridesize,tjsch->max_cps_ram,tjsch->rctx);
    1285:     }
    1286:   }
    1287:   if (!stack->solution_only && localstepnum == 1 && !tjsch->rctx2->reverseonestep) {
    1288:     PetscRevolveIntCast((tjsch->total_steps+tjsch->stride-1)/tjsch->stride,&rtotal_steps);
    1289:     PetscRevolveIntCast(stridenum,&rstepnum);
    1290:     ApplyRevolve(tj->monitor,tjsch->stype,tjsch->rctx2,rtotal_steps,rstepnum,rstepnum,PETSC_TRUE,&tjsch->store_stride);
    1291:     if (laststridesize < tjsch->stride && stepnum == tjsch->total_steps-laststridesize+1) {
    1292:       InitRevolve(laststridesize,tjsch->max_cps_ram,tjsch->rctx);
    1293:     }
    1294:   }
    1295:   if (tjsch->store_stride) {
    1296:     TopLevelStore(tj,ts,tjsch,stepnum,localstepnum,laststridesize,&done);
    1297:     if (done) {
    1298:       InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
    1299:       return(0);
    1300:     }
    1301:   }
    1302:   if (stepnum < tjsch->total_steps-laststridesize) {
    1303:     if (tjsch->save_stack && !tjsch->store_stride && !tjsch->rctx2->reverseonestep) return(0); /* store or forward-and-reverse at top level trigger revolve at bottom level */
    1304:     if (!tjsch->save_stack && !tjsch->rctx2->reverseonestep) return(0); /* store operation does not require revolve be called at bottom level */
    1305:   }
    1306:   /* Skipping stepnum=0 for !stack->only is enough for TLR, but not for TLTR. Here we skip the first step for each stride so that the top-level revolve is applied (always at localstepnum=1) ahead of the bottom-level revolve */
    1307:   if (!stack->solution_only && localstepnum == 0 && stepnum != tjsch->total_steps && !tjsch->recompute) return(0);
    1308:   PetscRevolveIntCast(tjsch->total_steps,&rtotal_steps);
    1309:   PetscRevolveIntCast(stepnum,&rstepnum);
    1310:   PetscRevolveIntCast(localstepnum,&rlocalstepnum);
    1311:   ApplyRevolve(tj->monitor,tjsch->stype,tjsch->rctx,rtotal_steps,rstepnum,rlocalstepnum,PETSC_FALSE,&store);
    1312:   if (store == 1) {
    1313:     if (localstepnum < stack->top) SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_MEMC,"Illegal modification of a non-top stack element");
    1314:     ElementCreate(ts,stack,&e);
    1315:     ElementSet(ts,stack,&e,stepnum,time,X);
    1316:     StackPush(stack,e);
    1317:   }
    1318:   return(0);
    1319: }

    1321: static PetscErrorCode GetTrajTLTR(TSTrajectory tj,TS ts,TJScheduler *tjsch,PetscInt stepnum)
    1322: {
    1323:   Stack           *stack = &tjsch->stack;
    1324:   DiskStack       *diskstack = &tjsch->diskstack;
    1325:   PetscInt        localstepnum,stridenum,restoredstridenum,laststridesize,store;
    1326:   StackElement    e;
    1327:   PetscRevolveInt whattodo,shift;
    1328:   PetscRevolveInt rtotal_steps,rstepnum,rlocalstepnum;
    1329:   PetscErrorCode  ierr;

    1332:   localstepnum = stepnum%tjsch->stride;
    1333:   stridenum    = stepnum/tjsch->stride;
    1334:   if (stepnum == tjsch->total_steps) {
    1335:     TurnBackward(ts);
    1336:     tjsch->rctx->reverseonestep = PETSC_FALSE;
    1337:     return(0);
    1338:   }
    1339:   laststridesize = tjsch->total_steps%tjsch->stride;
    1340:   if (!laststridesize) laststridesize = tjsch->stride;
    1341:   /*
    1342:    Last stride can be adjoined directly. All the other strides require that the stack in memory be ready before an adjoint step is taken (at the end of each stride). The following two cases need to be addressed differently:
    1343:      Case 1 (save_stack)
    1344:        Restore a disk checkpoint; update TS with the last element in the restored data; recompute to the current point.
    1345:      Case 2 (!save_stack)
    1346:        Restore a disk checkpoint; update TS with the restored point; recompute to the current point.
    1347:   */
    1348:   if (localstepnum == 0 && stepnum <= tjsch->total_steps-laststridesize) {
    1349:     /* restore the top element in the stack for disk checkpoints */
    1350:     restoredstridenum = diskstack->container[diskstack->top];
    1351:     tjsch->rctx2->reverseonestep = PETSC_FALSE;
    1352:     /* top-level revolve must be applied before current step, just like the solution_only mode for single-level revolve */
    1353:     if (!tjsch->save_stack && stack->solution_only) { /* start with restoring a checkpoint */
    1354:       PetscRevolveIntCast(stridenum,&rstepnum);
    1355:       tjsch->rctx2->capo = rstepnum;
    1356:       tjsch->rctx2->oldcapo = tjsch->rctx2->capo;
    1357:       shift = 0;
    1358:       whattodo = revolve2_action(&tjsch->rctx2->check,&tjsch->rctx2->capo,&tjsch->rctx2->fine,tjsch->rctx2->snaps_in,&tjsch->rctx2->info,&tjsch->rctx2->where);
    1359:       printwhattodo2(tj->monitor,whattodo,tjsch->rctx2,shift);
    1360:     } else { /* 2 revolve actions: restore a checkpoint and then advance */
    1361:       PetscRevolveIntCast((tjsch->total_steps+tjsch->stride-1)/tjsch->stride,&rtotal_steps);
    1362:       PetscRevolveIntCast(stridenum,&rstepnum);
    1363:       ApplyRevolve(tj->monitor,tjsch->stype,tjsch->rctx2,rtotal_steps,rstepnum,rstepnum,PETSC_TRUE,&tjsch->store_stride);
    1364:       if (tj->monitor) {
    1365:         PetscViewerASCIIAddTab(tj->monitor,((PetscObject)tj)->tablevel);
    1366:         PetscViewerASCIIPrintf(tj->monitor,"[Top Level] Skip the stride from %D to %D (stage values already checkpointed)\n",tjsch->rctx2->oldcapo,tjsch->rctx2->oldcapo+1);
    1367:         PetscViewerASCIISubtractTab(tj->monitor,((PetscObject)tj)->tablevel);
    1368:       }
    1369:       if (!tjsch->rctx2->reverseonestep && tjsch->rctx2->stepsleft > 0) tjsch->rctx2->stepsleft--;
    1370:     }
    1371:     /* fill stack */
    1372:     if (stack->solution_only) {
    1373:       if (tjsch->save_stack) {
    1374:         if (restoredstridenum < stridenum) {
    1375:           StackLoadLast(tj,ts,stack,restoredstridenum);
    1376:         } else {
    1377:           StackLoadAll(tj,ts,stack,restoredstridenum);
    1378:         }
    1379:         /* recompute one step ahead */
    1380:         tjsch->skip_trajectory = PETSC_TRUE;
    1381:         TurnForward(ts);
    1382:         ReCompute(ts,tjsch,stridenum*tjsch->stride-1,stridenum*tjsch->stride);
    1383:         tjsch->skip_trajectory = PETSC_FALSE;
    1384:         if (restoredstridenum < stridenum) {
    1385:           InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
    1386:           TurnForward(ts);
    1387:           ReCompute(ts,tjsch,restoredstridenum*tjsch->stride,stepnum);
    1388:         } else { /* stack ready, fast forward revolve status */
    1389:           InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
    1390:           FastForwardRevolve(tjsch->rctx);
    1391:         }
    1392:       } else {
    1393:         LoadSingle(tj,ts,stack,restoredstridenum);
    1394:         InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
    1395:         TurnForward(ts);
    1396:         ReCompute(ts,tjsch,(restoredstridenum-1)*tjsch->stride,stepnum);
    1397:       }
    1398:     } else {
    1399:       if (tjsch->save_stack) {
    1400:         if (restoredstridenum < stridenum) {
    1401:           StackLoadLast(tj,ts,stack,restoredstridenum);
    1402:           /* reset revolve */
    1403:           InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
    1404:           TurnForward(ts);
    1405:           ReCompute(ts,tjsch,restoredstridenum*tjsch->stride,stepnum);
    1406:         } else { /* stack ready, fast forward revolve status */
    1407:           StackLoadAll(tj,ts,stack,restoredstridenum);
    1408:           InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
    1409:           FastForwardRevolve(tjsch->rctx);
    1410:         }
    1411:       } else {
    1412:         LoadSingle(tj,ts,stack,restoredstridenum);
    1413:         InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
    1414:         /* push first element to stack */
    1415:         if (tjsch->store_stride || tjsch->rctx2->reverseonestep) {
    1416:           PetscRevolveIntCast(tjsch->total_steps,&rtotal_steps);
    1417:           PetscRevolveIntCast((restoredstridenum-1)*tjsch->stride+1,&rstepnum);
    1418:           ApplyRevolve(tj->monitor,tjsch->stype,tjsch->rctx,rtotal_steps,rstepnum,1,PETSC_FALSE,&store);
    1419:           if (tj->monitor) {
    1420:             PetscViewerASCIIAddTab(tj->monitor,((PetscObject)tj)->tablevel);
    1421:             PetscViewerASCIIPrintf(tj->monitor,"Skip the step from %D to %D (stage values already checkpointed)\n",(restoredstridenum-1)*tjsch->stride,(restoredstridenum-1)*tjsch->stride+1);
    1422:             PetscViewerASCIISubtractTab(tj->monitor,((PetscObject)tj)->tablevel);
    1423:           }
    1424:           ElementCreate(ts,stack,&e);
    1425:           ElementSet(ts,stack,&e,(restoredstridenum-1)*tjsch->stride+1,ts->ptime,ts->vec_sol);
    1426:           StackPush(stack,e);
    1427:         }
    1428:         TurnForward(ts);
    1429:         ReCompute(ts,tjsch,(restoredstridenum-1)*tjsch->stride+1,stepnum);
    1430:       }
    1431:     }
    1432:     if (restoredstridenum == stridenum) diskstack->top--;
    1433:     tjsch->rctx->reverseonestep = PETSC_FALSE;
    1434:     return(0);
    1435:   }

    1437:   if (stack->solution_only) {
    1438:     /* restore a checkpoint */
    1439:     StackTop(stack,&e);
    1440:     UpdateTS(ts,stack,e,PETSC_TRUE);
    1441:     /* start with restoring a checkpoint */
    1442:     PetscRevolveIntCast(stepnum,&rstepnum);
    1443:     PetscRevolveIntCast(localstepnum,&rlocalstepnum);
    1444:     tjsch->rctx->capo = rstepnum;
    1445:     tjsch->rctx->oldcapo = tjsch->rctx->capo;
    1446:     shift = rstepnum-rlocalstepnum;
    1447:     whattodo = revolve_action(&tjsch->rctx->check,&tjsch->rctx->capo,&tjsch->rctx->fine,tjsch->rctx->snaps_in,&tjsch->rctx->info,&tjsch->rctx->where);
    1448:     printwhattodo(tj->monitor,whattodo,tjsch->rctx,shift);
    1449:     TurnForward(ts);
    1450:     ReCompute(ts,tjsch,e->stepnum,stepnum);
    1451:     if (e->stepnum+1 == stepnum) {
    1452:       StackPop(stack,&e);
    1453:     }
    1454:   } else {
    1455:     PetscRevolveInt rlocalstepnum;
    1456:     /* restore a checkpoint */
    1457:     StackTop(stack,&e);
    1458:     UpdateTS(ts,stack,e,PETSC_TRUE);
    1459:     /* 2 revolve actions: restore a checkpoint and then advance */
    1460:     PetscRevolveIntCast(tjsch->total_steps,&rtotal_steps);
    1461:     PetscRevolveIntCast(stridenum,&rstepnum);
    1462:     PetscRevolveIntCast(localstepnum,&rlocalstepnum);
    1463:     ApplyRevolve(tj->monitor,tjsch->stype,tjsch->rctx,rtotal_steps,rstepnum,rlocalstepnum,PETSC_FALSE,&store);
    1464:     if (tj->monitor) {
    1465:       PetscViewerASCIIAddTab(tj->monitor,((PetscObject)tj)->tablevel);
    1466:       PetscViewerASCIIPrintf(tj->monitor,"Skip the step from %D to %D (stage values already checkpointed)\n",stepnum-localstepnum+tjsch->rctx->oldcapo,stepnum-localstepnum+tjsch->rctx->oldcapo+1);
    1467:       PetscViewerASCIISubtractTab(tj->monitor,((PetscObject)tj)->tablevel);
    1468:     }
    1469:     if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
    1470:     if (e->stepnum < stepnum) {
    1471:       TurnForward(ts);
    1472:       ReCompute(ts,tjsch,e->stepnum,stepnum);
    1473:     }
    1474:     if (e->stepnum == stepnum) {
    1475:       StackPop(stack,&e);
    1476:     }
    1477:   }
    1478:   tjsch->rctx->reverseonestep = PETSC_FALSE;
    1479:   return(0);
    1480: }

    1482: static PetscErrorCode SetTrajRMS(TSTrajectory tj,TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
    1483: {
    1484:   Stack           *stack = &tjsch->stack;
    1485:   PetscInt        store;
    1486:   StackElement    e;
    1487:   PetscRevolveInt rtotal_steps,rstepnum;
    1488:   PetscErrorCode  ierr;

    1491:   if (!stack->solution_only && stepnum == 0) return(0);
    1492:   if (stack->solution_only && stepnum == tjsch->total_steps) return(0);
    1493:   PetscRevolveIntCast(tjsch->total_steps,&rtotal_steps);
    1494:   PetscRevolveIntCast(stepnum,&rstepnum);
    1495:   ApplyRevolve(tj->monitor,tjsch->stype,tjsch->rctx,rtotal_steps,rstepnum,rstepnum,PETSC_FALSE,&store);
    1496:   if (store == 1){
    1497:     if (stepnum < stack->top) SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_MEMC,"Illegal modification of a non-top stack element");
    1498:     ElementCreate(ts,stack,&e);
    1499:     ElementSet(ts,stack,&e,stepnum,time,X);
    1500:     StackPush(stack,e);
    1501:   } else if (store == 2) {
    1502:     DumpSingle(tj,ts,stack,tjsch->rctx->check+1);
    1503:   }
    1504:   return(0);
    1505: }

    1507: static PetscErrorCode GetTrajRMS(TSTrajectory tj,TS ts,TJScheduler *tjsch,PetscInt stepnum)
    1508: {
    1509:   Stack           *stack = &tjsch->stack;
    1510:   PetscRevolveInt whattodo,shift,rstepnum;
    1511:   PetscInt        restart;
    1512:   PetscBool       ondisk;
    1513:   StackElement    e;
    1514:   PetscErrorCode  ierr;

    1517:   if (stepnum == 0 || stepnum == tjsch->total_steps) {
    1518:     TurnBackward(ts);
    1519:     tjsch->rctx->reverseonestep = PETSC_FALSE;
    1520:     return(0);
    1521:   }
    1522:   PetscRevolveIntCast(stepnum,&rstepnum);
    1523:   tjsch->rctx->capo = rstepnum;
    1524:   tjsch->rctx->oldcapo = tjsch->rctx->capo;
    1525:   shift = 0;
    1526:   whattodo = revolve_action(&tjsch->rctx->check,&tjsch->rctx->capo,&tjsch->rctx->fine,tjsch->rctx->snaps_in,&tjsch->rctx->info,&tjsch->rctx->where); /* whattodo=restore */
    1527:   printwhattodo(tj->monitor,whattodo,tjsch->rctx,shift);
    1528:   /* restore a checkpoint */
    1529:   restart = tjsch->rctx->capo;
    1530:   if (!tjsch->rctx->where) {
    1531:     ondisk = PETSC_TRUE;
    1532:     LoadSingle(tj,ts,stack,tjsch->rctx->check+1);
    1533:     TurnBackward(ts);
    1534:   } else {
    1535:     ondisk = PETSC_FALSE;
    1536:     StackTop(stack,&e);
    1537:     UpdateTS(ts,stack,e,PETSC_TRUE);
    1538:   }
    1539:   if (!stack->solution_only) { /* whattodo must be 5 or 8 */
    1540:     /* ask Revolve what to do next */
    1541:     tjsch->rctx->oldcapo = tjsch->rctx->capo;
    1542:     whattodo = revolve_action(&tjsch->rctx->check,&tjsch->rctx->capo,&tjsch->rctx->fine,tjsch->rctx->snaps_in,&tjsch->rctx->info,&tjsch->rctx->where); /* must return 1 or 3 or 4*/
    1543:     printwhattodo(tj->monitor,whattodo,tjsch->rctx,shift);
    1544:     if (whattodo == 3 || whattodo == 4) tjsch->rctx->reverseonestep = PETSC_TRUE;
    1545:     if (whattodo == 1) tjsch->rctx->stepsleft = tjsch->rctx->capo-tjsch->rctx->oldcapo;
    1546:     if (tj->monitor) {
    1547:       PetscViewerASCIIAddTab(tj->monitor,((PetscObject)tj)->tablevel);
    1548:       PetscViewerASCIIPrintf(tj->monitor,"Skip the step from %D to %D (stage values already checkpointed)\n",tjsch->rctx->oldcapo,tjsch->rctx->oldcapo+1);
    1549:       PetscViewerASCIISubtractTab(tj->monitor,((PetscObject)tj)->tablevel);
    1550:     }
    1551:     if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
    1552:     restart++; /* skip one step */
    1553:   }
    1554:   if (stack->solution_only || (!stack->solution_only && restart < stepnum)) {
    1555:     TurnForward(ts);
    1556:     ReCompute(ts,tjsch,restart,stepnum);
    1557:   }
    1558:   if (!ondisk && ( (stack->solution_only && e->stepnum+1 == stepnum) || (!stack->solution_only && e->stepnum == stepnum))) {
    1559:     StackPop(stack,&e);
    1560:   }
    1561:   tjsch->rctx->reverseonestep = PETSC_FALSE;
    1562:   return(0);
    1563: }
    1564: #endif

    1566: static PetscErrorCode TSTrajectorySet_Memory(TSTrajectory tj,TS ts,PetscInt stepnum,PetscReal time,Vec X)
    1567: {
    1568:   TJScheduler *tjsch = (TJScheduler*)tj->data;

    1572:   if (!tjsch->recompute) { /* use global stepnum in the forward sweep */
    1573:     TSGetStepNumber(ts,&stepnum);
    1574:   }
    1575:   /* for consistency */
    1576:   if (!tjsch->recompute && stepnum == 0) ts->ptime_prev = ts->ptime-ts->time_step;
    1577:   switch (tjsch->stype) {
    1578:     case NONE:
    1579:       if (tj->adjoint_solve_mode) {
    1580:         SetTrajN(ts,tjsch,stepnum,time,X);
    1581:       } else {
    1582:         SetTrajN_2(ts,tjsch,stepnum,time,X);
    1583:       }
    1584:       break;
    1585:     case TWO_LEVEL_NOREVOLVE:
    1586:       if (!tj->adjoint_solve_mode) SETERRQ(PetscObjectComm((PetscObject)tj),PETSC_ERR_SUP,"Not implemented");
    1587:       SetTrajTLNR(tj,ts,tjsch,stepnum,time,X);
    1588:       break;
    1589: #if defined(PETSC_HAVE_REVOLVE)
    1590:     case TWO_LEVEL_REVOLVE:
    1591:       if (!tj->adjoint_solve_mode) SETERRQ(PetscObjectComm((PetscObject)tj),PETSC_ERR_SUP,"Not implemented");
    1592:       SetTrajTLR(tj,ts,tjsch,stepnum,time,X);
    1593:       break;
    1594:     case TWO_LEVEL_TWO_REVOLVE:
    1595:       if (!tj->adjoint_solve_mode) SETERRQ(PetscObjectComm((PetscObject)tj),PETSC_ERR_SUP,"Not implemented");
    1596:       SetTrajTLTR(tj,ts,tjsch,stepnum,time,X);
    1597:       break;
    1598:     case REVOLVE_OFFLINE:
    1599:       if (!tj->adjoint_solve_mode) SETERRQ(PetscObjectComm((PetscObject)tj),PETSC_ERR_SUP,"Not implemented");
    1600:       SetTrajROF(tj,ts,tjsch,stepnum,time,X);
    1601:       break;
    1602:     case REVOLVE_ONLINE:
    1603:       if (!tj->adjoint_solve_mode) SETERRQ(PetscObjectComm((PetscObject)tj),PETSC_ERR_SUP,"Not implemented");
    1604:       SetTrajRON(tj,ts,tjsch,stepnum,time,X);
    1605:       break;
    1606:     case REVOLVE_MULTISTAGE:
    1607:       if (!tj->adjoint_solve_mode) SETERRQ(PetscObjectComm((PetscObject)tj),PETSC_ERR_SUP,"Not implemented");
    1608:       SetTrajRMS(tj,ts,tjsch,stepnum,time,X);
    1609:       break;
    1610: #endif
    1611:     default:
    1612:       break;
    1613:   }
    1614:   return(0);
    1615: }

    1617: static PetscErrorCode TSTrajectoryGet_Memory(TSTrajectory tj,TS ts,PetscInt stepnum,PetscReal *t)
    1618: {
    1619:   TJScheduler *tjsch = (TJScheduler*)tj->data;

    1623:   if (tj->adjoint_solve_mode && stepnum == 0) {
    1624:     TSTrajectoryReset(tj); /* reset TSTrajectory so users do not need to reset TSTrajectory */
    1625:     return(0);
    1626:   }
    1627:   switch (tjsch->stype) {
    1628:     case NONE:
    1629:       if (tj->adjoint_solve_mode) {
    1630:         GetTrajN(ts,tjsch,stepnum);
    1631:       } else {
    1632:         GetTrajN_2(ts,tjsch,stepnum);
    1633:       }
    1634:       break;
    1635:     case TWO_LEVEL_NOREVOLVE:
    1636:       if (!tj->adjoint_solve_mode) SETERRQ(PetscObjectComm((PetscObject)tj),PETSC_ERR_SUP,"Not implemented");
    1637:       GetTrajTLNR(tj,ts,tjsch,stepnum);
    1638:       break;
    1639: #if defined(PETSC_HAVE_REVOLVE)
    1640:     case TWO_LEVEL_REVOLVE:
    1641:       if (!tj->adjoint_solve_mode) SETERRQ(PetscObjectComm((PetscObject)tj),PETSC_ERR_SUP,"Not implemented");
    1642:       GetTrajTLR(tj,ts,tjsch,stepnum);
    1643:       break;
    1644:     case TWO_LEVEL_TWO_REVOLVE:
    1645:       if (!tj->adjoint_solve_mode) SETERRQ(PetscObjectComm((PetscObject)tj),PETSC_ERR_SUP,"Not implemented");
    1646:       GetTrajTLTR(tj,ts,tjsch,stepnum);
    1647:       break;
    1648:     case REVOLVE_OFFLINE:
    1649:       if (!tj->adjoint_solve_mode) SETERRQ(PetscObjectComm((PetscObject)tj),PETSC_ERR_SUP,"Not implemented");
    1650:       GetTrajROF(tj,ts,tjsch,stepnum);
    1651:       break;
    1652:     case REVOLVE_ONLINE:
    1653:       if (!tj->adjoint_solve_mode) SETERRQ(PetscObjectComm((PetscObject)tj),PETSC_ERR_SUP,"Not implemented");
    1654:       GetTrajRON(tj,ts,tjsch,stepnum);
    1655:       break;
    1656:     case REVOLVE_MULTISTAGE:
    1657:       if (!tj->adjoint_solve_mode) SETERRQ(PetscObjectComm((PetscObject)tj),PETSC_ERR_SUP,"Not implemented");
    1658:       GetTrajRMS(tj,ts,tjsch,stepnum);
    1659:       break;
    1660: #endif
    1661:     default:
    1662:       break;
    1663:   }
    1664:   return(0);
    1665: }

    1667: PETSC_UNUSED static PetscErrorCode TSTrajectorySetStride_Memory(TSTrajectory tj,PetscInt stride)
    1668: {
    1669:   TJScheduler *tjsch = (TJScheduler*)tj->data;

    1672:   tjsch->stride = stride;
    1673:   return(0);
    1674: }

    1676: PETSC_UNUSED static PetscErrorCode TSTrajectorySetMaxCpsRAM_Memory(TSTrajectory tj,PetscInt max_cps_ram)
    1677: {
    1678:   TJScheduler *tjsch = (TJScheduler*)tj->data;

    1681:   tjsch->max_cps_ram = max_cps_ram;
    1682:   return(0);
    1683: }

    1685: PETSC_UNUSED static PetscErrorCode TSTrajectorySetMaxCpsDisk_Memory(TSTrajectory tj,PetscInt max_cps_disk)
    1686: {
    1687:   TJScheduler *tjsch = (TJScheduler*)tj->data;

    1690:   tjsch->max_cps_disk = max_cps_disk;
    1691:   return(0);
    1692: }

    1694: #if defined(PETSC_HAVE_REVOLVE)
    1695: PETSC_UNUSED static PetscErrorCode TSTrajectorySetRevolveOnline(TSTrajectory tj,PetscBool use_online)
    1696: {
    1697:   TJScheduler *tjsch = (TJScheduler*)tj->data;

    1700:   tjsch->use_online = use_online;
    1701:   return(0);
    1702: }
    1703: #endif

    1705: PETSC_UNUSED static PetscErrorCode TSTrajectorySetSaveStack(TSTrajectory tj,PetscBool save_stack)
    1706: {
    1707:   TJScheduler *tjsch = (TJScheduler*)tj->data;

    1710:   tjsch->save_stack = save_stack;
    1711:   return(0);
    1712: }

    1714: PETSC_UNUSED static PetscErrorCode TSTrajectorySetUseDRAM(TSTrajectory tj,PetscBool use_dram)
    1715: {
    1716:   TJScheduler *tjsch = (TJScheduler*)tj->data;

    1719:   tjsch->stack.use_dram = use_dram;
    1720:   return(0);
    1721: }

    1723: static PetscErrorCode TSTrajectorySetFromOptions_Memory(PetscOptionItems *PetscOptionsObject,TSTrajectory tj)
    1724: {
    1725:   TJScheduler    *tjsch = (TJScheduler*)tj->data;

    1729:   PetscOptionsHead(PetscOptionsObject,"Memory based TS trajectory options");
    1730:   {
    1731:     PetscOptionsInt("-ts_trajectory_max_cps_ram","Maximum number of checkpoints in RAM","TSTrajectorySetMaxCpsRAM_Memory",tjsch->max_cps_ram,&tjsch->max_cps_ram,NULL);
    1732:     PetscOptionsInt("-ts_trajectory_max_cps_disk","Maximum number of checkpoints on disk","TSTrajectorySetMaxCpsDisk_Memory",tjsch->max_cps_disk,&tjsch->max_cps_disk,NULL);
    1733:     PetscOptionsInt("-ts_trajectory_stride","Stride to save checkpoints to file","TSTrajectorySetStride_Memory",tjsch->stride,&tjsch->stride,NULL);
    1734: #if defined(PETSC_HAVE_REVOLVE)
    1735:     PetscOptionsBool("-ts_trajectory_revolve_online","Trick TS trajectory into using online mode of revolve","TSTrajectorySetRevolveOnline",tjsch->use_online,&tjsch->use_online,NULL);
    1736: #endif
    1737:     PetscOptionsBool("-ts_trajectory_save_stack","Save all stack to disk","TSTrajectorySetSaveStack",tjsch->save_stack,&tjsch->save_stack,NULL);
    1738:     PetscOptionsBool("-ts_trajectory_use_dram","Use DRAM for checkpointing","TSTrajectorySetUseDRAM",tjsch->stack.use_dram,&tjsch->stack.use_dram,NULL);
    1739:   }
    1740:   PetscOptionsTail();
    1741:   tjsch->stack.solution_only = tj->solution_only;
    1742:   return(0);
    1743: }

    1745: static PetscErrorCode TSTrajectorySetUp_Memory(TSTrajectory tj,TS ts)
    1746: {
    1747:   TJScheduler    *tjsch = (TJScheduler*)tj->data;
    1748:   Stack          *stack = &tjsch->stack;
    1749: #if defined(PETSC_HAVE_REVOLVE)
    1750:   RevolveCTX     *rctx,*rctx2;
    1751:   DiskStack      *diskstack = &tjsch->diskstack;
    1752:   PetscInt       diskblocks;
    1753: #endif
    1754:   PetscInt       numY,total_steps;
    1755:   PetscBool      fixedtimestep;

    1759:   if (ts->adapt) {
    1760:     PetscObjectTypeCompare((PetscObject)ts->adapt,TSADAPTNONE,&fixedtimestep);
    1761:   } else fixedtimestep = PETSC_TRUE;
    1762:   total_steps = (PetscInt)(PetscCeilReal((ts->max_time-ts->ptime)/ts->time_step));
    1763:   total_steps = total_steps < 0 ? PETSC_MAX_INT : total_steps;
    1764:   if (fixedtimestep) tjsch->total_steps = PetscMin(ts->max_steps,total_steps);
    1765:   if (tjsch->max_cps_ram > 0) stack->stacksize = tjsch->max_cps_ram;

    1767:   if (tjsch->stride > 1) { /* two level mode */
    1768:     if (tjsch->save_stack && tjsch->max_cps_disk > 1 && tjsch->max_cps_disk <= tjsch->max_cps_ram) SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_ARG_INCOMP,"The specified disk capacity is not enough to store a full stack of RAM checkpoints. You might want to change the disk capacity or use single level checkpointing instead.");
    1769:     if (tjsch->max_cps_disk <= 1 && tjsch->max_cps_ram > 1 && tjsch->max_cps_ram <= tjsch->stride-1) tjsch->stype = TWO_LEVEL_REVOLVE; /* use revolve_offline for each stride */
    1770:     if (tjsch->max_cps_disk > 1 && tjsch->max_cps_ram > 1 && tjsch->max_cps_ram <= tjsch->stride-1) tjsch->stype = TWO_LEVEL_TWO_REVOLVE;  /* use revolve_offline for each stride */
    1771:     if (tjsch->max_cps_disk <= 1 && (tjsch->max_cps_ram >= tjsch->stride || tjsch->max_cps_ram == -1)) tjsch->stype = TWO_LEVEL_NOREVOLVE; /* can also be handled by TWO_LEVEL_REVOLVE */
    1772:   } else { /* single level mode */
    1773:     if (fixedtimestep) {
    1774:       if (tjsch->max_cps_ram >= tjsch->total_steps-1 || tjsch->max_cps_ram < 1) tjsch->stype = NONE; /* checkpoint all */
    1775:       else tjsch->stype = (tjsch->max_cps_disk>1) ? REVOLVE_MULTISTAGE : REVOLVE_OFFLINE;
    1776:     } else tjsch->stype = NONE; /* checkpoint all for adaptive time step */
    1777: #if defined(PETSC_HAVE_REVOLVE)
    1778:     if (tjsch->use_online) tjsch->stype = REVOLVE_ONLINE; /* trick into online (for testing purpose only) */
    1779: #endif
    1780:   }

    1782:   if (tjsch->stype > TWO_LEVEL_NOREVOLVE) {
    1783: #ifndef PETSC_HAVE_REVOLVE
    1784:     SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_SUP,"revolve is needed when there is not enough memory to checkpoint all time steps according to the user's settings, please reconfigure with the additional option --download-revolve.");
    1785: #else
    1786:     PetscRevolveInt rfine,rsnaps,rsnaps2;
    1787:     switch (tjsch->stype) {
    1788:       case TWO_LEVEL_REVOLVE:
    1789:         PetscRevolveIntCast(tjsch->stride,&rfine);
    1790:         PetscRevolveIntCast(tjsch->max_cps_ram,&rsnaps);
    1791:         revolve_create_offline(rfine,rsnaps);
    1792:         break;
    1793:       case TWO_LEVEL_TWO_REVOLVE:
    1794:         diskblocks = tjsch->save_stack ? tjsch->max_cps_disk/(tjsch->max_cps_ram+1) : tjsch->max_cps_disk; /* The block size depends on whether the stack is saved. */
    1795:         diskstack->stacksize = diskblocks;
    1796:         PetscRevolveIntCast(tjsch->stride,&rfine);
    1797:         PetscRevolveIntCast(tjsch->max_cps_ram,&rsnaps);
    1798:         revolve_create_offline(rfine,rsnaps);
    1799:         PetscRevolveIntCast((tjsch->total_steps+tjsch->stride-1)/tjsch->stride,&rfine);
    1800:         PetscRevolveIntCast(diskblocks,&rsnaps);
    1801:         revolve2_create_offline(rfine,rsnaps);
    1802:         PetscNew(&rctx2);
    1803:         rctx2->snaps_in       = rsnaps;
    1804:         rctx2->reverseonestep = PETSC_FALSE;
    1805:         rctx2->check          = 0;
    1806:         rctx2->oldcapo        = 0;
    1807:         rctx2->capo           = 0;
    1808:         rctx2->info           = 2;
    1809:         rctx2->fine           = rfine;
    1810:         tjsch->rctx2          = rctx2;
    1811:         diskstack->top        = -1;
    1812:         PetscMalloc1(diskstack->stacksize*sizeof(PetscInt),&diskstack->container);
    1813:         break;
    1814:       case REVOLVE_OFFLINE:
    1815:         PetscRevolveIntCast(tjsch->total_steps,&rfine);
    1816:         PetscRevolveIntCast(tjsch->max_cps_ram,&rsnaps);
    1817:         revolve_create_offline(rfine,rsnaps);
    1818:         break;
    1819:       case REVOLVE_ONLINE:
    1820:         stack->stacksize = tjsch->max_cps_ram;
    1821:         PetscRevolveIntCast(tjsch->max_cps_ram,&rsnaps);
    1822:         revolve_create_online(rsnaps);
    1823:         break;
    1824:       case REVOLVE_MULTISTAGE:
    1825:         PetscRevolveIntCast(tjsch->total_steps,&rfine);
    1826:         PetscRevolveIntCast(tjsch->max_cps_ram,&rsnaps);
    1827:         PetscRevolveIntCast(tjsch->max_cps_ram+tjsch->max_cps_disk,&rsnaps2);
    1828:         revolve_create_multistage(rfine,rsnaps2,rsnaps);
    1829:         break;
    1830:       default:
    1831:         break;
    1832:     }
    1833:     PetscNew(&rctx);
    1834:     PetscRevolveIntCast(tjsch->max_cps_ram,&rsnaps);
    1835:     rctx->snaps_in       = rsnaps; /* for theta methods snaps_in=2*max_cps_ram */
    1836:     rctx->reverseonestep = PETSC_FALSE;
    1837:     rctx->check          = 0;
    1838:     rctx->oldcapo        = 0;
    1839:     rctx->capo           = 0;
    1840:     rctx->info           = 2;
    1841:     if (tjsch->stride > 1) {
    1842:       PetscRevolveIntCast(tjsch->stride,&rfine);
    1843:     } else {
    1844:       PetscRevolveIntCast(tjsch->total_steps,&rfine);
    1845:     }
    1846:     rctx->fine           = rfine;
    1847:     tjsch->rctx          = rctx;
    1848:     if (tjsch->stype == REVOLVE_ONLINE) rctx->fine = -1;
    1849: #endif
    1850:   } else {
    1851:     if (tjsch->stype == TWO_LEVEL_NOREVOLVE) stack->stacksize = tjsch->stride-1; /* need tjsch->stride-1 at most */
    1852:     if (tjsch->stype == NONE) {
    1853:       if (fixedtimestep) stack->stacksize = stack->solution_only ? tjsch->total_steps : tjsch->total_steps-1;
    1854:       else { /* adaptive time step */
    1855:         /* if max_cps_ram is not specified, use maximal allowed number of steps for stack size */
    1856:         if (tjsch->max_cps_ram == -1) stack->stacksize = ts->max_steps < PETSC_MAX_INT ? ts->max_steps : 10000;
    1857:         tjsch->total_steps = stack->solution_only ? stack->stacksize : stack->stacksize+1; /* will be updated as time integration advances */
    1858:       }
    1859:     }
    1860:   }

    1862:   if ((tjsch->stype >= TWO_LEVEL_NOREVOLVE && tjsch->stype < REVOLVE_OFFLINE) || tjsch->stype == REVOLVE_MULTISTAGE) { /* these types need to use disk */
    1863:     TSTrajectorySetUp_Basic(tj,ts);
    1864:   }

    1866:   stack->stacksize = PetscMax(stack->stacksize,1);
    1867:   tjsch->recompute = PETSC_FALSE;
    1868:   TSGetStages(ts,&numY,PETSC_IGNORE);
    1869:   StackInit(stack,stack->stacksize,numY);
    1870:   return(0);
    1871: }

    1873: static PetscErrorCode TSTrajectoryReset_Memory(TSTrajectory tj)
    1874: {
    1875: #if defined(PETSC_HAVE_REVOLVE)
    1876:   TJScheduler    *tjsch = (TJScheduler*)tj->data;

    1880:   if (tjsch->stype > TWO_LEVEL_NOREVOLVE) {
    1881:     revolve_reset();
    1882:     if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) {
    1883:       revolve2_reset();
    1884:       PetscFree(tjsch->diskstack.container);
    1885:     }
    1886:   }
    1887:   if (tjsch->stype > TWO_LEVEL_NOREVOLVE) {
    1888:     PetscFree(tjsch->rctx);
    1889:     PetscFree(tjsch->rctx2);
    1890:   }
    1891: #endif
    1892:   return(0);
    1893: }

    1895: static PetscErrorCode TSTrajectoryDestroy_Memory(TSTrajectory tj)
    1896: {
    1897:   TJScheduler    *tjsch = (TJScheduler*)tj->data;

    1901:   StackDestroy(&tjsch->stack);
    1902:   PetscViewerDestroy(&tjsch->viewer);
    1903:   PetscFree(tjsch);
    1904:   return(0);
    1905: }

    1907: /*MC
    1908:       TSTRAJECTORYMEMORY - Stores each solution of the ODE/ADE in memory

    1910:   Level: intermediate

    1912: .seealso:  TSTrajectoryCreate(), TS, TSTrajectorySetType()

    1914: M*/
    1915: PETSC_EXTERN PetscErrorCode TSTrajectoryCreate_Memory(TSTrajectory tj,TS ts)
    1916: {
    1917:   TJScheduler    *tjsch;

    1921:   tj->ops->set            = TSTrajectorySet_Memory;
    1922:   tj->ops->get            = TSTrajectoryGet_Memory;
    1923:   tj->ops->setup          = TSTrajectorySetUp_Memory;
    1924:   tj->ops->setfromoptions = TSTrajectorySetFromOptions_Memory;
    1925:   tj->ops->reset          = TSTrajectoryReset_Memory;
    1926:   tj->ops->destroy        = TSTrajectoryDestroy_Memory;

    1928:   PetscNew(&tjsch);
    1929:   tjsch->stype        = NONE;
    1930:   tjsch->max_cps_ram  = -1; /* -1 indicates that it is not set */
    1931:   tjsch->max_cps_disk = -1; /* -1 indicates that it is not set */
    1932:   tjsch->stride       = 0; /* if not zero, two-level checkpointing will be used */
    1933: #if defined(PETSC_HAVE_REVOLVE)
    1934:   tjsch->use_online   = PETSC_FALSE;
    1935: #endif
    1936:   tjsch->save_stack   = PETSC_TRUE;

    1938:   tjsch->stack.solution_only = tj->solution_only;
    1939:   PetscViewerCreate(PetscObjectComm((PetscObject)tj),&tjsch->viewer);
    1940:   PetscViewerSetType(tjsch->viewer,PETSCVIEWERBINARY);
    1941:   PetscViewerPushFormat(tjsch->viewer,PETSC_VIEWER_NATIVE);
    1942:   PetscViewerFileSetMode(tjsch->viewer,FILE_MODE_WRITE);

    1944:   tj->data = tjsch;
    1945:   return(0);
    1946: }