Actual source code: init.c

      1: /*

      3:    This file defines part of the initialization of PETSc

      5:   This file uses regular malloc and free because it cannot be known
      6:   what malloc is being used until it has already processed the input.
      7: */

      9: #include <petscsys.h>
     10: #include <petsc/private/petscimpl.h>
     11: #include <petscvalgrind.h>
     12: #include <petscviewer.h>
     13: #if defined(PETSC_USE_LOG)
     14: PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
     15: #endif

     17: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
     18: #include <sys/sysinfo.h>
     19: #endif
     20: #if defined(PETSC_HAVE_UNISTD_H)
     21: #include <unistd.h>
     22: #endif

     24: #if defined(PETSC_HAVE_CUDA)
     25:   #include <cuda_runtime.h>
     26: #include <petsccublas.h>
     27: #endif

     29: #if defined(PETSC_HAVE_HIP)
     30:   #include <hip/hip_runtime.h>
     31: #endif

     33: #if defined(PETSC_HAVE_DEVICE)
     34:   #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION)
     35:     #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */
     36:   #endif
     37: #endif

     39: #if defined(PETSC_HAVE_VIENNACL)
     40: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
     41: #endif

     44: /* ------------------------Nasty global variables -------------------------------*/
     45: /*
     46:      Indicates if PETSc started up MPI, or it was
     47:    already started before PETSc was initialized.
     48: */
     49: PetscBool   PetscBeganMPI                 = PETSC_FALSE;
     50: PetscBool   PetscErrorHandlingInitialized = PETSC_FALSE;
     51: PetscBool   PetscInitializeCalled         = PETSC_FALSE;
     52: PetscBool   PetscFinalizeCalled           = PETSC_FALSE;

     54: PetscMPIInt PetscGlobalRank               = -1;
     55: PetscMPIInt PetscGlobalSize               = -1;

     57: #if defined(PETSC_HAVE_KOKKOS)
     58: PetscBool   PetscBeganKokkos              = PETSC_FALSE;
     59: #endif

     61: #if defined(PETSC_HAVE_NVSHMEM)
     62: PetscBool   PetscBeganNvshmem             = PETSC_FALSE;
     63: PetscBool   PetscNvshmemInitialized       = PETSC_FALSE;
     64: #endif

     66: PetscBool   use_gpu_aware_mpi             = PETSC_TRUE;
     67: PetscBool   PetscCreatedGpuObjects        = PETSC_FALSE;

     69: #if defined(PETSC_HAVE_COMPLEX)
     70: #if defined(PETSC_COMPLEX_INSTANTIATE)
     71: template <> class std::complex<double>; /* instantiate complex template class */
     72: #endif

     74: /*MC
     75:    PETSC_i - the imaginary number i

     77:    Synopsis:
     78: #include <petscsys.h>
     79:    PetscComplex PETSC_i;

     81:    Level: beginner

     83:    Note:
     84:    Complex numbers are automatically available if PETSc located a working complex implementation

     86: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
     87: M*/
     88: PetscComplex PETSC_i;
     89: MPI_Datatype MPIU___COMPLEX128 = 0;
     90: #endif /* PETSC_HAVE_COMPLEX */
     91: #if defined(PETSC_USE_REAL___FLOAT128)
     92: MPI_Datatype MPIU___FLOAT128 = 0;
     93: #elif defined(PETSC_USE_REAL___FP16)
     94: MPI_Datatype MPIU___FP16 = 0;
     95: #endif
     96: MPI_Datatype MPIU_2SCALAR = 0;
     97: #if defined(PETSC_USE_64BIT_INDICES)
     98: MPI_Datatype MPIU_2INT = 0;
     99: #endif
    100: MPI_Datatype MPIU_BOOL;
    101: MPI_Datatype MPIU_ENUM;
    102: MPI_Datatype MPIU_FORTRANADDR;
    103: MPI_Datatype MPIU_SIZE_T;

    105: /*
    106:        Function that is called to display all error messages
    107: */
    108: PetscErrorCode (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
    109: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
    110: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
    111: /*
    112:   This is needed to turn on/off GPU synchronization
    113: */
    114: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;

    116: /* ------------------------------------------------------------------------------*/
    117: /*
    118:    Optional file where all PETSc output from various prints is saved
    119: */
    120: PETSC_INTERN FILE *petsc_history;
    121: FILE *petsc_history = NULL;

    123: PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
    124: {
    126:   PetscMPIInt    rank,size;
    127:   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
    128:   char           version[256];

    131:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
    132:   if (!rank) {
    133:     char        arch[10];
    134:     int         err;

    136:     PetscGetArchType(arch,10);
    137:     PetscGetDate(date,64);
    138:     PetscGetVersion(version,256);
    139:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
    140:     if (filename) {
    141:       PetscFixFilename(filename,fname);
    142:     } else {
    143:       PetscGetHomeDirectory(pfile,sizeof(pfile));
    144:       PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));
    145:       PetscFixFilename(pfile,fname);
    146:     }

    148:     *fd = fopen(fname,"a");
    149:     if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);

    151:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
    152:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
    153:     PetscGetProgramName(pname,sizeof(pname));
    154:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
    155:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");

    157:     err = fflush(*fd);
    158:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
    159:   }
    160:   return(0);
    161: }

    163: PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd)
    164: {
    166:   PetscMPIInt    rank;
    167:   char           date[64];
    168:   int            err;

    171:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
    172:   if (!rank) {
    173:     PetscGetDate(date,64);
    174:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
    175:     PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
    176:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
    177:     err  = fflush(*fd);
    178:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
    179:     err = fclose(*fd);
    180:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
    181:   }
    182:   return(0);
    183: }

    185: /* ------------------------------------------------------------------------------*/

    187: /*
    188:    This is ugly and probably belongs somewhere else, but I want to
    189:   be able to put a true MPI abort error handler with command line args.

    191:     This is so MPI errors in the debugger will leave all the stack
    192:   frames. The default MP_Abort() cleans up and exits thus providing no useful information
    193:   in the debugger hence we call abort() instead of MPI_Abort().
    194: */

    196: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
    197: {
    199:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
    200:   abort();
    201: }

    203: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
    204: {

    208:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
    209:   PetscAttachDebugger();
    210:   if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */
    211: }

    213: /*@C
    214:    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
    215:      wishes a clean exit somewhere deep in the program.

    217:    Collective on PETSC_COMM_WORLD

    219:    Options Database Keys are the same as for PetscFinalize()

    221:    Level: advanced

    223:    Note:
    224:    See PetscInitialize() for more general runtime options.

    226: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
    227: @*/
    228: PetscErrorCode  PetscEnd(void)
    229: {
    231:   PetscFinalize();
    232:   exit(0);
    233:   return 0;
    234: }

    236: PetscBool PetscOptionsPublish = PETSC_FALSE;
    237: PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void);
    238: PETSC_INTERN PetscBool      petscsetmallocvisited;
    239: static       char           emacsmachinename[256];

    241: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL;
    242: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = NULL;

    244: /*@C
    245:    PetscSetHelpVersionFunctions - Sets functions that print help and version information
    246:    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
    247:    This routine enables a "higher-level" package that uses PETSc to print its messages first.

    249:    Input Parameter:
    250: +  help - the help function (may be NULL)
    251: -  version - the version function (may be NULL)

    253:    Level: developer

    255: @*/
    256: PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
    257: {
    259:   PetscExternalHelpFunction    = help;
    260:   PetscExternalVersionFunction = version;
    261:   return(0);
    262: }

    264: #if defined(PETSC_USE_LOG)
    265: PETSC_INTERN PetscBool   PetscObjectsLog;
    266: #endif

    268: /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP.
    269:    Use the following macros to define CUDA/HIP initialization related vars/routines.
    270:  */
    271: #if defined(PETSC_HAVE_CUDA)
    272:   typedef cudaError_t                             cupmError_t;
    273:   typedef struct cudaDeviceProp                   cupmDeviceProp;
    274:   typedef cudaStream_t                            cupmStream_t;
    275:   #define cupmGetDeviceCount(x)                   cudaGetDeviceCount(x)
    276:   #define cupmGetDevice(x)                        cudaGetDevice(x)
    277:   #define cupmSetDevice(x)                        cudaSetDevice(x)
    278:   #define cupmSetDeviceFlags(x)                   cudaSetDeviceFlags(x)
    279:   #define cupmGetDeviceProperties(x,y)            cudaGetDeviceProperties(x,y)
    280:   #define cupmStreamCreate(x)                     cudaStreamCreate(x)
    281:   #define cupmGetLastError()                      cudaGetLastError()
    282:   #define cupmDeviceMapHost                       cudaDeviceMapHost
    283:   #define cupmSuccess                             cudaSuccess
    284:   #define cupmErrorMemoryAllocation               cudaErrorMemoryAllocation
    285:   #define cupmErrorLaunchOutOfResources           cudaErrorLaunchOutOfResources
    286:   #define cupmErrorSetOnActiveProcess             cudaErrorSetOnActiveProcess
    287:   #define CHKERRCUPM(x)                           CHKERRCUDA(x)
    288:   #define PetscCUPMBLASInitializeHandle()         PetscCUBLASInitializeHandle()
    289:   #define PetscCUPMSOLVERDnInitializeHandle()     PetscCUSOLVERDnInitializeHandle()
    290:   #define PetscCUPMInitialize                     PetscCUDAInitialize
    291:   #define PetscCUPMInitialized                    PetscCUDAInitialized
    292:   #define PetscCUPMInitializeCheck                PetscCUDAInitializeCheck
    293:   #define PetscCUPMInitializeAndView              PetscCUDAInitializeAndView
    294:   #define PetscCUPMSynchronize                    PetscCUDASynchronize
    295:   #define PetscNotUseCUPM                         PetscNotUseCUDA
    296:   #define cupmOptionsStr                          "CUDA options"
    297:   #define cupmSetDeviceStr                        "-cuda_device"
    298:   #define cupmViewStr                             "-cuda_view"
    299:   #define cupmSynchronizeStr                      "-cuda_synchronize"
    300:   #define PetscCUPMInitializeStr                  "PetscCUDAInitialize"
    301:   #define PetscOptionsCheckCUPM                   PetscOptionsCheckCUDA
    302:   #define PetscMPICUPMAwarenessCheck              PetscMPICUDAAwarenessCheck
    303:   #define PetscDefaultCupmStream                  PetscDefaultCudaStream
    304:   #include "cupminit.inc"
    305: #endif

    307: #if defined(PETSC_HAVE_HIP)
    308:   typedef hipError_t                              cupmError_t;
    309:   typedef hipDeviceProp_t                         cupmDeviceProp;
    310:   typedef hipStream_t                             cupmStream_t;
    311:   #define cupmGetDeviceCount(x)                   hipGetDeviceCount(x)
    312:   #define cupmGetDevice(x)                        hipGetDevice(x)
    313:   #define cupmSetDevice(x)                        hipSetDevice(x)
    314:   #define cupmSetDeviceFlags(x)                   hipSetDeviceFlags(x)
    315:   #define cupmGetDeviceProperties(x,y)            hipGetDeviceProperties(x,y)
    316:   #define cupmStreamCreate(x)                     hipStreamCreate(x)
    317:   #define cupmGetLastError()                      hipGetLastError()
    318:   #define cupmDeviceMapHost                       hipDeviceMapHost
    319:   #define cupmSuccess                             hipSuccess
    320:   #define cupmErrorMemoryAllocation               hipErrorMemoryAllocation
    321:   #define cupmErrorLaunchOutOfResources           hipErrorLaunchOutOfResources
    322:   #define cupmErrorSetOnActiveProcess             hipErrorSetOnActiveProcess
    323:   #define CHKERRCUPM(x)                           CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB)
    324:   #define PetscCUPMBLASInitializeHandle()         0
    325:   #define PetscCUPMSOLVERDnInitializeHandle()     0
    326:   #define PetscCUPMInitialize                     PetscHIPInitialize
    327:   #define PetscCUPMInitialized                    PetscHIPInitialized
    328:   #define PetscCUPMInitializeCheck                PetscHIPInitializeCheck
    329:   #define PetscCUPMInitializeAndView              PetscHIPInitializeAndView
    330:   #define PetscCUPMSynchronize                    PetscHIPSynchronize
    331:   #define PetscNotUseCUPM                         PetscNotUseHIP
    332:   #define cupmOptionsStr                          "HIP options"
    333:   #define cupmSetDeviceStr                        "-hip_device"
    334:   #define cupmViewStr                             "-hip_view"
    335:   #define cupmSynchronizeStr                      "-hip_synchronize"
    336:   #define PetscCUPMInitializeStr                  "PetscHIPInitialize"
    337:   #define PetscOptionsCheckCUPM                   PetscOptionsCheckHIP
    338:   #define PetscMPICUPMAwarenessCheck              PetscMPIHIPAwarenessCheck
    339:   #define PetscDefaultCupmStream                  PetscDefaultHipStream
    340:   #include "cupminit.inc"
    341: #endif

    343: PETSC_INTERN PetscErrorCode  PetscOptionsCheckInitial_Private(const char help[])
    344: {
    345:   char              string[64];
    346:   MPI_Comm          comm = PETSC_COMM_WORLD;
    347:   PetscBool         flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView;
    348:   PetscErrorCode    ierr;
    349:   PetscReal         si;
    350:   PetscInt          intensity;
    351:   int               i;
    352:   PetscMPIInt       rank;
    353:   char              version[256];
    354: #if defined(PETSC_USE_LOG)
    355:   char              mname[PETSC_MAX_PATH_LEN];
    356:   PetscViewerFormat format;
    357:   PetscBool         flg4 = PETSC_FALSE;
    358: #endif

    361:   MPI_Comm_rank(comm,&rank);

    363: #if !defined(PETSC_HAVE_THREADSAFETY)
    364:   if (!(PETSC_RUNNING_ON_VALGRIND)) {
    365:     /*
    366:       Setup the memory management; support for tracing malloc() usage
    367:     */
    368:     PetscBool         mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE;

    370:     if (PetscDefined(USE_DEBUG)) {
    371:       mdebug        = PETSC_TRUE;
    372:       initializenan = PETSC_TRUE;
    373:       PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
    374:     } else {
    375:       /* don't warn about unused option */
    376:       PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
    377:       flg1 = PETSC_FALSE;
    378:     }
    379:     PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);
    380:     if (flg1 || flg2) {
    381:       mdebug        = PETSC_TRUE;
    382:       eachcall      = PETSC_TRUE;
    383:       initializenan = PETSC_TRUE;
    384:     } else if (flg3 && !flg2) {
    385:       mdebug        = PETSC_FALSE;
    386:       eachcall      = PETSC_FALSE;
    387:       initializenan = PETSC_FALSE;
    388:     }

    390:     PetscOptionsGetBool(NULL,NULL,"-malloc_requested_size",&flg1,&flg2);
    391:     if (flg2) {PetscMallocLogRequestedSizeSet(flg1);}

    393:     PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);
    394:     if (mlog) {
    395:       mdebug = PETSC_TRUE;
    396:     }
    397:     /* the next line is deprecated */
    398:     PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);
    399:     PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);
    400:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);
    401:     if (mdebug) {
    402:       PetscMallocSetDebug(eachcall,initializenan);
    403:     }
    404:     if (mlog) {
    405:       PetscReal logthreshold = 0;
    406:       PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);
    407:       PetscMallocViewSet(logthreshold);
    408:     }
    409: #if defined(PETSC_USE_LOG)
    410:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);
    411: #endif
    412:   }

    414:   PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);
    415:   if (flg2) {PetscMallocSetCoalesce(flg1);}
    416:   flg1 = PETSC_FALSE;
    417:   PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
    418:   /* ignore this option if malloc is already set */
    419:   if (flg1 && !petscsetmallocvisited) {PetscSetUseHBWMalloc_Private();}

    421:   flg1 = PETSC_FALSE;
    422:   PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
    423:   if (!flg1) {
    424:     flg1 = PETSC_FALSE;
    425:     PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
    426:   }
    427:   if (flg1) {
    428:     PetscMemorySetGetMaximumUsage();
    429:   }
    430: #endif

    432: #if defined(PETSC_USE_LOG)
    433:   PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
    434: #endif

    436:   /*
    437:       Set the display variable for graphics
    438:   */
    439:   PetscSetDisplay();

    441:   /*
    442:      Print main application help message
    443:   */
    444:   PetscOptionsHasHelp(NULL,&hasHelp);
    445:   if (help && hasHelp) {
    446:     PetscPrintf(comm,help);
    447:     PetscPrintf(comm,"----------------------------------------\n");
    448:   }

    450:   /*
    451:       Print the PETSc version information
    452:   */
    453:   PetscOptionsHasName(NULL,NULL,"-version",&flg1);
    454:   if (flg1 || hasHelp) {
    455:     /*
    456:        Print "higher-level" package version message
    457:     */
    458:     if (PetscExternalVersionFunction) {
    459:       (*PetscExternalVersionFunction)(comm);
    460:     }

    462:     PetscGetVersion(version,256);
    463:     (*PetscHelpPrintf)(comm,"%s\n",version);
    464:     (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
    465:     (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
    466:     (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
    467:     (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
    468:     (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
    469:     (*PetscHelpPrintf)(comm,"----------------------------------------\n");
    470:   }

    472:   /*
    473:        Print "higher-level" package help message
    474:   */
    475:   if (hasHelp) {
    476:     PetscBool hasHelpIntro;

    478:     if (PetscExternalHelpFunction) {
    479:       (*PetscExternalHelpFunction)(comm);
    480:     }
    481:     PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);
    482:     if (hasHelpIntro) {
    483:       PetscOptionsDestroyDefault();
    484:       PetscFreeMPIResources();
    485:       MPI_Finalize();
    486:       exit(0);
    487:     }
    488:   }

    490:   /*
    491:       Setup the error handling
    492:   */
    493:   flg1 = PETSC_FALSE;
    494:   PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
    495:   if (flg1) {
    496:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);
    497:     PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
    498:   }
    499:   flg1 = PETSC_FALSE;
    500:   PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
    501:   if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);}
    502:   flg1 = PETSC_FALSE;
    503:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
    504:   if (flg1) {
    505:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
    506:   }
    507:   flg1 = PETSC_FALSE;
    508:   PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
    509:   if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}

    511:   /*
    512:       Setup debugger information
    513:   */
    514:   PetscSetDefaultDebugger();
    515:   PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);
    516:   if (flg1) {
    517:     MPI_Errhandler err_handler;

    519:     PetscSetDebuggerFromString(string);
    520:     MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);
    521:     MPI_Comm_set_errhandler(comm,err_handler);
    522:     PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);
    523:   }
    524:   PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);
    525:   if (flg1) { PetscSetDebugTerminal(string); }
    526:   PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);
    527:   PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);
    528:   if (flg1 || flg2) {
    529:     PetscMPIInt    size;
    530:     PetscInt       lsize,*ranks;
    531:     MPI_Errhandler err_handler;
    532:     /*
    533:        we have to make sure that all processors have opened
    534:        connections to all other processors, otherwise once the
    535:        debugger has stated it is likely to receive a SIGUSR1
    536:        and kill the program.
    537:     */
    538:     MPI_Comm_size(comm,&size);
    539:     if (size > 2) {
    540:       PetscMPIInt dummy = 0;
    541:       MPI_Status  status;
    542:       for (i=0; i<size; i++) {
    543:         if (rank != i) {
    544:           MPI_Send(&dummy,1,MPI_INT,i,109,comm);
    545:         }
    546:       }
    547:       for (i=0; i<size; i++) {
    548:         if (rank != i) {
    549:           MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);
    550:         }
    551:       }
    552:     }
    553:     /* check if this processor node should be in debugger */
    554:     PetscMalloc1(size,&ranks);
    555:     lsize = size;
    556:     /* Deprecated in 3.14 */
    557:     PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",ranks,&lsize,&flag);
    558:     if (flag) {
    559:       const char * const quietopt="-options_suppress_deprecated_warnings";
    560:       char               msg[4096];
    561:       PetscBool          quiet = PETSC_FALSE;

    563:       PetscOptionsGetBool(NULL,NULL,quietopt,&quiet,NULL);
    564:       if (!quiet) {
    565:         PetscStrcpy(msg,"** PETSc DEPRECATION WARNING ** : the option ");
    566:         PetscStrcat(msg,"-debugger_nodes");
    567:         PetscStrcat(msg," is deprecated as of version ");
    568:         PetscStrcat(msg,"3.14");
    569:         PetscStrcat(msg," and will be removed in a future release.");
    570:         PetscStrcat(msg," Please use the option ");
    571:         PetscStrcat(msg,"-debugger_ranks");
    572:         PetscStrcat(msg," instead.");
    573:         PetscStrcat(msg," (Silence this warning with ");
    574:         PetscStrcat(msg,quietopt);
    575:         PetscStrcat(msg,")\n");
    576:         PetscPrintf(comm,msg);
    577:       }
    578:     } else {
    579:       lsize = size;
    580:       PetscOptionsGetIntArray(NULL,NULL,"-debugger_ranks",ranks,&lsize,&flag);
    581:     }
    582:     if (flag) {
    583:       for (i=0; i<lsize; i++) {
    584:         if (ranks[i] == rank) { flag = PETSC_FALSE; break; }
    585:       }
    586:     }
    587:     if (!flag) {
    588:       PetscSetDebuggerFromString(string);
    589:       PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
    590:       if (flg1) {
    591:         PetscAttachDebugger();
    592:       } else {
    593:         PetscStopForDebugger();
    594:       }
    595:       MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);
    596:       MPI_Comm_set_errhandler(comm,err_handler);
    597:     } else {
    598:       PetscWaitOnError();
    599:     }
    600:     PetscFree(ranks);
    601:   }

    603:   PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);
    604:   if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}

    606:   /*
    607:         Setup profiling and logging
    608:   */
    609: #if defined(PETSC_USE_INFO)
    610:   {
    611:     PetscInfoSetFromOptions(NULL);
    612:   }
    613: #endif
    614:   PetscDetermineInitialFPTrap();
    615:   flg1 = PETSC_FALSE;
    616:   PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);
    617:   if (flag) {PetscSetFPTrap((PetscFPTrap)flg1);}
    618:   PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);
    620: #if defined(PETSC_USE_LOG)
    621:   mname[0] = 0;
    622:   PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);
    623:   if (flg1) {
    624:     if (mname[0]) {
    625:       PetscOpenHistoryFile(mname,&petsc_history);
    626:     } else {
    627:       PetscOpenHistoryFile(NULL,&petsc_history);
    628:     }
    629:   }

    631:   PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);

    633: #if defined(PETSC_HAVE_MPE)
    634:   flg1 = PETSC_FALSE;
    635:   PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
    636:   if (flg1) {PetscLogMPEBegin();}
    637: #endif
    638:   flg1 = PETSC_FALSE;
    639:   flg3 = PETSC_FALSE;
    640:   PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
    641:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
    642:   if (flg1)                      { PetscLogAllBegin(); }
    643:   else if (flg3)                 { PetscLogDefaultBegin();}

    645:   PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);
    646:   if (flg1) {
    647:     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
    648:     FILE *file;
    649:     if (mname[0]) {
    650:       PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
    651:       PetscFixFilename(name,fname);
    652:       file = fopen(fname,"w");
    653:       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
    654:     } else file = PETSC_STDOUT;
    655:     PetscLogTraceBegin(file);
    656:   }

    658:   PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);
    659:   if (flg4) {
    660:     if (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH) {
    661:       PetscLogNestedBegin();
    662:     } else {
    663:       PetscLogDefaultBegin();
    664:     }
    665:   }
    666:   if (flg4 && (format == PETSC_VIEWER_ASCII_XML || format == PETSC_VIEWER_ASCII_FLAMEGRAPH)) {
    667:     PetscReal threshold = PetscRealConstant(0.01);
    668:     PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);
    669:     if (flg1) {PetscLogSetThreshold((PetscLogDouble)threshold,NULL);}
    670:   }
    671: #endif

    673:   PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);
    674:   PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);
    675:   /*
    676:     If collecting logging information, by default, wait for device to complete its operations
    677:     before returning to the CPU in order to get accurate timings of each event
    678:   */
    679:   PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);
    680:   if (!logView) {PetscOptionsHasName(NULL,NULL,"-log_view",&logView);}

    682: #if defined(PETSC_HAVE_CUDA)
    683:   PetscOptionsCheckCUDA(logView);
    684: #endif

    686: #if defined(PETSC_HAVE_HIP)
    687:   PetscOptionsCheckHIP(logView);
    688: #endif

    690:   /*
    691:        Print basic help message
    692:   */
    693:   if (hasHelp) {
    694:     (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
    695:     (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");
    696:     (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");
    697:     (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");
    698:     (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
    699:     (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");
    700:     (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
    701:     (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");
    702:     (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");
    703:     (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
    704:     (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");
    705:     (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
    706:     (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");
    707:     (*PetscHelpPrintf)(comm," -debugger_ranks [n1,n2,..] Ranks to start in debugger\n");
    708:     (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
    709:     (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
    710:     (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");
    711:     (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
    712:     (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
    713:     (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
    714:     (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
    715:     (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");
    716:     (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
    717:     (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");
    718:     (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");
    719:     (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
    720:     (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");
    721:     (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");
    722:     (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");
    723:     (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
    724:     (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
    725:     (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
    726:     (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
    727:     (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
    728:     (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
    729: #if defined(PETSC_USE_LOG)
    730:     (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
    731:     (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");
    732:     (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
    733:     (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");
    734: #if defined(PETSC_HAVE_MPE)
    735:     (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
    736: #endif
    737: #endif
    738: #if defined(PETSC_USE_INFO)
    739:     (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");
    740: #endif
    741:     (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
    742:     (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");
    743:     (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");
    744:     (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
    745:   }

    747: #if defined(PETSC_HAVE_POPEN)
    748:   {
    749:   char machine[128];
    750:   PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);
    751:   if (flg1) {
    752:     PetscPOpenSetMachine(machine);
    753:   }
    754:   }
    755: #endif

    757:   PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
    758:   if (flg1) {
    759:     PetscSleep(si);
    760:   }

    762: #if defined(PETSC_HAVE_VIENNACL)
    763:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
    764:   if (!flg3) {
    765:     PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
    766:   }
    767:   PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
    768:   PetscViennaCLSynchronize = flg3;
    769:   PetscViennaCLInit();
    770: #endif

    772:   /*
    773:      Creates the logging data structures; this is enabled even if logging is not turned on
    774:      This is the last thing we do before returning to the user code to prevent having the
    775:      logging numbers contaminated by any startup time associated with MPI and the GPUs
    776:   */
    777: #if defined(PETSC_USE_LOG)
    778:   PetscLogInitialize();
    779: #endif

    781:   return(0);
    782: }