Actual source code: vecimpl.h

      1: #pragma once

      3: /*
      4:   This private file should not be included in users' code.  Defines the fields shared by all
      5:   vector implementations.
      6: */

      8: #include <petscvec.h>
      9: #include <petsc/private/petscimpl.h>

     11: PETSC_INTERN PetscBool VecRegisterAllCalled;
     12: PETSC_EXTERN MPI_Op    MPIU_MAXLOC;
     13: PETSC_EXTERN MPI_Op    MPIU_MINLOC;

     15: /* ----------------------------------------------------------------------------*/

     17: typedef struct _VecOps *VecOps;
     18: struct _VecOps {
     19:   PetscErrorCode (*duplicate)(Vec, Vec *);                                          /* get single vector */
     20:   PetscErrorCode (*duplicatevecs)(Vec, PetscInt, Vec **);                           /* get array of vectors */
     21:   PetscErrorCode (*destroyvecs)(PetscInt, Vec[]);                                   /* free array of vectors */
     22:   PetscErrorCode (*dot)(Vec, Vec, PetscScalar *);                                   /* z = x^H * y */
     23:   PetscErrorCode (*mdot)(Vec, PetscInt, const Vec[], PetscScalar *);                /* z[j] = x dot y[j] */
     24:   PetscErrorCode (*norm)(Vec, NormType, PetscReal *);                               /* z = sqrt(x^H * x) */
     25:   PetscErrorCode (*tdot)(Vec, Vec, PetscScalar *);                                  /* x'*y */
     26:   PetscErrorCode (*mtdot)(Vec, PetscInt, const Vec[], PetscScalar *);               /* z[j] = x dot y[j] */
     27:   PetscErrorCode (*scale)(Vec, PetscScalar);                                        /* x = alpha * x   */
     28:   PetscErrorCode (*copy)(Vec, Vec);                                                 /* y = x */
     29:   PetscErrorCode (*set)(Vec, PetscScalar);                                          /* y = alpha  */
     30:   PetscErrorCode (*swap)(Vec, Vec);                                                 /* exchange x and y */
     31:   PetscErrorCode (*axpy)(Vec, PetscScalar, Vec);                                    /* y = y + alpha * x */
     32:   PetscErrorCode (*axpby)(Vec, PetscScalar, PetscScalar, Vec);                      /* y = alpha * x + beta * y*/
     33:   PetscErrorCode (*maxpy)(Vec, PetscInt, const PetscScalar *, Vec *);               /* y = y + alpha[j] x[j] */
     34:   PetscErrorCode (*aypx)(Vec, PetscScalar, Vec);                                    /* y = x + alpha * y */
     35:   PetscErrorCode (*waxpy)(Vec, PetscScalar, Vec, Vec);                              /* w = y + alpha * x */
     36:   PetscErrorCode (*axpbypcz)(Vec, PetscScalar, PetscScalar, PetscScalar, Vec, Vec); /* z = alpha * x + beta *y + gamma *z*/
     37:   PetscErrorCode (*pointwisemult)(Vec, Vec, Vec);                                   /* w = x .* y */
     38:   PetscErrorCode (*pointwisedivide)(Vec, Vec, Vec);                                 /* w = x ./ y */
     39:   PetscErrorCode (*setvalues)(Vec, PetscInt, const PetscInt[], const PetscScalar[], InsertMode);
     40:   PetscErrorCode (*assemblybegin)(Vec);            /* start global assembly */
     41:   PetscErrorCode (*assemblyend)(Vec);              /* end global assembly */
     42:   PetscErrorCode (*getarray)(Vec, PetscScalar **); /* get data array */
     43:   PetscErrorCode (*getsize)(Vec, PetscInt *);
     44:   PetscErrorCode (*getlocalsize)(Vec, PetscInt *);
     45:   PetscErrorCode (*restorearray)(Vec, PetscScalar **); /* restore data array */
     46:   PetscErrorCode (*max)(Vec, PetscInt *, PetscReal *); /* z = max(x); idx=index of max(x) */
     47:   PetscErrorCode (*min)(Vec, PetscInt *, PetscReal *); /* z = min(x); idx=index of min(x) */
     48:   PetscErrorCode (*setrandom)(Vec, PetscRandom);       /* set y[j] = random numbers */
     49:   PetscErrorCode (*setoption)(Vec, VecOption, PetscBool);
     50:   PetscErrorCode (*setvaluesblocked)(Vec, PetscInt, const PetscInt[], const PetscScalar[], InsertMode);
     51:   PetscErrorCode (*destroy)(Vec);
     52:   PetscErrorCode (*view)(Vec, PetscViewer);
     53:   PetscErrorCode (*placearray)(Vec, const PetscScalar *);   /* place data array */
     54:   PetscErrorCode (*replacearray)(Vec, const PetscScalar *); /* replace data array */
     55:   PetscErrorCode (*dot_local)(Vec, Vec, PetscScalar *);
     56:   PetscErrorCode (*tdot_local)(Vec, Vec, PetscScalar *);
     57:   PetscErrorCode (*norm_local)(Vec, NormType, PetscReal *);
     58:   PetscErrorCode (*mdot_local)(Vec, PetscInt, const Vec[], PetscScalar *);
     59:   PetscErrorCode (*mtdot_local)(Vec, PetscInt, const Vec[], PetscScalar *);
     60:   PetscErrorCode (*load)(Vec, PetscViewer);
     61:   PetscErrorCode (*reciprocal)(Vec);
     62:   PetscErrorCode (*conjugate)(Vec);
     63:   PetscErrorCode (*setlocaltoglobalmapping)(Vec, ISLocalToGlobalMapping);
     64:   PetscErrorCode (*getlocaltoglobalmapping)(Vec, ISLocalToGlobalMapping *);
     65:   PetscErrorCode (*resetarray)(Vec); /* vector points to its original array, i.e. undoes any VecPlaceArray() */
     66:   PetscErrorCode (*setfromoptions)(Vec, PetscOptionItems);
     67:   PetscErrorCode (*maxpointwisedivide)(Vec, Vec, PetscReal *); /* m = max abs(x ./ y) */
     68:   PetscErrorCode (*pointwisemax)(Vec, Vec, Vec);
     69:   PetscErrorCode (*pointwisemaxabs)(Vec, Vec, Vec);
     70:   PetscErrorCode (*pointwisemin)(Vec, Vec, Vec);
     71:   PetscErrorCode (*getvalues)(Vec, PetscInt, const PetscInt[], PetscScalar[]);
     72:   PetscErrorCode (*sqrt)(Vec);
     73:   PetscErrorCode (*abs)(Vec);
     74:   PetscErrorCode (*exp)(Vec);
     75:   PetscErrorCode (*log)(Vec);
     76:   PetscErrorCode (*shift)(Vec, PetscScalar);
     77:   PetscErrorCode (*create)(Vec);
     78:   PetscErrorCode (*stridegather)(Vec, PetscInt, Vec, InsertMode);
     79:   PetscErrorCode (*stridescatter)(Vec, PetscInt, Vec, InsertMode);
     80:   PetscErrorCode (*dotnorm2)(Vec, Vec, PetscScalar *, PetscScalar *);
     81:   PetscErrorCode (*getsubvector)(Vec, IS, Vec *);
     82:   PetscErrorCode (*restoresubvector)(Vec, IS, Vec *);
     83:   PetscErrorCode (*getarrayread)(Vec, const PetscScalar **);
     84:   PetscErrorCode (*restorearrayread)(Vec, const PetscScalar **);
     85:   PetscErrorCode (*stridesubsetgather)(Vec, PetscInt, const PetscInt[], const PetscInt[], Vec, InsertMode);
     86:   PetscErrorCode (*stridesubsetscatter)(Vec, PetscInt, const PetscInt[], const PetscInt[], Vec, InsertMode);
     87:   PetscErrorCode (*viewnative)(Vec, PetscViewer);
     88:   PetscErrorCode (*loadnative)(Vec, PetscViewer);
     89:   PetscErrorCode (*createlocalvector)(Vec, Vec *);
     90:   PetscErrorCode (*getlocalvector)(Vec, Vec);
     91:   PetscErrorCode (*restorelocalvector)(Vec, Vec);
     92:   PetscErrorCode (*getlocalvectorread)(Vec, Vec);
     93:   PetscErrorCode (*restorelocalvectorread)(Vec, Vec);
     94:   PetscErrorCode (*bindtocpu)(Vec, PetscBool);
     95:   PetscErrorCode (*getarraywrite)(Vec, PetscScalar **);
     96:   PetscErrorCode (*restorearraywrite)(Vec, PetscScalar **);
     97:   PetscErrorCode (*getarrayandmemtype)(Vec, PetscScalar **, PetscMemType *);
     98:   PetscErrorCode (*restorearrayandmemtype)(Vec, PetscScalar **);
     99:   PetscErrorCode (*getarrayreadandmemtype)(Vec, const PetscScalar **, PetscMemType *);
    100:   PetscErrorCode (*restorearrayreadandmemtype)(Vec, const PetscScalar **);
    101:   PetscErrorCode (*getarraywriteandmemtype)(Vec, PetscScalar **, PetscMemType *);
    102:   PetscErrorCode (*restorearraywriteandmemtype)(Vec, PetscScalar **, PetscMemType *);
    103:   PetscErrorCode (*concatenate)(PetscInt, const Vec[], Vec *, IS *[]);
    104:   PetscErrorCode (*sum)(Vec, PetscScalar *);
    105:   PetscErrorCode (*setpreallocationcoo)(Vec, PetscCount, const PetscInt[]);
    106:   PetscErrorCode (*setvaluescoo)(Vec, const PetscScalar[], InsertMode);
    107:   PetscErrorCode (*errorwnorm)(Vec, Vec, Vec, NormType, PetscReal, Vec, PetscReal, Vec, PetscReal, PetscReal *, PetscInt *, PetscReal *, PetscInt *, PetscReal *, PetscInt *);
    108:   PetscErrorCode (*maxpby)(Vec, PetscInt, const PetscScalar *, PetscScalar, Vec *); /* y = beta y + alpha[j] x[j] */
    109: };

    111: #if defined(offsetof) && (defined(__cplusplus) || (PETSC_C_VERSION >= 23))
    112: static_assert(offsetof(struct _VecOps, duplicate) == sizeof(PetscErrorCodeFn *) * VECOP_DUPLICATE, "");
    113: static_assert(offsetof(struct _VecOps, set) == sizeof(PetscErrorCodeFn *) * VECOP_SET, "");
    114: static_assert(offsetof(struct _VecOps, view) == sizeof(PetscErrorCodeFn *) * VECOP_VIEW, "");
    115: static_assert(offsetof(struct _VecOps, load) == sizeof(PetscErrorCodeFn *) * VECOP_LOAD, "");
    116: static_assert(offsetof(struct _VecOps, viewnative) == sizeof(PetscErrorCodeFn *) * VECOP_VIEWNATIVE, "");
    117: static_assert(offsetof(struct _VecOps, loadnative) == sizeof(PetscErrorCodeFn *) * VECOP_LOADNATIVE, "");
    118: #endif

    120: /*
    121:     The stash is used to temporarily store inserted vec values that
    122:   belong to another processor. During the assembly phase the stashed
    123:   values are moved to the correct processor and
    124: */

    126: typedef struct {
    127:   PetscInt     nmax;     /* maximum stash size */
    128:   PetscInt     umax;     /* max stash size user wants */
    129:   PetscInt     oldnmax;  /* the nmax value used previously */
    130:   PetscInt     n;        /* stash size */
    131:   PetscInt     bs;       /* block size of the stash */
    132:   PetscInt     reallocs; /* preserve the no of mallocs invoked */
    133:   PetscInt    *idx;      /* global row numbers in stash */
    134:   PetscScalar *array;    /* array to hold stashed values */
    135:   /* The following variables are used for communication */
    136:   MPI_Comm     comm;
    137:   PetscMPIInt  size, rank;
    138:   PetscMPIInt  tag1, tag2;
    139:   MPI_Request *send_waits;        /* array of send requests */
    140:   MPI_Request *recv_waits;        /* array of receive requests */
    141:   MPI_Status  *send_status;       /* array of send status */
    142:   PetscMPIInt  nsends, nrecvs;    /* numbers of sends and receives */
    143:   PetscScalar *svalues, *rvalues; /* sending and receiving data */
    144:   PetscInt    *sindices, *rindices;
    145:   PetscInt     rmax;       /* maximum message length */
    146:   PetscInt    *nprocs;     /* tmp data used both during scatterbegin and end */
    147:   PetscMPIInt  nprocessed; /* number of messages already processed */
    148:   PetscBool    donotstash;
    149:   PetscBool    ignorenegidx; /* ignore negative indices passed into VecSetValues/VetGetValues */
    150:   InsertMode   insertmode;
    151:   PetscInt    *bowners;
    152: } VecStash;

    154: struct _p_Vec {
    155:   PETSCHEADER(struct _VecOps);
    156:   PetscLayout map;
    157:   void       *data; /* implementation-specific data */
    158:   PetscBool   array_gotten;
    159:   VecStash    stash, bstash; /* used for storing off-proc values during assembly */
    160:   PetscBool   petscnative;   /* means the ->data starts with VECHEADER and can use VecGetArrayFast()*/
    161:   PetscInt    lock;          /* lock state. vector can be free (=0), locked for read (>0) or locked for write(<0) */
    162: #if PetscDefined(USE_DEBUG)
    163:   PetscStack lockstack; /* the file,func,line of where locks are added */
    164: #endif
    165:   PetscOffloadMask offloadmask; /* a mask which indicates where the valid vector data is (GPU, CPU or both) */
    166: #if defined(PETSC_HAVE_DEVICE)
    167:   void     *spptr; /* this is the special pointer to the array on the GPU */
    168:   PetscBool boundtocpu;
    169:   PetscBool bindingpropagates;
    170:   size_t    minimum_bytes_pinned_memory; /* minimum data size in bytes for which pinned memory will be allocated */
    171:   PetscBool pinned_memory;               /* PETSC_TRUE if the current host allocation has been made from pinned memory. */
    172: #endif
    173:   char *defaultrandtype;
    174: };

    176: PETSC_EXTERN PetscLogEvent VEC_SetRandom;
    177: PETSC_EXTERN PetscLogEvent VEC_View;
    178: PETSC_EXTERN PetscLogEvent VEC_Max;
    179: PETSC_EXTERN PetscLogEvent VEC_Min;
    180: PETSC_EXTERN PetscLogEvent VEC_Dot;
    181: PETSC_EXTERN PetscLogEvent VEC_MDot;
    182: PETSC_EXTERN PetscLogEvent VEC_TDot;
    183: PETSC_EXTERN PetscLogEvent VEC_MTDot;
    184: PETSC_EXTERN PetscLogEvent VEC_Norm;
    185: PETSC_EXTERN PetscLogEvent VEC_Normalize;
    186: PETSC_EXTERN PetscLogEvent VEC_Scale;
    187: PETSC_EXTERN PetscLogEvent VEC_Shift;
    188: PETSC_EXTERN PetscLogEvent VEC_Copy;
    189: PETSC_EXTERN PetscLogEvent VEC_Set;
    190: PETSC_EXTERN PetscLogEvent VEC_AXPY;
    191: PETSC_EXTERN PetscLogEvent VEC_AYPX;
    192: PETSC_EXTERN PetscLogEvent VEC_WAXPY;
    193: PETSC_EXTERN PetscLogEvent VEC_MAXPY;
    194: PETSC_EXTERN PetscLogEvent VEC_AssemblyEnd;
    195: PETSC_EXTERN PetscLogEvent VEC_PointwiseMult;
    196: PETSC_EXTERN PetscLogEvent VEC_PointwiseDivide;
    197: PETSC_EXTERN PetscLogEvent VEC_Reciprocal;
    198: PETSC_EXTERN PetscLogEvent VEC_SetValues;
    199: PETSC_EXTERN PetscLogEvent VEC_SetPreallocateCOO;
    200: PETSC_EXTERN PetscLogEvent VEC_SetValuesCOO;
    201: PETSC_EXTERN PetscLogEvent VEC_Load;
    202: PETSC_EXTERN PetscLogEvent VEC_ScatterBegin;
    203: PETSC_EXTERN PetscLogEvent VEC_ScatterEnd;
    204: PETSC_EXTERN PetscLogEvent VEC_ReduceArithmetic;
    205: PETSC_EXTERN PetscLogEvent VEC_ReduceCommunication;
    206: PETSC_EXTERN PetscLogEvent VEC_ReduceBegin;
    207: PETSC_EXTERN PetscLogEvent VEC_ReduceEnd;
    208: PETSC_EXTERN PetscLogEvent VEC_Swap;
    209: PETSC_EXTERN PetscLogEvent VEC_AssemblyBegin;
    210: PETSC_EXTERN PetscLogEvent VEC_DotNorm2;
    211: PETSC_EXTERN PetscLogEvent VEC_AXPBYPCZ;
    212: PETSC_EXTERN PetscLogEvent VEC_Ops;
    213: PETSC_EXTERN PetscLogEvent VEC_ViennaCLCopyToGPU;
    214: PETSC_EXTERN PetscLogEvent VEC_ViennaCLCopyFromGPU;
    215: PETSC_EXTERN PetscLogEvent VEC_CUDACopyToGPU;
    216: PETSC_EXTERN PetscLogEvent VEC_CUDACopyFromGPU;
    217: PETSC_EXTERN PetscLogEvent VEC_HIPCopyToGPU;
    218: PETSC_EXTERN PetscLogEvent VEC_HIPCopyFromGPU;

    220: PETSC_SINGLE_LIBRARY_INTERN PetscErrorCode VecView_Seq(Vec, PetscViewer);
    221: #if defined(PETSC_HAVE_VIENNACL)
    222: PETSC_EXTERN PetscErrorCode VecViennaCLAllocateCheckHost(Vec v);
    223: PETSC_EXTERN PetscErrorCode VecViennaCLCopyFromGPU(Vec v);
    224: #endif

    226: /*
    227:      Common header shared by array based vectors,
    228:    currently Vec_Seq and Vec_MPI
    229: */
    230: #define VECHEADER \
    231:   PetscScalar *array; \
    232:   PetscScalar *array_allocated; /* if the array was allocated by PETSc this is its pointer */ \
    233:   PetscScalar *unplacedarray;   /* if one called VecPlaceArray(), this is where it stashed the original */

    235: /* Get Root type of vector. e.g. VECSEQ -> VECSTANDARD, VECMPICUDA -> VECCUDA */
    236: PETSC_EXTERN PetscErrorCode VecGetRootType_Private(Vec, VecType *);

    238: /* Default obtain and release vectors; can be used by any implementation */
    239: PETSC_INTERN PetscErrorCode VecDuplicateVecs_Default(Vec, PetscInt, Vec *[]);
    240: PETSC_INTERN PetscErrorCode VecDestroyVecs_Default(PetscInt, Vec[]);
    241: PETSC_INTERN PetscErrorCode VecView_Binary(Vec, PetscViewer);

    243: PETSC_SINGLE_LIBRARY_INTERN PetscErrorCode VecLoad_Default(Vec, PetscViewer);

    245: PETSC_INTERN PetscInt NormIds[4]; /* map from NormType to IDs used to cache/retrieve values of norms, 1_AND_2 is excluded */

    247: PETSC_INTERN PetscErrorCode VecStashCreate_Private(MPI_Comm, PetscInt, VecStash *);
    248: PETSC_INTERN PetscErrorCode VecStashDestroy_Private(VecStash *);
    249: PETSC_INTERN PetscErrorCode VecStashScatterEnd_Private(VecStash *);
    250: PETSC_INTERN PetscErrorCode VecStashSetInitialSize_Private(VecStash *, PetscInt);
    251: PETSC_INTERN PetscErrorCode VecStashGetInfo_Private(VecStash *, PetscInt *, PetscInt *);
    252: PETSC_INTERN PetscErrorCode VecStashScatterBegin_Private(VecStash *, const PetscInt *);
    253: PETSC_INTERN PetscErrorCode VecStashScatterGetMesg_Private(VecStash *, PetscMPIInt *, PetscInt **, PetscScalar **, PetscInt *);
    254: PETSC_INTERN PetscErrorCode VecStashSortCompress_Private(VecStash *);
    255: PETSC_INTERN PetscErrorCode VecStashGetOwnerList_Private(VecStash *, PetscLayout, PetscMPIInt *, PetscMPIInt **);

    257: PETSC_INTERN PetscErrorCode VecStashExpand_Private(VecStash *, PetscInt);

    259: /*
    260:   VecStashValue_Private - inserts a single value into the stash.

    262:   Input Parameters:
    263:   stash  - the stash
    264:   idx    - the global of the inserted value
    265:   values - the value inserted
    266: */
    267: static inline PetscErrorCode VecStashValue_Private(VecStash *stash, PetscInt row, PetscScalar value)
    268: {
    269:   /* Check and see if we have sufficient memory */
    270:   PetscFunctionBegin;
    271:   if (((stash)->n + 1) > (stash)->nmax) PetscCall(VecStashExpand_Private(stash, 1));
    272:   (stash)->idx[(stash)->n]   = row;
    273:   (stash)->array[(stash)->n] = value;
    274:   (stash)->n++;
    275:   PetscFunctionReturn(PETSC_SUCCESS);
    276: }

    278: /*
    279:   VecStashValuesBlocked_Private - inserts 1 block of values into the stash.

    281:   Input Parameters:
    282:   stash  - the stash
    283:   idx    - the global block index
    284:   values - the values inserted
    285: */
    286: static inline PetscErrorCode VecStashValuesBlocked_Private(VecStash *stash, PetscInt row, PetscScalar *values)
    287: {
    288:   PetscInt     stash_bs = (stash)->bs;
    289:   PetscScalar *array;

    291:   PetscFunctionBegin;
    292:   if (((stash)->n + 1) > (stash)->nmax) PetscCall(VecStashExpand_Private(stash, 1));
    293:   array                    = (stash)->array + stash_bs * (stash)->n;
    294:   (stash)->idx[(stash)->n] = row;
    295:   if (values) PetscCall(PetscArraycpy(array, values, stash_bs));
    296:   else PetscCall(PetscArrayzero(array, stash_bs));
    297:   (stash)->n++;
    298:   PetscFunctionReturn(PETSC_SUCCESS);
    299: }

    301: PETSC_INTERN PetscErrorCode VecStrideGather_Default(Vec, PetscInt, Vec, InsertMode);
    302: PETSC_INTERN PetscErrorCode VecStrideScatter_Default(Vec, PetscInt, Vec, InsertMode);
    303: PETSC_INTERN PetscErrorCode VecStrideSubSetGather_Default(Vec, PetscInt, const PetscInt[], const PetscInt[], Vec, InsertMode);
    304: PETSC_INTERN PetscErrorCode VecStrideSubSetScatter_Default(Vec, PetscInt, const PetscInt[], const PetscInt[], Vec, InsertMode);

    306: PETSC_INTERN PetscErrorCode VecReciprocal_Default(Vec);
    307: #if defined(PETSC_HAVE_MATLAB)
    308: PETSC_EXTERN PetscErrorCode VecMatlabEnginePut_Default(PetscObject, void *);
    309: PETSC_EXTERN PetscErrorCode VecMatlabEngineGet_Default(PetscObject, void *);
    310: #endif

    312: PETSC_SINGLE_LIBRARY_INTERN PetscErrorCode PetscSectionGetField_Internal(PetscSection, PetscSection, Vec, PetscInt, PetscInt, PetscInt, IS *, Vec *);
    313: PETSC_SINGLE_LIBRARY_INTERN PetscErrorCode PetscSectionRestoreField_Internal(PetscSection, PetscSection, Vec, PetscInt, PetscInt, PetscInt, IS *, Vec *);

    315: #define VecCheckSameLocalSize(x, ar1, y, ar2) \
    316:   do { \
    317:     PetscCheck((x)->map->n == (y)->map->n, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Incompatible vector local lengths parameter # %d local size %" PetscInt_FMT " != parameter # %d local size %" PetscInt_FMT, ar1, (x)->map->n, ar2, (y)->map->n); \
    318:   } while (0)

    320: #define VecCheckSameSize(x, ar1, y, ar2) \
    321:   do { \
    322:     PetscCheck((x)->map->N == (y)->map->N, PetscObjectComm((PetscObject)x), PETSC_ERR_ARG_INCOMP, "Incompatible vector global lengths parameter # %d global size %" PetscInt_FMT " != parameter # %d global size %" PetscInt_FMT, ar1, (x)->map->N, ar2, \
    323:                (y)->map->N); \
    324:     VecCheckSameLocalSize(x, ar1, y, ar2); \
    325:   } while (0)

    327: #define VecCheckLocalSize(x, ar1, n) \
    328:   do { \
    329:     PetscCheck((x)->map->n == (n), PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Incorrect vector local size: parameter # %d local size %" PetscInt_FMT " != %" PetscInt_FMT, ar1, (x)->map->n, n); \
    330:   } while (0)

    332: #define VecCheckSize(x, ar1, n, N) \
    333:   do { \
    334:     PetscCheck((x)->map->N == (N), PetscObjectComm((PetscObject)x), PETSC_ERR_ARG_INCOMP, "Incorrect vector global size: parameter # %d global size %" PetscInt_FMT " != %" PetscInt_FMT, ar1, (x)->map->N, N); \
    335:     VecCheckLocalSize(x, ar1, n); \
    336:   } while (0)

    338: typedef struct _VecTaggerOps *VecTaggerOps;
    339: struct _VecTaggerOps {
    340:   PetscErrorCode (*create)(VecTagger);
    341:   PetscErrorCode (*destroy)(VecTagger);
    342:   PetscErrorCode (*setfromoptions)(VecTagger, PetscOptionItems);
    343:   PetscErrorCode (*setup)(VecTagger);
    344:   PetscErrorCode (*view)(VecTagger, PetscViewer);
    345:   PetscErrorCode (*computeboxes)(VecTagger, Vec, PetscInt *, VecTaggerBox **, PetscBool *);
    346:   PetscErrorCode (*computeis)(VecTagger, Vec, IS *, PetscBool *);
    347: };
    348: struct _p_VecTagger {
    349:   PETSCHEADER(struct _VecTaggerOps);
    350:   void     *data;
    351:   PetscInt  blocksize;
    352:   PetscBool invert;
    353:   PetscBool setupcalled;
    354: };

    356: PETSC_INTERN PetscBool      VecTaggerRegisterAllCalled;
    357: PETSC_INTERN PetscErrorCode VecTaggerComputeIS_FromBoxes(VecTagger, Vec, IS *, PetscBool *);
    358: PETSC_INTERN PetscMPIInt    Petsc_Reduction_keyval;

    360: PETSC_INTERN PetscInt       VecGetSubVectorSavedStateId;
    361: PETSC_INTERN PetscErrorCode VecGetSubVectorContiguityAndBS_Private(Vec, IS, PetscBool *, PetscInt *, PetscInt *);
    362: PETSC_INTERN PetscErrorCode VecGetSubVectorThroughVecScatter_Private(Vec, IS, PetscInt, Vec *);

    364: #if PetscDefined(HAVE_CUDA)
    365: PETSC_INTERN PetscErrorCode VecCreate_CUDA(Vec);
    366: PETSC_INTERN PetscErrorCode VecCreate_SeqCUDA(Vec);
    367: PETSC_INTERN PetscErrorCode VecCreate_MPICUDA(Vec);
    368: PETSC_INTERN PetscErrorCode VecCUDAGetArrays_Private(Vec, const PetscScalar **, const PetscScalar **, PetscOffloadMask *);
    369: PETSC_INTERN PetscErrorCode VecConvert_Seq_SeqCUDA_inplace(Vec);
    370: PETSC_INTERN PetscErrorCode VecConvert_MPI_MPICUDA_inplace(Vec);
    371: #endif

    373: #if PetscDefined(HAVE_HIP)
    374: PETSC_INTERN PetscErrorCode VecCreate_HIP(Vec);
    375: PETSC_INTERN PetscErrorCode VecCreate_SeqHIP(Vec);
    376: PETSC_INTERN PetscErrorCode VecCreate_MPIHIP(Vec);
    377: PETSC_INTERN PetscErrorCode VecHIPGetArrays_Private(Vec, const PetscScalar **, const PetscScalar **, PetscOffloadMask *);
    378: PETSC_INTERN PetscErrorCode VecConvert_Seq_SeqHIP_inplace(Vec);
    379: PETSC_INTERN PetscErrorCode VecConvert_MPI_MPIHIP_inplace(Vec);
    380: #endif

    382: #if defined(PETSC_HAVE_KOKKOS)
    383: PETSC_INTERN PetscErrorCode VecCreateMPIKokkosWithArrays_Private(MPI_Comm, PetscInt, PetscInt, PetscInt, const PetscScalar *, const PetscScalar *, Vec *);
    384: PETSC_INTERN PetscErrorCode VecConvert_Seq_SeqKokkos_inplace(Vec);
    385: PETSC_INTERN PetscErrorCode VecConvert_MPI_MPIKokkos_inplace(Vec);
    386: #endif

    388: PETSC_SINGLE_LIBRARY_INTERN PetscErrorCode VecCreateWithLayout_Private(PetscLayout, Vec *);
    389: PETSC_SINGLE_LIBRARY_INTERN PetscErrorCode VecCreateSeqWithLayoutAndArray_Private(PetscLayout, const PetscScalar *, Vec *);
    390: PETSC_SINGLE_LIBRARY_INTERN PetscErrorCode VecCreateMPIWithLayoutAndArray_Private(PetscLayout, const PetscScalar *, Vec *);

    392: /* std::upper_bound(): Given a sorted array, return index of the first element in range [first,last) whose value
    393:    is greater than value, or last if there is no such element.
    394: */
    395: static inline PetscErrorCode PetscSortedIntUpperBound(const PetscInt *array, PetscCount first, PetscCount last, PetscInt value, PetscCount *upper)
    396: {
    397:   PetscCount count = last - first;

    399:   PetscFunctionBegin;
    400:   while (count > 0) {
    401:     const PetscCount step = count / 2;
    402:     PetscCount       it   = first + step;

    404:     if (value < array[it]) {
    405:       count = step;
    406:     } else {
    407:       first = it + 1;
    408:       count -= step + 1;
    409:     }
    410:   }
    411:   *upper = first;
    412:   PetscFunctionReturn(PETSC_SUCCESS);
    413: }

    415: #define VEC_ASYNC_FN_NAME(Base) "Vec" Base "Async_Private_C"
    416: #define VecAsyncFnName(Base)    VEC_##Base##_ASYNC_FN_NAME

    418: #define VEC_Abs_ASYNC_FN_NAME             VEC_ASYNC_FN_NAME("Abs")
    419: #define VEC_AXPBY_ASYNC_FN_NAME           VEC_ASYNC_FN_NAME("AXPBY")
    420: #define VEC_AXPBYPCZ_ASYNC_FN_NAME        VEC_ASYNC_FN_NAME("AXPBYPCZ")
    421: #define VEC_AXPY_ASYNC_FN_NAME            VEC_ASYNC_FN_NAME("AXPY")
    422: #define VEC_AYPX_ASYNC_FN_NAME            VEC_ASYNC_FN_NAME("AYPX")
    423: #define VEC_Conjugate_ASYNC_FN_NAME       VEC_ASYNC_FN_NAME("Conjugate")
    424: #define VEC_Copy_ASYNC_FN_NAME            VEC_ASYNC_FN_NAME("Copy")
    425: #define VEC_Exp_ASYNC_FN_NAME             VEC_ASYNC_FN_NAME("Exp")
    426: #define VEC_Log_ASYNC_FN_NAME             VEC_ASYNC_FN_NAME("Log")
    427: #define VEC_MAXPY_ASYNC_FN_NAME           VEC_ASYNC_FN_NAME("MAXPY")
    428: #define VEC_PointwiseDivide_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("PointwiseDivide")
    429: #define VEC_PointwiseMax_ASYNC_FN_NAME    VEC_ASYNC_FN_NAME("PointwiseMax")
    430: #define VEC_PointwiseMaxAbs_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("PointwiseMaxAbs")
    431: #define VEC_PointwiseMin_ASYNC_FN_NAME    VEC_ASYNC_FN_NAME("PointwiseMin")
    432: #define VEC_PointwiseMult_ASYNC_FN_NAME   VEC_ASYNC_FN_NAME("PointwiseMult")
    433: #define VEC_Reciprocal_ASYNC_FN_NAME      VEC_ASYNC_FN_NAME("Reciprocal")
    434: #define VEC_Scale_ASYNC_FN_NAME           VEC_ASYNC_FN_NAME("Scale")
    435: #define VEC_Set_ASYNC_FN_NAME             VEC_ASYNC_FN_NAME("Set")
    436: #define VEC_Shift_ASYNC_FN_NAME           VEC_ASYNC_FN_NAME("Shift")
    437: #define VEC_SqrtAbs_ASYNC_FN_NAME         VEC_ASYNC_FN_NAME("SqrtAbs")
    438: #define VEC_Swap_ASYNC_FN_NAME            VEC_ASYNC_FN_NAME("Swap")
    439: #define VEC_WAXPY_ASYNC_FN_NAME           VEC_ASYNC_FN_NAME("WAXPY")

    441: PETSC_INTERN PetscErrorCode VecAbsAsync_Private(Vec, PetscDeviceContext);
    442: PETSC_INTERN PetscErrorCode VecAXPBYAsync_Private(Vec, PetscScalar, PetscScalar, Vec, PetscDeviceContext);
    443: PETSC_INTERN PetscErrorCode VecAXPBYPCZAsync_Private(Vec, PetscScalar, PetscScalar, PetscScalar, Vec, Vec, PetscDeviceContext);
    444: PETSC_INTERN PetscErrorCode VecAXPYAsync_Private(Vec, PetscScalar, Vec, PetscDeviceContext);
    445: PETSC_INTERN PetscErrorCode VecAYPXAsync_Private(Vec, PetscScalar, Vec, PetscDeviceContext);
    446: PETSC_INTERN PetscErrorCode VecConjugateAsync_Private(Vec, PetscDeviceContext);
    447: PETSC_INTERN PetscErrorCode VecCopyAsync_Private(Vec, Vec, PetscDeviceContext);
    448: PETSC_INTERN PetscErrorCode VecExpAsync_Private(Vec, PetscDeviceContext);
    449: PETSC_INTERN PetscErrorCode VecLogAsync_Private(Vec, PetscDeviceContext);
    450: PETSC_INTERN PetscErrorCode VecMAXPYAsync_Private(Vec, PetscInt, const PetscScalar *, Vec[], PetscDeviceContext);
    451: PETSC_INTERN PetscErrorCode VecPointwiseDivideAsync_Private(Vec, Vec, Vec, PetscDeviceContext);
    452: PETSC_INTERN PetscErrorCode VecPointwiseMaxAsync_Private(Vec, Vec, Vec, PetscDeviceContext);
    453: PETSC_INTERN PetscErrorCode VecPointwiseMaxAbsAsync_Private(Vec, Vec, Vec, PetscDeviceContext);
    454: PETSC_INTERN PetscErrorCode VecPointwiseMinAsync_Private(Vec, Vec, Vec, PetscDeviceContext);
    455: PETSC_INTERN PetscErrorCode VecPointwiseMultAsync_Private(Vec, Vec, Vec, PetscDeviceContext);
    456: PETSC_INTERN PetscErrorCode VecReciprocalAsync_Private(Vec, PetscDeviceContext);
    457: PETSC_INTERN PetscErrorCode VecScaleAsync_Private(Vec, PetscScalar, PetscDeviceContext);
    458: PETSC_INTERN PetscErrorCode VecSetAsync_Private(Vec, PetscScalar, PetscDeviceContext);
    459: PETSC_INTERN PetscErrorCode VecShiftAsync_Private(Vec, PetscScalar, PetscDeviceContext);
    460: PETSC_INTERN PetscErrorCode VecSqrtAbsAsync_Private(Vec, PetscDeviceContext);
    461: PETSC_INTERN PetscErrorCode VecSwapAsync_Private(Vec, Vec, PetscDeviceContext);
    462: PETSC_INTERN PetscErrorCode VecWAXPYAsync_Private(Vec, PetscScalar, Vec, Vec, PetscDeviceContext);

    464: #define VecMethodDispatch(v, dctx, async_name, name, async_arg_types, ...) \
    465:   do { \
    466:     PetscErrorCode(*_8_f) async_arg_types = NULL; \
    467:     if (dctx) PetscCall(PetscObjectQueryFunction((PetscObject)(v), async_name, &_8_f)); \
    468:     if (_8_f) { \
    469:       PetscCall((*_8_f)(v, __VA_ARGS__, dctx)); \
    470:     } else { \
    471:       PetscUseTypeMethod(v, name, __VA_ARGS__); \
    472:     } \
    473:   } while (0)

    475: // return in lda the vector's local size aligned to <alignment> bytes, where lda is an integer pointer
    476: #define VecGetLocalSizeAligned(v, alignment, lda) \
    477:   do { \
    478:     PetscInt     n = (v)->map->n; \
    479:     const size_t s = (alignment) / sizeof(PetscScalar); \
    480:     *(lda)         = ((n + s - 1) / s) * s; \
    481:   } while (0)