Actual source code: mark_dcontext.cxx

      1: #include "petscdevice_interface_internal.hpp" /*I <petscdevice.h> I*/

      3: #include <petsc/private/cpp/object_pool.hpp>
      4: #include <petsc/private/cpp/utility.hpp>
      5: #include <petsc/private/cpp/unordered_map.hpp>

      7: #include <algorithm> // std::remove_if(), std::find_if()
      8: #include <vector>
      9: #include <string>
     10: #include <sstream> // std::ostringstream

     12: #if defined(__clang__)
     13: PETSC_PRAGMA_DIAGNOSTIC_IGNORED_BEGIN("-Wgnu-zero-variadic-macro-arguments")
     14: #endif

     16: // ==========================================================================================
     17: // PetscEvent
     18: // ==========================================================================================

     20: class PetscEventConstructor : public Petsc::ConstructorInterface<_n_PetscEvent, PetscEventConstructor> {
     21: public:
     22:   PetscErrorCode construct_(PetscEvent event) const noexcept
     23:   {
     24:     PetscFunctionBegin;
     25:     PetscCall(PetscMemzero(event, sizeof(*event)));
     26:     PetscCall(underlying().reset(event));
     27:     PetscFunctionReturn(PETSC_SUCCESS);
     28:   }

     30:   PetscErrorCode destroy_(PetscEvent event) const noexcept
     31:   {
     32:     PetscFunctionBegin;
     33:     PetscCall(underlying().reset(event));
     34:     PetscFunctionReturn(PETSC_SUCCESS);
     35:   }

     37:   static PetscErrorCode reset_(PetscEvent event) noexcept
     38:   {
     39:     PetscFunctionBegin;
     40:     if (auto &destroy = event->destroy) {
     41:       PetscCall((*destroy)(event));
     42:       destroy = nullptr;
     43:     }
     44:     PetscAssert(!event->data, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Event failed to destroy its data member: %p", event->data);
     45:     event->dctx_id    = 0;
     46:     event->dctx_state = 0;
     47:     event->dtype      = PETSC_DEVICE_DEFAULT();
     48:     PetscFunctionReturn(PETSC_SUCCESS);
     49:   }

     51:   static PetscErrorCode invalidate_(PetscEvent) noexcept { return PETSC_SUCCESS; }
     52: };

     54: static Petsc::ObjectPool<_n_PetscEvent, PetscEventConstructor> event_pool;

     56: static PetscErrorCode PetscDeviceContextCreateEvent_Private(PetscDeviceContext dctx, PetscEvent *event)
     57: {
     58:   PetscFunctionBegin;
     60:   PetscAssertPointer(event, 2);
     61:   PetscCall(event_pool.allocate(event));
     62:   PetscCall(PetscDeviceContextGetDeviceType(dctx, &(*event)->dtype));
     63:   PetscTryTypeMethod(dctx, createevent, *event);
     64:   PetscFunctionReturn(PETSC_SUCCESS);
     65: }

     67: static PetscErrorCode PetscEventDestroy_Private(PetscEvent *event)
     68: {
     69:   PetscFunctionBegin;
     70:   PetscAssertPointer(event, 1);
     71:   if (*event) PetscCall(event_pool.deallocate(event));
     72:   PetscFunctionReturn(PETSC_SUCCESS);
     73: }

     75: static PetscErrorCode PetscDeviceContextRecordEvent_Private(PetscDeviceContext dctx, PetscEvent event)
     76: {
     77:   PetscObjectId    id;
     78:   PetscObjectState state;

     80:   PetscFunctionBegin;
     82:   PetscAssertPointer(event, 2);
     83:   id    = PetscObjectCast(dctx)->id;
     84:   state = PetscObjectCast(dctx)->state;
     85:   // technically state can never be less than event->dctx_state (only equal) but we include
     86:   // it in the check just in case
     87:   if ((id == event->dctx_id) && (state <= event->dctx_state)) PetscFunctionReturn(PETSC_SUCCESS);
     88:   if (dctx->ops->recordevent) {
     89:     // REVIEW ME:
     90:     // TODO maybe move this to impls, as they can determine whether they can interoperate with
     91:     // other device types more readily
     92:     if (PetscDefined(USE_DEBUG) && (event->dtype != PETSC_DEVICE_HOST)) {
     93:       PetscDeviceType dtype;

     95:       PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
     96:       PetscCheck(event->dtype == dtype, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Event type %s does not match device context type %s", PetscDeviceTypes[event->dtype], PetscDeviceTypes[dtype]);
     97:     }
     98:     PetscUseTypeMethod(dctx, recordevent, event);
     99:   }
    100:   event->dctx_id    = id;
    101:   event->dctx_state = state;
    102:   PetscFunctionReturn(PETSC_SUCCESS);
    103: }

    105: static PetscErrorCode PetscDeviceContextWaitForEvent_Private(PetscDeviceContext dctx, PetscEvent event)
    106: {
    107:   PetscFunctionBegin;
    109:   PetscAssertPointer(event, 2);
    110:   // empty data implies you cannot wait on this event
    111:   if (!event->data) PetscFunctionReturn(PETSC_SUCCESS);
    112:   if (PetscDefined(USE_DEBUG)) {
    113:     const auto      etype = event->dtype;
    114:     PetscDeviceType dtype;

    116:     PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
    117:     PetscCheck(etype == dtype, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Event type %s does not match device context type %s", PetscDeviceTypes[etype], PetscDeviceTypes[dtype]);
    118:   }
    119:   if (PetscObjectCast(dctx)->id == event->dctx_id) PetscFunctionReturn(PETSC_SUCCESS);
    120:   PetscTryTypeMethod(dctx, waitforevent, event);
    121:   PetscFunctionReturn(PETSC_SUCCESS);
    122: }

    124: // ==========================================================================================
    125: // PetscStackFrame
    126: //
    127: // A helper class that (when debugging is enabled) contains the stack frame from which
    128: // PetscDeviceContextMakrIntentFromID(). It is intended to be derived from, since this enables
    129: // empty-base-class optimization to kick in when debugging is disabled.
    130: // ==========================================================================================

    132: template <bool use_debug>
    133: struct PetscStackFrame;

    135: template <>
    136: struct PetscStackFrame</* use_debug = */ true> {
    137:   std::string file{};
    138:   std::string function{};
    139:   int         line{};

    141:   PetscStackFrame() = default;

    143:   PetscStackFrame(const char *file_, const char *func_, int line_) noexcept : file(split_on_petsc_path_(file_)), function(func_), line(line_) { }

    145:   bool operator==(const PetscStackFrame &other) const noexcept { return line == other.line && file == other.file && function == other.function; }

    147:   PETSC_NODISCARD std::string to_string() const noexcept
    148:   {
    149:     std::string ret;

    151:     ret = '(' + function + "() at " + file + ':' + std::to_string(line) + ')';
    152:     return ret;
    153:   }

    155: private:
    156:   static std::string split_on_petsc_path_(std::string &&in) noexcept
    157:   {
    158:     auto pos = in.find("petsc/src");

    160:     if (pos == std::string::npos) pos = in.find("petsc/include");
    161:     if (pos == std::string::npos) pos = 0;
    162:     return in.substr(pos);
    163:   }

    165:   friend std::ostream &operator<<(std::ostream &os, const PetscStackFrame &frame)
    166:   {
    167:     os << frame.to_string();
    168:     return os;
    169:   }

    171:   friend void swap(PetscStackFrame &lhs, PetscStackFrame &rhs) noexcept
    172:   {
    173:     using std::swap;

    175:     swap(lhs.file, rhs.file);
    176:     swap(lhs.function, rhs.function);
    177:     swap(lhs.line, rhs.line);
    178:   }
    179: };

    181: template <>
    182: struct PetscStackFrame</* use_debug = */ false> {
    183:   template <typename... T>
    184:   constexpr PetscStackFrame(T &&...) noexcept
    185:   {
    186:   }

    188:   constexpr bool operator==(const PetscStackFrame &) const noexcept { return true; }

    190:   PETSC_NODISCARD static std::string to_string() noexcept { return "(unknown)"; }

    192:   friend std::ostream &operator<<(std::ostream &os, const PetscStackFrame &) noexcept
    193:   {
    194:     os << "(unknown)";
    195:     return os;
    196:   }
    197: };

    199: // ==========================================================================================
    200: // MarkedObjectMap
    201: //
    202: // A mapping from a PetscObjectId to a PetscEvent and (if debugging is enabled) a
    203: // PetscStackFrame containing the location where PetscDeviceContextMarkIntentFromID was called
    204: // ==========================================================================================

    206: class MarkedObjectMap : public Petsc::RegisterFinalizeable<MarkedObjectMap> {
    207: public:
    208:   // Note we derive from PetscStackFrame so that the empty base class optimization can kick
    209:   // in. If it were just a member it would still take up storage in optimized builds
    210:   class snapshot_type : private PetscStackFrame<PetscDefined(USE_DEBUG) && !PetscDefined(HAVE_THREADSAFETY)> {
    211:   public:
    212:     using frame_type = PetscStackFrame<PetscDefined(USE_DEBUG) && !PetscDefined(HAVE_THREADSAFETY)>;

    214:     snapshot_type() = default;
    215:     snapshot_type(PetscDeviceContext, frame_type) noexcept;

    217:     ~snapshot_type() noexcept;

    219:     // movable
    220:     snapshot_type(snapshot_type &&) noexcept;
    221:     snapshot_type &operator=(snapshot_type &&) noexcept;

    223:     // not copyable
    224:     snapshot_type(const snapshot_type &) noexcept            = delete;
    225:     snapshot_type &operator=(const snapshot_type &) noexcept = delete;

    227:     PETSC_NODISCARD PetscEvent        event() const noexcept { return event_; }
    228:     PETSC_NODISCARD const frame_type &frame() const noexcept { return *this; }
    229:     PETSC_NODISCARD frame_type       &frame() noexcept { return *this; }

    231:     PETSC_NODISCARD PetscObjectId dctx_id() const noexcept
    232:     {
    233:       PetscFunctionBegin;
    234:       PetscAssertAbort(event(), PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Snapshot %s does not contain an event!", frame().to_string().c_str());
    235:       PetscFunctionReturn(event()->dctx_id);
    236:     }

    238:     PetscErrorCode ensure_event(PetscDeviceContext) noexcept;

    240:     friend void swap(snapshot_type &, snapshot_type &) noexcept;

    242:   private:
    243:     PetscEvent event_{}; // the state of device context when this snapshot was recorded

    245:     PETSC_NODISCARD static PetscEvent init_event_(PetscDeviceContext) noexcept;
    246:   };

    248:   // the "value" each key maps to
    249:   struct mapped_type {
    250:     using dependency_type = std::vector<snapshot_type>;

    252:     mapped_type() noexcept;

    254:     PetscMemoryAccessMode mode{PETSC_MEMORY_ACCESS_READ};
    255:     snapshot_type         last_write{};
    256:     dependency_type       dependencies{};
    257:   };

    259:   using map_type = Petsc::UnorderedMap<PetscObjectId, mapped_type>;

    261:   map_type map;

    263: private:
    264:   friend RegisterFinalizeable;

    266:   PetscErrorCode finalize_() noexcept;
    267: };

    269: // ==========================================================================================
    270: // MarkedObjectMap::mapped_type -- Public API
    271: // ==========================================================================================

    273: // workaround for clang bug that produces the following warning
    274: //
    275: // src/sys/objects/device/interface/mark_dcontext.cxx:253:5: error: default member initializer
    276: // for 'mode' needed within definition of enclosing class 'MarkedObjectMap' outside of member
    277: // functions
    278: //     mapped_type() noexcept = default;
    279: //     ^
    280: // https://stackoverflow.com/questions/53408962/try-to-understand-compiler-error-message-default-member-initializer-required-be
    281: MarkedObjectMap::mapped_type::mapped_type() noexcept = default;

    283: // ==========================================================================================
    284: // MarkedObjectMap Private API
    285: // ==========================================================================================

    287: inline PetscErrorCode MarkedObjectMap::finalize_() noexcept
    288: {
    289:   PetscFunctionBegin;
    290:   PetscCall(PetscInfo(nullptr, "Finalizing marked object map\n"));
    291:   PetscCall(map.clear());
    292:   PetscFunctionReturn(PETSC_SUCCESS);
    293: }

    295: // ==========================================================================================
    296: // MarkedObjectMap::snapshot_type Private API
    297: // ==========================================================================================

    299: inline PetscEvent MarkedObjectMap::snapshot_type::init_event_(PetscDeviceContext dctx) noexcept
    300: {
    301:   PetscEvent event = nullptr;

    303:   PetscFunctionBegin;
    304:   PetscCallAbort(PETSC_COMM_SELF, PetscDeviceContextCreateEvent_Private(dctx, &event));
    305:   PetscCallAbort(PETSC_COMM_SELF, PetscDeviceContextRecordEvent_Private(dctx, event));
    306:   PetscFunctionReturn(event);
    307: }

    309: // ==========================================================================================
    310: // MarkedObjectMap::snapshot_type Public API
    311: // ==========================================================================================

    313: MarkedObjectMap::snapshot_type::snapshot_type(PetscDeviceContext dctx, frame_type frame) noexcept : frame_type(std::move(frame)), event_(init_event_(dctx)) { }

    315: MarkedObjectMap::snapshot_type::~snapshot_type() noexcept
    316: {
    317:   PetscFunctionBegin;
    318:   PetscCallAbort(PETSC_COMM_SELF, PetscEventDestroy_Private(&event_));
    319:   PetscFunctionReturnVoid();
    320: }

    322: // movable
    323: MarkedObjectMap::snapshot_type::snapshot_type(snapshot_type &&other) noexcept : frame_type(std::move(other)), event_(Petsc::util::exchange(other.event_, nullptr)) { }

    325: MarkedObjectMap::snapshot_type &MarkedObjectMap::snapshot_type::operator=(snapshot_type &&other) noexcept
    326: {
    327:   PetscFunctionBegin;
    328:   if (this != &other) {
    329:     frame_type::operator=(std::move(other));
    330:     PetscCallAbort(PETSC_COMM_SELF, PetscEventDestroy_Private(&event_));
    331:     event_ = Petsc::util::exchange(other.event_, nullptr);
    332:   }
    333:   PetscFunctionReturn(*this);
    334: }

    336: PetscErrorCode MarkedObjectMap::snapshot_type::ensure_event(PetscDeviceContext dctx) noexcept
    337: {
    338:   PetscFunctionBegin;
    339:   if (PetscUnlikely(!event_)) PetscCall(PetscDeviceContextCreateEvent_Private(dctx, &event_));
    340:   PetscFunctionReturn(PETSC_SUCCESS);
    341: }

    343: void swap(MarkedObjectMap::snapshot_type &lhs, MarkedObjectMap::snapshot_type &rhs) noexcept
    344: {
    345:   using std::swap;

    347:   swap(lhs.frame(), rhs.frame());
    348:   swap(lhs.event_, rhs.event_);
    349: }

    351: // A mapping between PetscObjectId (i.e. some PetscObject) to the list of PetscEvent's encoding
    352: // the last time the PetscObject was accessed
    353: static MarkedObjectMap marked_object_map;

    355: // ==========================================================================================
    356: // Utility Functions
    357: // ==========================================================================================

    359: PetscErrorCode PetscGetMarkedObjectMap_Internal(std::size_t *nkeys, PetscObjectId **keys, PetscMemoryAccessMode **modes, std::size_t **ndeps, PetscEvent ***dependencies)
    360: {
    361:   std::size_t i    = 0;
    362:   const auto &map  = marked_object_map.map;
    363:   const auto  size = *nkeys = map.size();

    365:   PetscFunctionBegin;
    366:   PetscCall(PetscMalloc4(size, keys, size, modes, size, ndeps, size, dependencies));
    367:   for (auto it_ = map.begin(); it_ != map.end(); ++it_) {
    368:     auto       &it = *it_;
    369:     std::size_t j  = 0;

    371:     (*keys)[i]         = it.first;
    372:     (*modes)[i]        = it.second.mode;
    373:     (*ndeps)[i]        = it.second.dependencies.size();
    374:     (*dependencies)[i] = nullptr;
    375:     PetscCall(PetscMalloc1((*ndeps)[i], (*dependencies) + i));
    376:     for (auto &&dep : it.second.dependencies) (*dependencies)[i][j++] = dep.event();
    377:     ++i;
    378:   }
    379:   PetscFunctionReturn(PETSC_SUCCESS);
    380: }

    382: PetscErrorCode PetscRestoreMarkedObjectMap_Internal(std::size_t nkeys, PetscObjectId **keys, PetscMemoryAccessMode **modes, std::size_t **ndeps, PetscEvent ***dependencies)
    383: {
    384:   PetscFunctionBegin;
    385:   for (std::size_t i = 0; i < nkeys; ++i) PetscCall(PetscFree((*dependencies)[i]));
    386:   PetscCall(PetscFree4(*keys, *modes, *ndeps, *dependencies));
    387:   PetscFunctionReturn(PETSC_SUCCESS);
    388: }

    390: template <typename T>
    391: static PetscErrorCode PetscDeviceContextMapIterVisitor(PetscDeviceContext dctx, T &&callback) noexcept
    392: {
    393:   const auto dctx_id    = PetscObjectCast(dctx)->id;
    394:   auto     &&marked     = CxxDataCast(dctx)->marked_objects();
    395:   auto      &object_map = marked_object_map.map;

    397:   PetscFunctionBegin;
    398:   for (auto &&dep : marked) {
    399:     const auto mapit = object_map.find(dep);

    401:     // Need this check since the final PetscDeviceContext may run through this *after* the map
    402:     // has been finalized (and cleared), and hence might fail to find its dependencies. This is
    403:     // perfectly valid since the user no longer cares about dangling dependencies after PETSc
    404:     // is finalized
    405:     if (PetscLikely(mapit != object_map.end())) {
    406:       auto      &deps = mapit->second.dependencies;
    407:       const auto end  = deps.end();
    408:       const auto it   = std::remove_if(deps.begin(), end, [&](const MarkedObjectMap::snapshot_type &obj) { return obj.dctx_id() == dctx_id; });

    410:       PetscCall(callback(mapit, deps.cbegin(), static_cast<decltype(deps.cend())>(it)));
    411:       // remove ourselves
    412:       PetscCallCXX(deps.erase(it, end));
    413:       // continue to next object, but erase this one if it has no more dependencies
    414:       if (deps.empty()) PetscCallCXX(object_map.erase(mapit));
    415:     }
    416:   }
    417:   PetscCallCXX(marked.clear());
    418:   PetscFunctionReturn(PETSC_SUCCESS);
    419: }

    421: PetscErrorCode PetscDeviceContextSyncClearMap_Internal(PetscDeviceContext dctx)
    422: {
    423:   using map_iterator = MarkedObjectMap::map_type::const_iterator;
    424:   using dep_iterator = MarkedObjectMap::mapped_type::dependency_type::const_iterator;

    426:   PetscFunctionBegin;
    427:   PetscCall(PetscDeviceContextMapIterVisitor(dctx, [&](map_iterator mapit, dep_iterator it, dep_iterator end) {
    428:     PetscFunctionBegin;
    429:     if (PetscDefined(USE_DEBUG_AND_INFO)) {
    430:       std::ostringstream oss;
    431:       const auto         mode = PetscMemoryAccessModeToString(mapit->second.mode);

    433:       oss << "synced dctx " << PetscObjectCast(dctx)->id << ", remaining leaves for obj " << mapit->first << ": {";
    434:       while (it != end) {
    435:         oss << "[dctx " << it->dctx_id() << ", " << mode << ' ' << it->frame() << ']';
    436:         if (++it != end) oss << ", ";
    437:       }
    438:       oss << '}';
    439:       PetscCall(PetscInfo(nullptr, "%s\n", oss.str().c_str()));
    440:     }
    441:     PetscFunctionReturn(PETSC_SUCCESS);
    442:   }));
    443:   {
    444:     // the recursive sync clear map call is unbounded in case of a dependenct loop so we make a
    445:     // copy
    446:     const auto cxx_data = CxxDataCast(dctx);
    447:     // clang-format off
    448:     const std::vector<CxxData::upstream_type::value_type> upstream_copy(
    449:       std::make_move_iterator(cxx_data->upstream().begin()),
    450:       std::make_move_iterator(cxx_data->upstream().end())
    451:     );
    452:     // clang-format on

    454:     // aftermath, clear our set of parents (to avoid infinite recursion) and mark ourselves as no
    455:     // longer contained (while the empty graph technically *is* always contained, it is not what
    456:     // we mean by it)
    457:     PetscCall(cxx_data->clear());
    458:     for (auto &&upstrm : upstream_copy) {
    459:       if (const auto udctx = upstrm.second.weak_dctx().lock()) {
    460:         // check that this parent still points to what we originally thought it was
    461:         PetscCheck(upstrm.first == PetscObjectCast(udctx.get())->id, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Upstream dctx %" PetscInt64_FMT " no longer exists, now has id %" PetscInt64_FMT, upstrm.first, PetscObjectCast(udctx.get())->id);
    462:         PetscCall(PetscDeviceContextSyncClearMap_Internal(udctx.get()));
    463:       }
    464:     }
    465:   }
    466:   PetscFunctionReturn(PETSC_SUCCESS);
    467: }

    469: PetscErrorCode PetscDeviceContextCheckNotOrphaned_Internal(PetscDeviceContext dctx)
    470: {
    471:   std::ostringstream oss;
    472:   //const auto         allow = dctx->options.allow_orphans, contained = dctx->contained;
    473:   const auto allow = true, contained = true;
    474:   auto       wrote_to_oss = false;
    475:   using map_iterator      = MarkedObjectMap::map_type::const_iterator;
    476:   using dep_iterator      = MarkedObjectMap::mapped_type::dependency_type::const_iterator;

    478:   PetscFunctionBegin;
    479:   PetscCall(PetscDeviceContextMapIterVisitor(dctx, [&](map_iterator mapit, dep_iterator it, dep_iterator end) {
    480:     PetscFunctionBegin;
    481:     if (allow || contained) PetscFunctionReturn(PETSC_SUCCESS);
    482:     wrote_to_oss = true;
    483:     oss << "- PetscObject (id " << mapit->first << "), intent " << PetscMemoryAccessModeToString(mapit->second.mode) << ' ' << it->frame();
    484:     if (std::distance(it, end) == 0) oss << " (orphaned)"; // we were the only dependency
    485:     oss << '\n';
    486:     PetscFunctionReturn(PETSC_SUCCESS);
    487:   }));
    488:   PetscCheck(!wrote_to_oss, PETSC_COMM_SELF, PETSC_ERR_ORDER, "Destroying PetscDeviceContext ('%s', id %" PetscInt64_FMT ") would leave the following dangling (possibly orphaned) dependents: %s. Must synchronize before destroying it, or allow it to be destroyed with orphans",
    489:              PetscObjectCast(dctx)->name ? PetscObjectCast(dctx)->name : "unnamed", PetscObjectCast(dctx)->id, oss.str().c_str());
    490:   PetscCall(CxxDataCast(dctx)->clear());
    491:   PetscFunctionReturn(PETSC_SUCCESS);
    492: }

    494: #define DEBUG_INFO(mess, ...) PetscDebugInfo(dctx, "dctx %" PetscInt64_FMT " (%s) - obj %" PetscInt64_FMT " (%s): " mess, PetscObjectCast(dctx)->id, PetscObjectCast(dctx)->name ? PetscObjectCast(dctx)->name : "unnamed", id, name, ##__VA_ARGS__)

    496: // The current mode is compatible with the previous mode (i.e. read-read) so we need only
    497: // update the existing version and possibly appeand ourselves to the dependency list

    499: template <bool use_debug>
    500: static PetscErrorCode MarkFromID_CompatibleModes(MarkedObjectMap::mapped_type &marked, PetscDeviceContext dctx, PetscObjectId id, PetscMemoryAccessMode mode, PetscStackFrame<use_debug> &frame, PETSC_UNUSED const char *name, bool *update_object_dependencies)
    501: {
    502:   const auto dctx_id             = PetscObjectCast(dctx)->id;
    503:   auto      &object_dependencies = marked.dependencies;
    504:   const auto end                 = object_dependencies.end();
    505:   const auto it                  = std::find_if(object_dependencies.begin(), end, [&](const MarkedObjectMap::snapshot_type &obj) { return obj.dctx_id() == dctx_id; });

    507:   PetscFunctionBegin;
    508:   PetscCall(DEBUG_INFO("new mode (%s) COMPATIBLE with %s mode (%s), no need to serialize\n", PetscMemoryAccessModeToString(mode), object_dependencies.empty() ? "default" : "old", PetscMemoryAccessModeToString(marked.mode)));
    509:   (void)mode;
    510:   if (it != end) {
    511:     using std::swap;

    513:     // we have been here before, all we must do is update our entry then we can bail
    514:     PetscCall(DEBUG_INFO("found old self as dependency, updating\n"));
    515:     PetscAssert(CxxDataCast(dctx)->has_marked(id), PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscDeviceContext %" PetscInt64_FMT " listed as dependency for object %" PetscInt64_FMT " (%s), but does not have the object in private dependency list!", dctx_id, id, name);
    516:     swap(it->frame(), frame);
    517:     PetscCall(PetscDeviceContextRecordEvent_Private(dctx, it->event()));
    518:     *update_object_dependencies = false;
    519:     PetscFunctionReturn(PETSC_SUCCESS);
    520:   }

    522:   // we have not been here before, need to serialize with the last write event (if it exists)
    523:   // and add ourselves to the dependency list
    524:   if (const auto event = marked.last_write.event()) PetscCall(PetscDeviceContextWaitForEvent_Private(dctx, event));
    525:   PetscFunctionReturn(PETSC_SUCCESS);
    526: }

    528: template <bool use_debug>
    529: static PetscErrorCode MarkFromID_IncompatibleModes_UpdateLastWrite(MarkedObjectMap::mapped_type &marked, PetscDeviceContext dctx, PetscObjectId id, PetscMemoryAccessMode mode, PetscStackFrame<use_debug> &frame, PETSC_UNUSED const char *name, bool *update_object_dependencies)
    530: {
    531:   const auto      dctx_id    = PetscObjectCast(dctx)->id;
    532:   auto           &last_write = marked.last_write;
    533:   auto           &last_dep   = marked.dependencies.back();
    534:   PetscDeviceType dtype;

    536:   PetscFunctionBegin;
    537:   PetscAssert(marked.dependencies.size() == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Can only have a single writer as dependency, have %zu!", marked.dependencies.size());
    538:   PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
    539:   if (last_dep.event()->dtype != dtype) {
    540:     PetscCall(DEBUG_INFO("moving last write dependency (intent %s)\n", PetscMemoryAccessModeToString(marked.mode)));
    541:     last_write = std::move(last_dep);
    542:     PetscFunctionReturn(PETSC_SUCCESS);
    543:   }

    545:   // we match the device type of the dependency, we can reuse its event!
    546:   const auto cxx_data               = CxxDataCast(dctx);
    547:   const auto last_write_was_also_us = last_write.event() && (last_write.dctx_id() == dctx_id);
    548:   using std::swap;

    550:   PetscCall(DEBUG_INFO("we matched the previous write dependency's (intent %s) device type (%s), swapping last dependency with last write\n", PetscMemoryAccessModeToString(marked.mode), PetscDeviceTypes[dtype]));
    551:   if (last_dep.event()->dctx_id != dctx_id) PetscCall(cxx_data->add_mark(id));
    552:   PetscAssert(cxx_data->has_marked(id), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Did not find id %" PetscInt64_FMT "in object dependencies, but we have apparently recorded the last dependency %s!", id, last_write.frame().to_string().c_str());
    553:   swap(last_write, last_dep);
    554:   if (last_write_was_also_us) {
    555:     PetscCall(DEBUG_INFO("we were also the last write event (intent %s), updating\n", PetscMemoryAccessModeToString(mode)));
    556:     (void)mode;
    557:     // we are both the last to write *and* the last to leave a write event. This is the
    558:     // fast path, we only need to update the frame and update the recorded event
    559:     swap(last_dep.frame(), frame);
    560:     // last used to be last_write which is not guaranteed to have an event, so must
    561:     // create it now
    562:     PetscCall(last_dep.ensure_event(dctx));
    563:     PetscCall(PetscDeviceContextRecordEvent_Private(dctx, last_dep.event()));
    564:     *update_object_dependencies = false;
    565:   }
    566:   PetscFunctionReturn(PETSC_SUCCESS);
    567: }

    569: // The current mode is NOT compatible with the previous mode. We must serialize with all events
    570: // in the dependency list, possibly clear it, and update the previous write event

    572: template <bool use_debug>
    573: static PetscErrorCode MarkFromID_IncompatibleModes(MarkedObjectMap::mapped_type &marked, PetscDeviceContext dctx, PetscObjectId id, PetscMemoryAccessMode mode, PetscStackFrame<use_debug> &frame, const char *name, bool *update_object_dependencies)
    574: {
    575:   auto &old_mode            = marked.mode;
    576:   auto &object_dependencies = marked.dependencies;

    578:   PetscFunctionBegin;
    579:   // we are NOT compatible with the previous mode
    580:   PetscCall(DEBUG_INFO("new mode (%s) NOT COMPATIBLE with %s mode (%s), serializing then clearing (%zu) %s\n", PetscMemoryAccessModeToString(mode), object_dependencies.empty() ? "default" : "old", PetscMemoryAccessModeToString(old_mode),
    581:                        object_dependencies.size(), object_dependencies.size() == 1 ? "dependency" : "dependencies"));

    583:   for (const auto &dep : object_dependencies) PetscCall(PetscDeviceContextWaitForEvent_Private(dctx, dep.event()));
    584:   // if the previous mode wrote, update the last write node with it
    585:   if (PetscMemoryAccessWrite(old_mode)) PetscCall(MarkFromID_IncompatibleModes_UpdateLastWrite(marked, dctx, id, mode, frame, name, update_object_dependencies));

    587:   old_mode = mode;
    588:   // clear out the old dependencies if are about to append ourselves
    589:   if (*update_object_dependencies) object_dependencies.clear();
    590:   PetscFunctionReturn(PETSC_SUCCESS);
    591: }

    593: template <bool use_debug>
    594: static PetscErrorCode PetscDeviceContextMarkIntentFromID_Private(PetscDeviceContext dctx, PetscObjectId id, PetscMemoryAccessMode mode, PetscStackFrame<use_debug> frame, const char *name)
    595: {
    596:   auto &marked                     = marked_object_map.map[id];
    597:   auto &object_dependencies        = marked.dependencies;
    598:   auto  update_object_dependencies = true;

    600:   PetscFunctionBegin;
    601:   if ((marked.mode == PETSC_MEMORY_ACCESS_READ) && (mode == PETSC_MEMORY_ACCESS_READ)) {
    602:     PetscCall(MarkFromID_CompatibleModes(marked, dctx, id, mode, frame, name, &update_object_dependencies));
    603:   } else {
    604:     PetscCall(MarkFromID_IncompatibleModes(marked, dctx, id, mode, frame, name, &update_object_dependencies));
    605:   }
    606:   if (update_object_dependencies) {
    607:     // become the new leaf by appending ourselves
    608:     PetscCall(DEBUG_INFO("%s with intent %s\n", object_dependencies.empty() ? "dependency list is empty, creating new leaf" : "appending to existing leaves", PetscMemoryAccessModeToString(mode)));
    609:     PetscCallCXX(object_dependencies.emplace_back(dctx, std::move(frame)));
    610:     PetscCall(CxxDataCast(dctx)->add_mark(id));
    611:   }
    612:   PetscFunctionReturn(PETSC_SUCCESS);
    613: }

    615: #undef DEBUG_INFO

    617: /*@C
    618:   PetscDeviceContextMarkIntentFromID - Indicate a `PetscDeviceContext`s access intent to the
    619:   auto-dependency system

    621:   Not Collective

    623:   Input Parameters:
    624: + dctx - The `PetscDeviceContext`
    625: . id   - The `PetscObjectId` to mark
    626: . mode - The desired access intent
    627: - name - The object name (for debug purposes, ignored in optimized builds)

    629:   Notes:
    630:   This routine formally informs the dependency system that `dctx` will access the object
    631:   represented by `id` with `mode` and adds `dctx` to `id`'s list of dependencies (termed
    632:   "leaves").

    634:   If the existing set of leaves have an incompatible `PetscMemoryAccessMode` to `mode`, `dctx`
    635:   will be serialized against them.

    637:   Level: intermediate

    639: .seealso: `PetscDeviceContextWaitForContext()`, `PetscDeviceContextSynchronize()`,
    640: `PetscObjectGetId()`, `PetscMemoryAccessMode`
    641: @*/
    642: PetscErrorCode PetscDeviceContextMarkIntentFromID(PetscDeviceContext dctx, PetscObjectId id, PetscMemoryAccessMode mode, const char name[])
    643: {
    644: #if PetscDefined(USE_DEBUG) && !PetscDefined(HAVE_THREADSAFETY)
    645:   const auto index    = petscstack.currentsize > 2 ? petscstack.currentsize - 2 : 0;
    646:   const auto file     = petscstack.file[index];
    647:   const auto function = petscstack.function[index];
    648:   const auto line     = petscstack.line[index];
    649: #else
    650:   constexpr const char *file     = nullptr;
    651:   constexpr const char *function = nullptr;
    652:   constexpr auto        line     = 0;
    653: #endif

    655:   PetscFunctionBegin;
    656:   PetscCall(PetscDeviceContextGetOptionalNullContext_Internal(&dctx));
    657:   if (name) PetscAssertPointer(name, 4);
    658:   PetscCall(marked_object_map.register_finalize());
    659:   PetscCall(PetscLogEventBegin(DCONTEXT_Mark, dctx, nullptr, nullptr, nullptr));
    660:   PetscCall(PetscDeviceContextMarkIntentFromID_Private(dctx, id, mode, MarkedObjectMap::snapshot_type::frame_type{file, function, line}, name ? name : "unknown object"));
    661:   PetscCall(PetscLogEventEnd(DCONTEXT_Mark, dctx, nullptr, nullptr, nullptr));
    662:   PetscFunctionReturn(PETSC_SUCCESS);
    663: }

    665: #if defined(__clang__)
    666: PETSC_PRAGMA_DIAGNOSTIC_IGNORED_END()
    667: #endif