scheduler.h
Go to the documentation of this file.
/*
Copyright (c) 2005-2020 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#ifndef _TBB_scheduler_H
#define _TBB_scheduler_H
#include "scheduler_common.h"
#include "tbb/spin_mutex.h"
#include "mailbox.h"
#include "tbb_misc.h" // for FastRandom
#include "itt_notify.h"
#include "../rml/include/rml_tbb.h"
#include "intrusive_list.h"
#if __TBB_SURVIVE_THREAD_SWITCH
#include "cilk-tbb-interop.h"
#endif /* __TBB_SURVIVE_THREAD_SWITCH */
#if __TBB_PREVIEW_RESUMABLE_TASKS
#include "co_context.h"
#endif
namespace tbb {
namespace internal {
template<typename SchedulerTraits> class custom_scheduler;
//------------------------------------------------------------------------
// generic_scheduler
//------------------------------------------------------------------------
#define EmptyTaskPool ((task**)0)
#define LockedTaskPool ((task**)~(intptr_t)0)
struct scheduler_properties {
static const bool worker = false;
static const bool master = true;
bool type : 1;
bool outermost : 1;
#if __TBB_PREVIEW_CRITICAL_TASKS
bool has_taken_critical_task : 1;
#endif
#if __TBB_PREVIEW_RESUMABLE_TASKS
bool genuine : 1;
#endif
unsigned char :
#if __TBB_PREVIEW_RESUMABLE_TASKS
4;
#elif __TBB_PREVIEW_CRITICAL_TASKS
5;
#else
6;
#endif
};
struct scheduler_state {
size_t my_arena_index; // TODO: make it unsigned and pair with my_affinity_id to fit into cache line
arena_slot* my_arena_slot;
arena* my_arena;
task* my_innermost_running_task;
mail_inbox my_inbox;
affinity_id my_affinity_id;
scheduler_properties my_properties;
#if __TBB_SCHEDULER_OBSERVER
observer_proxy* my_last_global_observer;
#endif
#if __TBB_ARENA_OBSERVER
observer_proxy* my_last_local_observer;
#endif
#if __TBB_TASK_PRIORITY
volatile intptr_t *my_ref_top_priority;
volatile uintptr_t *my_ref_reload_epoch;
#endif /* __TBB_TASK_PRIORITY */
#if __TBB_PREVIEW_RESUMABLE_TASKS
task* my_wait_task;
tbb::atomic<bool>* my_current_is_recalled;
#endif
};
class generic_scheduler: public scheduler
, public ::rml::job
, public intrusive_list_node
, public scheduler_state {
public: // almost every class in TBB uses generic_scheduler
static const size_t quick_task_size = 256-task_prefix_reservation_size;
static bool is_version_3_task( task& t ) {
#if __TBB_PREVIEW_CRITICAL_TASKS
return (t.prefix().extra_state & 0x7)>=0x1;
#else
return (t.prefix().extra_state & 0x0F)>=0x1;
#endif
}
uintptr_t my_stealing_threshold;
#if __TBB_ipf
uintptr_t my_rsb_stealing_threshold;
#endif
static const size_t null_arena_index = ~size_t(0);
inline bool is_task_pool_published () const;
inline bool is_local_task_pool_quiescent () const;
inline bool is_quiescent_local_task_pool_empty () const;
inline bool is_quiescent_local_task_pool_reset () const;
market* my_market;
FastRandom my_random;
task* my_free_list;
#if __TBB_HOARD_NONLOCAL_TASKS
task* my_nonlocal_free_list;
#endif
task* my_dummy_task;
long my_ref_count;
inline void attach_mailbox( affinity_id id );
/* A couple of bools can be located here because space is otherwise just padding after my_affinity_id. */
bool my_auto_initialized;
#if __TBB_COUNT_TASK_NODES
intptr_t my_task_node_count;
#endif /* __TBB_COUNT_TASK_NODES */
#if __TBB_PREVIEW_RESUMABLE_TASKS
enum post_resume_action {
PRA_INVALID,
PRA_ABANDON,
PRA_CALLBACK,
PRA_CLEANUP,
PRA_NOTIFY,
PRA_NONE
};
typedef void(*suspend_callback_t)(void*, task::suspend_point);
struct callback_t {
suspend_callback_t suspend_callback;
void* user_callback;
task::suspend_point tag;
void operator()() {
if (suspend_callback) {
__TBB_ASSERT(suspend_callback && user_callback && tag, NULL);
suspend_callback(user_callback, tag);
}
}
};
co_context my_co_context;
post_resume_action my_post_resume_action;
void* my_post_resume_arg;
generic_scheduler* my_target_on_exit;
void set_post_resume_action(post_resume_action, void* arg);
void do_post_resume_action();
bool prepare_resume(generic_scheduler& target);
bool resume_original_scheduler();
void resume(generic_scheduler& target);
friend void recall_function(task::suspend_point tag);
#endif /* __TBB_PREVIEW_RESUMABLE_TASKS */
void init_stack_info ();
bool can_steal () {
int anchor;
// TODO IDEA: Add performance warning?
#if __TBB_ipf
return my_stealing_threshold < (uintptr_t)&anchor && (uintptr_t)__TBB_get_bsp() < my_rsb_stealing_threshold;
#else
return my_stealing_threshold < (uintptr_t)&anchor;
#endif
}
void publish_task_pool();
void leave_task_pool();
inline void reset_task_pool_and_leave ();
task** lock_task_pool( arena_slot* victim_arena_slot ) const;
void unlock_task_pool( arena_slot* victim_arena_slot, task** victim_task_pool ) const;
void acquire_task_pool() const;
void release_task_pool() const;
task* prepare_for_spawning( task* t );
inline void commit_spawned_tasks( size_t new_tail );
inline void commit_relocated_tasks( size_t new_tail );
task* get_task( __TBB_ISOLATION_EXPR( isolation_tag isolation ) );
#if __TBB_TASK_ISOLATION
task* get_task( size_t T, isolation_tag isolation, bool& tasks_omitted );
#else
task* get_task( size_t T );
#endif /* __TBB_TASK_ISOLATION */
task* get_mailbox_task( __TBB_ISOLATION_EXPR( isolation_tag isolation ) );
static bool is_proxy( const task& t ) {
return t.prefix().extra_state==es_task_proxy;
}
task* steal_task( __TBB_ISOLATION_EXPR(isolation_tag isolation) );
task* steal_task_from( __TBB_ISOLATION_ARG( arena_slot& victim_arena_slot, isolation_tag isolation ) );
#if __TBB_PREVIEW_CRITICAL_TASKS
task* get_critical_task( __TBB_ISOLATION_EXPR(isolation_tag isolation) );
bool handled_as_critical( task& t );
#endif
static const size_t min_task_pool_size = 64;
size_t prepare_task_pool( size_t n );
static generic_scheduler* create_master( arena* a );
bool cleanup_master( bool blocking_terminate );
static generic_scheduler* create_worker( market& m, size_t index, bool geniune );
static void cleanup_worker( void* arg, bool worker );
protected:
template<typename SchedulerTraits> friend class custom_scheduler;
generic_scheduler( market &, bool );
public:
#if TBB_USE_ASSERT > 1
void assert_task_pool_valid() const;
#else
void assert_task_pool_valid() const {}
#endif /* TBB_USE_ASSERT <= 1 */
void attach_arena( arena*, size_t index, bool is_master );
void nested_arena_entry( arena*, size_t );
void nested_arena_exit();
void wait_until_empty();
void spawn( task& first, task*& next ) __TBB_override;
void spawn_root_and_wait( task& first, task*& next ) __TBB_override;
void enqueue( task&, void* reserved ) __TBB_override;
void local_spawn( task* first, task*& next );
void local_spawn_root_and_wait( task* first, task*& next );
virtual void local_wait_for_all( task& parent, task* child ) = 0;
void destroy();
void cleanup_scheduler();
task& allocate_task( size_t number_of_bytes,
__TBB_CONTEXT_ARG(task* parent, task_group_context* context) );
template<free_task_hint h>
void free_task( task& t );
inline void deallocate_task( task& t );
inline bool is_worker() const;
inline bool outermost_level() const;
inline bool master_outermost_level () const;
inline bool worker_outermost_level () const;
unsigned max_threads_in_arena();
#if __TBB_COUNT_TASK_NODES
intptr_t get_task_node_count( bool count_arena_workers = false );
#endif /* __TBB_COUNT_TASK_NODES */
static task* plugged_return_list() {return (task*)(intptr_t)(-1);}
__TBB_atomic intptr_t my_small_task_count;
// TODO IDEA: see if putting my_return_list on separate cache line improves performance
task* my_return_list;
virtual task* receive_or_steal_task( __TBB_ISOLATION_ARG( __TBB_atomic reference_count& completion_ref_count, isolation_tag isolation ) ) = 0;
void free_nonlocal_small_task( task& t );
#if __TBB_TASK_GROUP_CONTEXT
inline task_group_context* default_context ();
char _padding1[NFS_MaxLineSize - sizeof(context_list_node_t)];
context_list_node_t my_context_list_head;
// TODO: check whether it can be deadly preempted and replace by spinning/sleeping mutex
spin_mutex my_context_list_mutex;
uintptr_t my_context_state_propagation_epoch;
tbb::atomic<uintptr_t> my_local_ctx_list_update;
#if __TBB_TASK_PRIORITY
inline intptr_t effective_reference_priority () const;
// TODO: move into slots and fix is_out_of_work
task* my_offloaded_tasks;
task** my_offloaded_task_list_tail_link;
uintptr_t my_local_reload_epoch;
volatile bool my_pool_reshuffling_pending;
task* reload_tasks( __TBB_ISOLATION_EXPR( isolation_tag isolation ) );
task* reload_tasks( task*& offloaded_tasks, task**& offloaded_task_list_link, __TBB_ISOLATION_ARG( intptr_t top_priority, isolation_tag isolation ) );
task* winnow_task_pool ( __TBB_ISOLATION_EXPR( isolation_tag isolation ) );
task *get_task_and_activate_task_pool( size_t H0 , __TBB_ISOLATION_ARG( size_t T0, isolation_tag isolation ) );
inline void offload_task ( task& t, intptr_t task_priority );
#endif /* __TBB_TASK_PRIORITY */
void cleanup_local_context_list ();
template <typename T>
void propagate_task_group_state ( T task_group_context::*mptr_state, task_group_context& src, T new_state );
// check consistency
static void assert_context_valid(const task_group_context *tgc) {
suppress_unused_warning(tgc);
#if TBB_USE_ASSERT
__TBB_ASSERT(tgc, NULL);
uintptr_t ctx = tgc->my_version_and_traits;
__TBB_ASSERT(is_alive(ctx), "referenced task_group_context was destroyed");
static const char *msg = "task_group_context is invalid";
__TBB_ASSERT(!(ctx&~(3|(7<<task_group_context::traits_offset))), msg); // the value fits known values of versions and traits
__TBB_ASSERT(tgc->my_kind < task_group_context::dying, msg);
__TBB_ASSERT(tgc->my_cancellation_requested == 0 || tgc->my_cancellation_requested == 1, msg);
__TBB_ASSERT(tgc->my_state < task_group_context::low_unused_state_bit, msg);
if(tgc->my_kind != task_group_context::isolated) {
__TBB_ASSERT(tgc->my_owner, msg);
__TBB_ASSERT(tgc->my_node.my_next && tgc->my_node.my_prev, msg);
}
#if __TBB_TASK_PRIORITY
assert_priority_valid(tgc->my_priority);
#endif
if(tgc->my_parent)
#if TBB_USE_ASSERT > 1
assert_context_valid(tgc->my_parent);
#else
__TBB_ASSERT(is_alive(tgc->my_parent->my_version_and_traits), msg);
#endif
#endif
}
#endif /* __TBB_TASK_GROUP_CONTEXT */
#if _WIN32||_WIN64
private:
::rml::server::execution_resource_t master_exec_resource;
public:
#endif /* _WIN32||_WIN64 */
#if __TBB_TASK_GROUP_CONTEXT
tbb::atomic<uintptr_t> my_nonlocal_ctx_list_update;
#endif /* __TBB_TASK_GROUP_CONTEXT */
#if __TBB_SURVIVE_THREAD_SWITCH
__cilk_tbb_unwatch_thunk my_cilk_unwatch_thunk;
#if TBB_USE_ASSERT
enum cilk_state_t {
cs_none=0xF000, // Start at nonzero value so that we can detect use of zeroed memory.
cs_running,
cs_limbo,
cs_freed
};
cilk_state_t my_cilk_state;
#endif /* TBB_USE_ASSERT */
#endif /* __TBB_SURVIVE_THREAD_SWITCH */
#if __TBB_STATISTICS
mutable statistics_counters my_counters;
#endif /* __TBB_STATISTICS */
}; // class generic_scheduler
} // namespace internal
} // namespace tbb
#include "arena.h"
#include "governor.h"
namespace tbb {
namespace internal {
inline bool generic_scheduler::is_task_pool_published () const {
__TBB_ASSERT(my_arena_slot, 0);
return my_arena_slot->task_pool != EmptyTaskPool;
}
inline bool generic_scheduler::is_local_task_pool_quiescent () const {
__TBB_ASSERT(my_arena_slot, 0);
task** tp = my_arena_slot->task_pool;
return tp == EmptyTaskPool || tp == LockedTaskPool;
}
inline bool generic_scheduler::is_quiescent_local_task_pool_empty () const {
__TBB_ASSERT( is_local_task_pool_quiescent(), "Task pool is not quiescent" );
return __TBB_load_relaxed(my_arena_slot->head) == __TBB_load_relaxed(my_arena_slot->tail);
}
inline bool generic_scheduler::is_quiescent_local_task_pool_reset () const {
__TBB_ASSERT( is_local_task_pool_quiescent(), "Task pool is not quiescent" );
return __TBB_load_relaxed(my_arena_slot->head) == 0 && __TBB_load_relaxed(my_arena_slot->tail) == 0;
}
inline bool generic_scheduler::outermost_level () const {
return my_properties.outermost;
}
inline bool generic_scheduler::master_outermost_level () const {
return !is_worker() && outermost_level();
}
inline bool generic_scheduler::worker_outermost_level () const {
return is_worker() && outermost_level();
}
#if __TBB_TASK_GROUP_CONTEXT
inline task_group_context* generic_scheduler::default_context () {
return my_dummy_task->prefix().context;
}
#endif /* __TBB_TASK_GROUP_CONTEXT */
inline void generic_scheduler::attach_mailbox( affinity_id id ) {
__TBB_ASSERT(id>0,NULL);
my_inbox.attach( my_arena->mailbox(id) );
my_affinity_id = id;
}
inline bool generic_scheduler::is_worker() const {
return my_properties.type == scheduler_properties::worker;
}
inline unsigned generic_scheduler::max_threads_in_arena() {
__TBB_ASSERT(my_arena, NULL);
return my_arena->my_num_slots;
}
inline void generic_scheduler::deallocate_task( task& t ) {
#if TBB_USE_ASSERT
task_prefix& p = t.prefix();
p.state = 0xFF;
p.extra_state = 0xFF;
poison_pointer(p.next);
#endif /* TBB_USE_ASSERT */
NFS_Free((char*)&t-task_prefix_reservation_size);
#if __TBB_COUNT_TASK_NODES
--my_task_node_count;
#endif /* __TBB_COUNT_TASK_NODES */
}
#if __TBB_COUNT_TASK_NODES
inline intptr_t generic_scheduler::get_task_node_count( bool count_arena_workers ) {
return my_task_node_count + (count_arena_workers? my_arena->workers_task_node_count(): 0);
}
#endif /* __TBB_COUNT_TASK_NODES */
inline void generic_scheduler::reset_task_pool_and_leave () {
__TBB_ASSERT( my_arena_slot->task_pool == LockedTaskPool, "Task pool must be locked when resetting task pool" );
__TBB_store_relaxed( my_arena_slot->tail, 0 );
__TBB_store_relaxed( my_arena_slot->head, 0 );
leave_task_pool();
}
//TODO: move to arena_slot
inline void generic_scheduler::commit_spawned_tasks( size_t new_tail ) {
__TBB_ASSERT ( new_tail <= my_arena_slot->my_task_pool_size, "task deque end was overwritten" );
// emit "task was released" signal
ITT_NOTIFY(sync_releasing, (void*)((uintptr_t)my_arena_slot+sizeof(uintptr_t)));
// Release fence is necessary to make sure that previously stored task pointers
// are visible to thieves.
__TBB_store_with_release( my_arena_slot->tail, new_tail );
}
void generic_scheduler::commit_relocated_tasks ( size_t new_tail ) {
__TBB_ASSERT( is_local_task_pool_quiescent(),
"Task pool must be locked when calling commit_relocated_tasks()" );
__TBB_store_relaxed( my_arena_slot->head, 0 );
// Tail is updated last to minimize probability of a thread making arena
// snapshot being misguided into thinking that this task pool is empty.
__TBB_store_release( my_arena_slot->tail, new_tail );
release_task_pool();
}
template<free_task_hint hint>
void generic_scheduler::free_task( task& t ) {
#if __TBB_HOARD_NONLOCAL_TASKS
static const int h = hint&(~local_task);
#else
static const free_task_hint h = hint;
#endif
GATHER_STATISTIC(--my_counters.active_tasks);
task_prefix& p = t.prefix();
// Verify that optimization hints are correct.
__TBB_ASSERT( h!=small_local_task || p.origin==this, NULL );
__TBB_ASSERT( !(h&small_task) || p.origin, NULL );
__TBB_ASSERT( !(h&local_task) || (!p.origin || uintptr_t(p.origin) > uintptr_t(4096)), "local_task means allocated");
poison_value(p.depth);
poison_value(p.ref_count);
poison_pointer(p.owner);
#if __TBB_PREVIEW_RESUMABLE_TASKS
__TBB_ASSERT(1L << t.state() & (1L << task::executing | 1L << task::allocated | 1 << task::to_resume), NULL);
#else
__TBB_ASSERT(1L << t.state() & (1L << task::executing | 1L << task::allocated), NULL);
#endif
p.state = task::freed;
if( h==small_local_task || p.origin==this ) {
GATHER_STATISTIC(++my_counters.free_list_length);
p.next = my_free_list;
my_free_list = &t;
} else if( !(h&local_task) && p.origin && uintptr_t(p.origin) < uintptr_t(4096) ) {
// a special value reserved for future use, do nothing since
// origin is not pointing to a scheduler instance
} else if( !(h&local_task) && p.origin ) {
GATHER_STATISTIC(++my_counters.free_list_length);
#if __TBB_HOARD_NONLOCAL_TASKS
if( !(h&no_cache) ) {
p.next = my_nonlocal_free_list;
my_nonlocal_free_list = &t;
} else
#endif
free_nonlocal_small_task(t);
} else {
GATHER_STATISTIC(--my_counters.big_tasks);
deallocate_task(t);
}
}
#if __TBB_TASK_PRIORITY
inline intptr_t generic_scheduler::effective_reference_priority () const {
// Workers on the outermost dispatch level (i.e. with empty stack) use market's
// priority as a reference point (to speedup discovering process level priority
// changes). But when there are enough workers to service (even if only partially)
// a lower priority arena, they should use arena's priority as a reference, lest
// be trapped in a futile spinning (because market's priority would prohibit
// executing ANY tasks in this arena).
return !worker_outermost_level() ||
my_arena->my_num_workers_allotted < my_arena->num_workers_active() ? *my_ref_top_priority : my_arena->my_top_priority;
}
inline void generic_scheduler::offload_task ( task& t, intptr_t /*priority*/ ) {
GATHER_STATISTIC( ++my_counters.prio_tasks_offloaded );
__TBB_ASSERT( !is_proxy(t), "The proxy task cannot be offloaded" );
__TBB_ASSERT( my_offloaded_task_list_tail_link && !*my_offloaded_task_list_tail_link, NULL );
#if TBB_USE_ASSERT
t.prefix().state = task::ready;
#endif /* TBB_USE_ASSERT */
t.prefix().next_offloaded = my_offloaded_tasks;
my_offloaded_tasks = &t;
}
#endif /* __TBB_TASK_PRIORITY */
#if __TBB_PREVIEW_RESUMABLE_TASKS
inline void generic_scheduler::set_post_resume_action(post_resume_action pra, void* arg) {
__TBB_ASSERT(my_post_resume_action == PRA_NONE, "Post resume action has already been set.");
__TBB_ASSERT(!my_post_resume_arg, NULL);
my_post_resume_action = pra;
my_post_resume_arg = arg;
}
inline bool generic_scheduler::prepare_resume(generic_scheduler& target) {
// The second condition is valid for worker or cleanup operation for master
if (my_properties.outermost && my_wait_task == my_dummy_task) {
if (my_properties.genuine) {
// We are in someone's original scheduler.
target.set_post_resume_action(PRA_NOTIFY, my_current_is_recalled);
return true;
}
// We are in a coroutine on outermost level.
target.set_post_resume_action(PRA_CLEANUP, this);
my_target_on_exit = &target;
// Request to finish coroutine instead of immediate resume.
return false;
}
__TBB_ASSERT(my_wait_task != my_dummy_task, NULL);
// We are in the coroutine on a nested level.
my_wait_task->prefix().abandoned_scheduler = this;
target.set_post_resume_action(PRA_ABANDON, my_wait_task);
return true;
}
inline bool generic_scheduler::resume_original_scheduler() {
generic_scheduler& target = *my_arena_slot->my_scheduler;
if (!prepare_resume(target)) {
// We should return and finish the current coroutine.
return false;
}
resume(target);
return true;
}
inline void generic_scheduler::resume(generic_scheduler& target) {
// Do not create non-trivial objects on the stack of this function. They might never be destroyed.
__TBB_ASSERT(governor::is_set(this), NULL);
__TBB_ASSERT(target.my_post_resume_action != PRA_NONE,
"The post resume action is not set. Has prepare_resume been called?");
__TBB_ASSERT(target.my_post_resume_arg, NULL);
__TBB_ASSERT(&target != this, NULL);
__TBB_ASSERT(target.my_arena == my_arena, "Cross-arena switch is forbidden.");
// Transfer thread related data.
target.my_arena_index = my_arena_index;
target.my_arena_slot = my_arena_slot;
#if __TBB_SCHEDULER_OBSERVER
target.my_last_global_observer = my_last_global_observer;
#endif
#if __TBB_ARENA_OBSERVER
target.my_last_local_observer = my_last_local_observer;
#endif
target.attach_mailbox(affinity_id(target.my_arena_index + 1));
#if __TBB_TASK_PRIORITY
if (my_offloaded_tasks)
my_arena->orphan_offloaded_tasks(*this);
#endif /* __TBB_TASK_PRIORITY */
governor::assume_scheduler(&target);
my_co_context.resume(target.my_co_context);
__TBB_ASSERT(governor::is_set(this), NULL);
do_post_resume_action();
if (this == my_arena_slot->my_scheduler) {
my_arena_slot->my_scheduler_is_recalled->store<tbb::relaxed>(false);
}
}
inline void generic_scheduler::do_post_resume_action() {
__TBB_ASSERT(my_post_resume_action != PRA_NONE, "The post resume action is not set.");
__TBB_ASSERT(my_post_resume_arg, NULL);
switch (my_post_resume_action) {
case PRA_ABANDON:
{
task_prefix& wait_task_prefix = static_cast<task*>(my_post_resume_arg)->prefix();
reference_count old_ref_count = __TBB_FetchAndAddW(&wait_task_prefix.ref_count, internal::abandon_flag);
__TBB_ASSERT(old_ref_count > 0, NULL);
if (old_ref_count == 1) {
// Remove the abandon flag.
__TBB_store_with_release(wait_task_prefix.ref_count, 1);
// The wait has been completed. Spawn a resume task.
tbb::task::resume(wait_task_prefix.abandoned_scheduler);
}
break;
}
case PRA_CALLBACK:
{
callback_t callback = *static_cast<callback_t*>(my_post_resume_arg);
callback();
break;
}
case PRA_CLEANUP:
{
generic_scheduler* to_cleanup = static_cast<generic_scheduler*>(my_post_resume_arg);
__TBB_ASSERT(!to_cleanup->my_properties.genuine, NULL);
// Release coroutine's reference to my_arena.
to_cleanup->my_arena->on_thread_leaving<arena::ref_external>();
// Cache the coroutine for possible later re-usage
to_cleanup->my_arena->my_co_cache.push(to_cleanup);
break;
}
case PRA_NOTIFY:
{
tbb::atomic<bool>& scheduler_recall_flag = *static_cast<tbb::atomic<bool>*>(my_post_resume_arg);
scheduler_recall_flag = true;
// Do not access recall_flag because it can be destroyed after the notification.
break;
}
default:
__TBB_ASSERT(false, NULL);
}
my_post_resume_action = PRA_NONE;
my_post_resume_arg = NULL;
}
struct recall_functor {
tbb::atomic<bool>* scheduler_recall_flag;
recall_functor(tbb::atomic<bool>* recall_flag_) :
scheduler_recall_flag(recall_flag_) {}
void operator()(task::suspend_point /*tag*/) {
*scheduler_recall_flag = true;
}
};
#if _WIN32
/* [[noreturn]] */ inline void __stdcall co_local_wait_for_all(void* arg) {
#else
/* [[noreturn]] */ inline void co_local_wait_for_all(void* arg) {
#endif
// Do not create non-trivial objects on the stack of this function. They will never be destroyed.
generic_scheduler& s = *static_cast<generic_scheduler*>(arg);
__TBB_ASSERT(governor::is_set(&s), NULL);
// For correct task stealing threshold, calculate stack on a coroutine start
s.init_stack_info();
// Basically calls the user callback passed to the tbb::task::suspend function
s.do_post_resume_action();
// Endless loop here because coroutine could be reused
for( ;; ) {
__TBB_ASSERT(s.my_innermost_running_task == s.my_dummy_task, NULL);
__TBB_ASSERT(s.worker_outermost_level(), NULL);
s.local_wait_for_all(*s.my_dummy_task, NULL);
__TBB_ASSERT(s.my_target_on_exit, NULL);
__TBB_ASSERT(s.my_wait_task == NULL, NULL);
s.resume(*s.my_target_on_exit);
}
// This code is unreachable
}
#endif /* __TBB_PREVIEW_RESUMABLE_TASKS */
#if __TBB_TASK_GROUP_CONTEXT
template <bool report_tasks>
class context_guard_helper {
const task_group_context *curr_ctx;
#if __TBB_FP_CONTEXT
cpu_ctl_env guard_cpu_ctl_env;
cpu_ctl_env curr_cpu_ctl_env;
#endif
public:
context_guard_helper() : curr_ctx( NULL ) {
#if __TBB_FP_CONTEXT
guard_cpu_ctl_env.get_env();
curr_cpu_ctl_env = guard_cpu_ctl_env;
#endif
}
~context_guard_helper() {
#if __TBB_FP_CONTEXT
if ( curr_cpu_ctl_env != guard_cpu_ctl_env )
guard_cpu_ctl_env.set_env();
#endif
if ( report_tasks && curr_ctx )
ITT_TASK_END;
}
// The function is called from bypass dispatch loop on the hot path.
// Consider performance issues when refactoring.
void set_ctx( const task_group_context *ctx ) {
generic_scheduler::assert_context_valid( ctx );
#if __TBB_FP_CONTEXT
const cpu_ctl_env &ctl = *punned_cast<cpu_ctl_env*>( &ctx->my_cpu_ctl_env );
// Compare the FPU settings directly because the context can be reused between parallel algorithms.
if ( ctl != curr_cpu_ctl_env ) {
curr_cpu_ctl_env = ctl;
curr_cpu_ctl_env.set_env();
}
#endif
if ( report_tasks && ctx != curr_ctx ) {
// if task group context was active, report end of current execution frame.
if ( curr_ctx )
ITT_TASK_END;
// reporting begin of new task group context execution frame.
// using address of task group context object to group tasks (parent).
// id of task execution frame is NULL and reserved for future use.
ITT_TASK_BEGIN( ctx,ctx->my_name, NULL );
curr_ctx = ctx;
}
}
void restore_default() {
#if __TBB_FP_CONTEXT
if ( curr_cpu_ctl_env != guard_cpu_ctl_env ) {
guard_cpu_ctl_env.set_env();
curr_cpu_ctl_env = guard_cpu_ctl_env;
}
#endif
}
};
#else
template <bool T>
struct context_guard_helper {
void set_ctx() {}
void restore_default() {}
};
#endif /* __TBB_TASK_GROUP_CONTEXT */
} // namespace internal
} // namespace tbb
#endif /* _TBB_scheduler_H */
tbb::task_group_context::traits_offset
Definition: task.h:360
tbb::internal::generic_scheduler::can_steal
bool can_steal()
Returns true if stealing is allowed.
Definition: scheduler.h:270
tbb::task::prefix
internal::task_prefix & prefix(internal::version_tag *=NULL) const
Get reference to corresponding task_prefix.
Definition: task.h:991
__cilk_tbb_unwatch_thunk
Definition: cilk-tbb-interop.h:95
mailbox.h
tbb::internal::arena_slot
Definition: scheduler_common.h:377
scheduler_common.h
governor.h
tbb::spin_mutex
A lock that occupies a single byte.
Definition: spin_mutex.h:39
spin_mutex.h
internal
Definition: _flow_graph_async_msg_impl.h:24
tbb::relaxed
No ordering.
Definition: atomic.h:61
tbb::task_group_context::my_kind
__TBB_atomic kind_type my_kind
Flavor of this context: bound or isolated.
Definition: task.h:394
void
void
Definition: ittnotify_static.h:91
tbb::internal::generic_scheduler::nested_arena_exit
void nested_arena_exit()
Definition: arena.cpp:756
tbb::internal::generic_scheduler::spawn
void spawn(task &first, task *&next) __TBB_override
For internal use only.
Definition: scheduler.cpp:739
tbb::internal::generic_scheduler::local_spawn
void local_spawn(task *first, task *&next)
Definition: scheduler.cpp:651
tbb::internal::task_prefix::next_offloaded
task * next_offloaded
Pointer to the next offloaded lower priority task.
Definition: task.h:241
__TBB_ISOLATION_EXPR
#define __TBB_ISOLATION_EXPR(isolation)
Definition: scheduler_common.h:67
tbb::internal::custom_scheduler
A scheduler with a customized evaluation loop.
Definition: custom_scheduler.h:52
tbb::internal::scheduler_state
Definition: scheduler.h:77
__TBB_ASSERT
#define __TBB_ASSERT(predicate, comment)
No-op version of __TBB_ASSERT.
Definition: tbb_stddef.h:165
id
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id id
Definition: ittnotify_static.h:172
tbb::internal::generic_scheduler::my_return_list
task * my_return_list
List of small tasks that have been returned to this scheduler by other schedulers.
Definition: scheduler.h:465
LockedTaskPool
#define LockedTaskPool
Definition: scheduler.h:47
ITT_NOTIFY
#define ITT_NOTIFY(name, obj)
Definition: itt_notify.h:112
tbb::internal::scheduler_state::my_affinity_id
affinity_id my_affinity_id
The mailbox id assigned to this scheduler.
Definition: scheduler.h:99
tbb::internal::generic_scheduler::my_dummy_task
task * my_dummy_task
Fake root task created by slave threads.
Definition: scheduler.h:186
tbb::internal::co_local_wait_for_all
void co_local_wait_for_all(void *)
tbb::internal::poison_pointer
void poison_pointer(T *__TBB_atomic &)
Definition: tbb_stddef.h:305
tbb::internal::generic_scheduler::reset_task_pool_and_leave
void reset_task_pool_and_leave()
Resets head and tail indices to 0, and leaves task pool.
Definition: scheduler.h:702
tbb::internal::arena::ref_external
static const unsigned ref_external
Reference increment values for externals and workers.
Definition: arena.h:327
tbb::internal::generic_scheduler::commit_spawned_tasks
void commit_spawned_tasks(size_t new_tail)
Makes newly spawned tasks visible to thieves.
Definition: scheduler.h:710
tbb::internal::generic_scheduler::is_task_pool_published
bool is_task_pool_published() const
Definition: scheduler.h:628
tbb
The graph class.
Definition: serial/tbb/parallel_for.h:46
tbb::internal::generic_scheduler::null_arena_index
static const size_t null_arena_index
Definition: scheduler.h:161
tbb::internal::generic_scheduler::local_spawn_root_and_wait
void local_spawn_root_and_wait(task *first, task *&next)
Definition: scheduler.cpp:718
tbb::task_group_context::my_cancellation_requested
uintptr_t my_cancellation_requested
Specifies whether cancellation was requested for this task group.
Definition: task.h:429
tbb::internal::scheduler_properties::char
unsigned char
Reserved bits.
Definition: scheduler.h:73
tbb::internal::generic_scheduler::release_task_pool
void release_task_pool() const
Unlocks the local task pool.
Definition: scheduler.cpp:522
tbb::internal::generic_scheduler::cleanup_master
bool cleanup_master(bool blocking_terminate)
Perform necessary cleanup when a master thread stops using TBB.
Definition: scheduler.cpp:1339
tbb::internal::context_guard_helper
Definition: scheduler.h:1018
tbb::internal::scheduler
Definition: task.h:99
tbb::internal::es_task_proxy
Tag for v3 task_proxy.
Definition: scheduler_common.h:174
tbb::task_group_context
Used to form groups of tasks.
Definition: task.h:347
tbb::internal::scheduler_properties
Bit-field representing properties of a sheduler.
Definition: scheduler.h:50
tbb::internal::generic_scheduler::local_wait_for_all
virtual void local_wait_for_all(task &parent, task *child)=0
tbb::internal::mail_inbox
Class representing source of mail.
Definition: mailbox.h:185
tbb::task_group_context::isolated
Definition: task.h:367
tbb::internal::scheduler_state::my_arena
arena * my_arena
The arena that I own (if master) or am servicing at the moment (if worker)
Definition: scheduler.h:85
tbb::internal::generic_scheduler::my_auto_initialized
bool my_auto_initialized
True if *this was created by automatic TBB initialization.
Definition: scheduler.h:197
tbb::internal::arena_slot_line1::task_pool
task **__TBB_atomic task_pool
Definition: scheduler_common.h:339
tbb::internal::generic_scheduler::prepare_task_pool
size_t prepare_task_pool(size_t n)
Makes sure that the task pool can accommodate at least n more elements.
Definition: scheduler.cpp:439
tbb::internal::generic_scheduler::plugged_return_list
static task * plugged_return_list()
Special value used to mark my_return_list as not taking any more entries.
Definition: scheduler.h:458
tbb::internal::generic_scheduler::generic_scheduler
generic_scheduler(market &, bool)
Definition: scheduler.cpp:84
__TBB_store_release
#define __TBB_store_release
Definition: tbb_machine.h:860
itt_notify.h
ITT_TASK_END
#define ITT_TASK_END
Definition: itt_notify.h:122
tbb::task::executing
task is running, and will be destroyed after method execute() completes.
Definition: task.h:626
__TBB_atomic
#define __TBB_atomic
Definition: tbb_stddef.h:237
tbb::internal::scheduler_state::my_arena_index
size_t my_arena_index
Index of the arena slot the scheduler occupies now, or occupied last time.
Definition: scheduler.h:79
tbb::internal::generic_scheduler::assert_task_pool_valid
void assert_task_pool_valid() const
Definition: scheduler.h:398
tbb::internal::arena_slot_line1::my_scheduler
generic_scheduler * my_scheduler
Scheduler of the thread attached to the slot.
Definition: scheduler_common.h:333
tbb::internal::generic_scheduler::cleanup_scheduler
void cleanup_scheduler()
Cleans up this scheduler (the scheduler might be destroyed).
Definition: scheduler.cpp:294
tbb::internal::generic_scheduler::leave_task_pool
void leave_task_pool()
Leave the task pool.
Definition: scheduler.cpp:1258
tbb::internal::generic_scheduler::my_small_task_count
__TBB_atomic intptr_t my_small_task_count
Number of small tasks that have been allocated by this scheduler.
Definition: scheduler.h:461
EmptyTaskPool
#define EmptyTaskPool
Definition: scheduler.h:46
tbb::internal::generic_scheduler::get_task
task * get_task(__TBB_ISOLATION_EXPR(isolation_tag isolation))
Get a task from the local pool.
Definition: scheduler.cpp:1010
tbb::internal::small_local_task
Bitwise-OR of local_task and small_task.
Definition: scheduler_common.h:196
tbb::internal::mail_inbox::attach
void attach(mail_outbox &putter)
Attach inbox to a corresponding outbox.
Definition: mailbox.h:193
tbb::internal::context_guard_helper::set_ctx
void set_ctx()
Definition: scheduler.h:1019
tbb::internal::generic_scheduler::steal_task_from
task * steal_task_from(__TBB_ISOLATION_ARG(arena_slot &victim_arena_slot, isolation_tag isolation))
Steal task from another scheduler's ready pool.
Definition: scheduler.cpp:1142
tbb::internal::__TBB_store_relaxed
void __TBB_store_relaxed(volatile T &location, V value)
Definition: tbb_machine.h:742
tbb::internal::generic_scheduler::max_threads_in_arena
unsigned max_threads_in_arena()
Returns the concurrency limit of the current arena.
Definition: scheduler.h:677
tbb::internal::generic_scheduler::create_master
static generic_scheduler * create_master(arena *a)
Initialize a scheduler for a master thread.
Definition: scheduler.cpp:1285
tbb::internal::generic_scheduler::destroy
void destroy()
Destroy and deallocate this scheduler object.
Definition: scheduler.cpp:285
GATHER_STATISTIC
#define GATHER_STATISTIC(x)
Definition: tbb_statistics.h:232
tbb::internal::affinity_id
unsigned short affinity_id
An id as used for specifying affinity.
Definition: task.h:128
tbb::internal::generic_scheduler::prepare_for_spawning
task * prepare_for_spawning(task *t)
Checks if t is affinitized to another thread, and if so, bundles it as proxy.
Definition: scheduler.cpp:595
tbb::internal::arena::mailbox
mail_outbox & mailbox(affinity_id id)
Get reference to mailbox corresponding to given affinity_id.
Definition: arena.h:305
tbb::internal::generic_scheduler::master_outermost_level
bool master_outermost_level() const
True if the scheduler is on the outermost dispatch level in a master thread.
Definition: scheduler.h:653
tbb::internal::__TBB_store_with_release
void __TBB_store_with_release(volatile T &location, V value)
Definition: tbb_machine.h:716
tbb::internal::generic_scheduler::attach_arena
void attach_arena(arena *, size_t index, bool is_master)
Definition: arena.cpp:80
tbb::internal::generic_scheduler::nested_arena_entry
void nested_arena_entry(arena *, size_t)
Definition: arena.cpp:729
tbb::internal::generic_scheduler::attach_mailbox
void attach_mailbox(affinity_id id)
Definition: scheduler.h:667
tbb::internal::task_prefix
Memory prefix to a task object.
Definition: task.h:192
__TBB_ISOLATION_ARG
#define __TBB_ISOLATION_ARG(arg1, isolation)
Definition: scheduler_common.h:68
tbb::internal::generic_scheduler::enqueue
void enqueue(task &, void *reserved) __TBB_override
For internal use only.
Definition: scheduler.cpp:747
tbb::internal::generic_scheduler::is_worker
bool is_worker() const
True if running on a worker thread, false otherwise.
Definition: scheduler.h:673
tbb::task::freed
task object is on free list, or is going to be put there, or was just taken off.
Definition: task.h:634
tbb::task_group_context::my_state
uintptr_t my_state
Internal state (combination of state flags, currently only may_have_children).
Definition: task.h:444
arena.h
tbb::task::state
state_type state() const
Current execution state.
Definition: task.h:901
tbb::internal::NFS_MaxLineSize
const size_t NFS_MaxLineSize
Compile-time constant that is upper bound on cache line/sector size.
Definition: tbb_stddef.h:216
tbb::internal::scheduler_properties::worker
static const bool worker
Definition: scheduler.h:51
tbb::internal::generic_scheduler::publish_task_pool
void publish_task_pool()
Used by workers to enter the task pool.
Definition: scheduler.cpp:1246
tbb::internal::generic_scheduler::is_proxy
static bool is_proxy(const task &t)
True if t is a task_proxy.
Definition: scheduler.h:348
tbb::internal::generic_scheduler::lock_task_pool
task ** lock_task_pool(arena_slot *victim_arena_slot) const
Locks victim's task pool, and returns pointer to it. The pointer can be NULL.
Definition: scheduler.cpp:537
tbb::internal::arena_base::my_num_workers_allotted
unsigned my_num_workers_allotted
The number of workers that have been marked out by the resource manager to service the arena.
Definition: arena.h:147
tbb::task_group_context::my_node
internal::context_list_node_t my_node
Used to form the thread specific list of contexts without additional memory allocation.
Definition: task.h:404
tbb::internal::generic_scheduler::unlock_task_pool
void unlock_task_pool(arena_slot *victim_arena_slot, task **victim_task_pool) const
Unlocks victim's task pool.
Definition: scheduler.cpp:586
parent
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id parent
Definition: ittnotify_static.h:176
tbb::task_group_context::my_version_and_traits
uintptr_t my_version_and_traits
Version for run-time checks and behavioral traits of the context.
Definition: task.h:435
tbb::internal::context_list_node_t
Definition: task.h:139
tbb::internal::arena_slot_line2::tail
__TBB_atomic size_t tail
Index of the element following the last ready task in the deque.
Definition: scheduler_common.h:363
tbb::internal::isolation_tag
intptr_t isolation_tag
A tag for task isolation.
Definition: task.h:132
tbb::internal::first
auto first(Container &c) -> decltype(begin(c))
Definition: _range_iterator.h:34
tbb::internal::scheduler_state::my_innermost_running_task
task * my_innermost_running_task
Innermost task whose task::execute() is running. A dummy task on the outermost level.
Definition: scheduler.h:88
tbb::internal::scheduler_state::my_arena_slot
arena_slot * my_arena_slot
Pointer to the slot in the arena we own at the moment.
Definition: scheduler.h:82
tbb::internal::generic_scheduler::receive_or_steal_task
virtual task * receive_or_steal_task(__TBB_ISOLATION_ARG(__TBB_atomic reference_count &completion_ref_count, isolation_tag isolation))=0
Try getting a task from other threads (via mailbox, stealing, FIFO queue, orphans adoption).
__TBB_override
#define __TBB_override
Definition: tbb_stddef.h:240
tbb_misc.h
tbb::task_group_context::dying
static const kind_type dying
Definition: task.h:581
__TBB_get_bsp
void * __TBB_get_bsp()
Retrieves the current RSE backing store pointer. IA64 specific.
tbb::internal::generic_scheduler::is_quiescent_local_task_pool_empty
bool is_quiescent_local_task_pool_empty() const
Definition: scheduler.h:639
tbb::internal::market
Definition: market.h:45
tbb::internal::no_cache
Disable caching for a small task.
Definition: scheduler_common.h:198
s
void const char const char int ITT_FORMAT __itt_group_sync s
Definition: ittnotify_static.h:91
ITT_TASK_BEGIN
#define ITT_TASK_BEGIN(type, name, id)
Definition: itt_notify.h:121
tbb::internal::generic_scheduler::is_version_3_task
static bool is_version_3_task(task &t)
Definition: scheduler.h:146
tbb::internal::scheduler_properties::outermost
bool outermost
Indicates that a scheduler is on outermost level.
Definition: scheduler.h:57
tbb::task::allocated
task object is freshly allocated or recycled.
Definition: task.h:632
tbb::internal::governor::assume_scheduler
static void assume_scheduler(generic_scheduler *s)
Temporarily set TLS slot to the given scheduler.
Definition: governor.cpp:116
tbb::internal::suppress_unused_warning
void suppress_unused_warning(const T1 &)
Utility template function to prevent "unused" warnings by various compilers.
Definition: tbb_stddef.h:398
tbb::internal::governor::is_set
static bool is_set(generic_scheduler *s)
Used to check validity of the local scheduler TLS contents.
Definition: governor.cpp:120
tbb::internal::generic_scheduler::spawn_root_and_wait
void spawn_root_and_wait(task &first, task *&next) __TBB_override
For internal use only.
Definition: scheduler.cpp:743
tbb::internal::task_prefix::extra_state
unsigned char extra_state
Miscellaneous state that is not directly visible to users, stored as a byte for compactness.
Definition: task.h:281
tbb::internal::arena_slot_line1::head
__TBB_atomic size_t head
Index of the first ready task in the deque.
Definition: scheduler_common.h:343
tbb::internal::generic_scheduler::worker_outermost_level
bool worker_outermost_level() const
True if the scheduler is on the outermost dispatch level in a worker thread.
Definition: scheduler.h:657
cilk-tbb-interop.h
tbb::task_group_context::low_unused_state_bit
Definition: task.h:388
tbb::internal::arena
Definition: arena.h:276
tbb::task_group_context::my_owner
internal::generic_scheduler * my_owner
Scheduler instance that registered this context in its thread specific list.
Definition: task.h:441
tbb::internal::scheduler_state::my_inbox
mail_inbox my_inbox
Definition: scheduler.h:90
tbb::internal::intrusive_list_node
Data structure to be inherited by the types that can form intrusive lists.
Definition: intrusive_list.h:30
poison_value
#define poison_value(g)
Definition: scheduler_common.h:235
tbb::internal::scheduler_state::my_properties
scheduler_properties my_properties
Definition: scheduler.h:101
tbb::task::ready
task is in ready pool, or is going to be put there, or was just taken off.
Definition: task.h:630
tbb::internal::task_prefix::state
unsigned char state
A task::state_type, stored as a byte for compactness.
Definition: task.h:272
tbb::internal::generic_scheduler::steal_task
task * steal_task(__TBB_ISOLATION_EXPR(isolation_tag isolation))
Attempts to steal a task from a randomly chosen thread/scheduler.
Definition: scheduler.cpp:1107
tbb::internal::generic_scheduler::free_nonlocal_small_task
void free_nonlocal_small_task(task &t)
Free a small task t that that was allocated by a different scheduler.
Definition: scheduler.cpp:412
tbb::internal::generic_scheduler::free_task
void free_task(task &t)
Put task on free list.
Definition: scheduler.h:730
tbb::task_group_context::my_parent
task_group_context * my_parent
Pointer to the context of the parent cancellation group. NULL for isolated contexts.
Definition: task.h:399
tbb::internal::local_task
Task is known to have been allocated by this scheduler.
Definition: scheduler_common.h:190
tbb::internal::context_list_node_t::my_next
context_list_node_t * my_next
Definition: task.h:140
tbb::internal::generic_scheduler::min_task_pool_size
static const size_t min_task_pool_size
Definition: scheduler.h:369
tbb::internal::generic_scheduler::is_local_task_pool_quiescent
bool is_local_task_pool_quiescent() const
Definition: scheduler.h:633
tbb::internal::generic_scheduler::deallocate_task
void deallocate_task(task &t)
Return task object to the memory allocator.
Definition: scheduler.h:683
tbb::internal::reference_count
intptr_t reference_count
A reference count.
Definition: task.h:120
tbb::internal::free_task_hint
free_task_hint
Optimization hint to free_task that enables it omit unnecessary tests and code.
Definition: scheduler_common.h:186
tbb::task
Base class for user-defined tasks.
Definition: task.h:604
tbb::internal::generic_scheduler
Work stealing task scheduler.
Definition: scheduler.h:137
tbb::internal::generic_scheduler::allocate_task
task & allocate_task(size_t number_of_bytes, __TBB_CONTEXT_ARG(task *parent, task_group_context *context))
Allocate task object, either from the heap or a free list.
Definition: scheduler.cpp:337
sync_releasing
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p sync_releasing
Definition: ittnotify_static.h:104
tbb::internal::FastRandom
A fast random number generator.
Definition: tbb_misc.h:135
tbb::internal::generic_scheduler::my_market
market * my_market
The market I am in.
Definition: scheduler.h:172
p
void const char const char int ITT_FORMAT __itt_group_sync p
Definition: ittnotify_static.h:91
tbb::internal::context_guard_helper::restore_default
void restore_default()
Definition: scheduler.h:1020
tbb::internal::generic_scheduler::is_quiescent_local_task_pool_reset
bool is_quiescent_local_task_pool_reset() const
Definition: scheduler.h:644
tbb::internal::generic_scheduler::my_ref_count
long my_ref_count
Reference count for scheduler.
Definition: scheduler.h:190
tbb::internal::NFS_Free
void __TBB_EXPORTED_FUNC NFS_Free(void *)
Free memory allocated by NFS_Allocate.
Definition: cache_aligned_allocator.cpp:198
tbb::internal::generic_scheduler::commit_relocated_tasks
void commit_relocated_tasks(size_t new_tail)
Makes relocated tasks visible to thieves and releases the local task pool.
Definition: scheduler.h:719
tbb::internal::task_prefix::context
task_group_context * context
Shared context that is used to communicate asynchronous state changes.
Definition: task.h:219
intrusive_list.h
tbb::internal::generic_scheduler::my_random
FastRandom my_random
Random number generator used for picking a random victim from which to steal.
Definition: scheduler.h:175
tbb::internal::context_list_node_t::my_prev
context_list_node_t * my_prev
Definition: task.h:140
tbb::internal::scheduler_properties::master
static const bool master
Definition: scheduler.h:52
tbb::internal::generic_scheduler::init_stack_info
void init_stack_info()
Sets up the data necessary for the stealing limiting heuristics.
Definition: scheduler.cpp:158
co_context.h
tbb::internal::arena_base::my_num_slots
unsigned my_num_slots
The number of slots in the arena.
Definition: arena.h:250
tbb::internal::generic_scheduler::acquire_task_pool
void acquire_task_pool() const
Locks the local task pool.
Definition: scheduler.cpp:493
tbb::internal::generic_scheduler::wait_until_empty
void wait_until_empty()
Definition: arena.cpp:772
tbb::internal::task_prefix_reservation_size
const size_t task_prefix_reservation_size
Number of bytes reserved for a task prefix.
Definition: scheduler_common.h:159
tbb::internal::generic_scheduler::my_stealing_threshold
uintptr_t my_stealing_threshold
Position in the call stack specifying its maximal filling when stealing is still allowed.
Definition: scheduler.h:155
tbb::internal::generic_scheduler::quick_task_size
static const size_t quick_task_size
If sizeof(task) is <=quick_task_size, it is handled on a free list instead of malloc'd.
Definition: scheduler.h:144
tbb::internal::small_task
Task is known to be a small task.
Definition: scheduler_common.h:193
tbb::internal::arena::num_workers_active
unsigned num_workers_active() const
The number of workers active in the arena.
Definition: arena.h:334
tbb::internal::generic_scheduler::create_worker
static generic_scheduler * create_worker(market &m, size_t index, bool geniune)
Initialize a scheduler for a worker thread.
Definition: scheduler.cpp:1271
tbb::internal::generic_scheduler::get_mailbox_task
task * get_mailbox_task(__TBB_ISOLATION_EXPR(isolation_tag isolation))
Attempt to get a task from the mailbox.
Definition: scheduler.cpp:1232
h
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function h
Definition: ittnotify_static.h:159
tbb::task_group_context::my_priority
intptr_t my_priority
Priority level of the task group (in normalized representation)
Definition: task.h:448
tbb::internal::__TBB_load_relaxed
T __TBB_load_relaxed(const volatile T &location)
Definition: tbb_machine.h:738
tbb::internal::generic_scheduler::outermost_level
bool outermost_level() const
True if the scheduler is on the outermost dispatch level.
Definition: scheduler.h:649
tbb::internal::generic_scheduler::my_free_list
task * my_free_list
Free list of small tasks that can be reused.
Definition: scheduler.h:178
tbb::internal::generic_scheduler::cleanup_worker
static void cleanup_worker(void *arg, bool worker)
Perform necessary cleanup when a worker thread finishes.
Definition: scheduler.cpp:1329
tbb::internal::scheduler_properties::type
bool type
Indicates that a scheduler acts as a master or a worker.
Definition: scheduler.h:54
__TBB_CONTEXT_ARG
#define __TBB_CONTEXT_ARG(arg1, context)
Definition: scheduler_common.h:60

Copyright © 2005-2020 Intel Corporation. All Rights Reserved.

Intel, Pentium, Intel Xeon, Itanium, Intel XScale and VTune are registered trademarks or trademarks of Intel Corporation or its subsidiaries in the United States and other countries.

* Other names and brands may be claimed as the property of others.