Loading...
Searching...
No Matches
workers.h
/* StarPU --- Runtime system for heterogeneous multicore architectures.
*
* Copyright (C) 2008-2023 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria
* Copyright (C) 2013 Thibaut Lambert
* Copyright (C) 2016 Uppsala University
*
* StarPU is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation; either version 2.1 of the License, or (at
* your option) any later version.
*
* StarPU is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
*
* See the GNU Lesser General Public License in COPYING.LGPL for more details.
*/
#ifndef __WORKERS_H__
#define __WORKERS_H__
/* @{ */
#include <limits.h>
#include <starpu.h>
#include <common/config.h>
#include <common/timing.h>
#include <common/fxt.h>
#include <common/thread.h>
#include <common/utils.h>
#include <core/jobs.h>
#include <core/perfmodel/perfmodel.h>
#include <core/sched_policy.h>
#include <core/topology.h>
#include <core/errorcheck.h>
#include <core/sched_ctx.h>
#include <core/sched_ctx_list.h>
#include <core/simgrid.h>
#ifdef STARPU_HAVE_HWLOC
#include <hwloc.h>
#endif
#include <common/knobs.h>
#include <core/drivers.h>
#include <drivers/cuda/driver_cuda.h>
#include <drivers/hip/driver_hip.h>
#include <drivers/opencl/driver_opencl.h>
#ifdef STARPU_USE_MPI_MASTER_SLAVE
#include <drivers/mpi/driver_mpi_source.h>
#endif
#ifdef STARPU_USE_TCPIP_MASTER_SLAVE
#include <drivers/tcpip/driver_tcpip_source.h>
#endif
#include <drivers/cpu/driver_cpu.h>
#include <datawizard/datawizard.h>
#include <datawizard/malloc.h>
#pragma GCC visibility push(hidden)
#define STARPU_MAX_PIPELINE 4
struct mc_cache_entry;
struct _starpu_node
{
/*
* used by memalloc.c
*/
/* Note: handle header lock is always taken before this (normal add/remove case) */
struct _starpu_spinlock mc_lock;
struct _starpu_mem_chunk_list mc_list;
struct _starpu_mem_chunk *mc_dirty_head;
/* TODO: introduce head of data to be evicted */
unsigned mc_nb, mc_clean_nb;
struct mc_cache_entry *mc_cache;
int mc_cache_nb;
starpu_ssize_t mc_cache_size;
unsigned tidying;
unsigned reclaiming;
volatile int prefetch_out_of_memory;
unsigned evictable;
/*
* used by data_request.c
*/
struct _starpu_data_request_prio_list data_requests[STARPU_MAXNODES][2];
struct _starpu_data_request_prio_list prefetch_requests[STARPU_MAXNODES][2]; /* Contains both task_prefetch and prefetch */
struct _starpu_data_request_prio_list idle_requests[STARPU_MAXNODES][2];
starpu_pthread_mutex_t data_requests_list_mutex[STARPU_MAXNODES][2];
struct _starpu_data_request_prio_list data_requests_pending[STARPU_MAXNODES][2];
unsigned data_requests_npending[STARPU_MAXNODES][2];
starpu_pthread_mutex_t data_requests_pending_list_mutex[STARPU_MAXNODES][2];
/*
* used by malloc.c
*/
int malloc_on_node_default_flags;
struct _starpu_chunk_list chunks;
int nfreechunks;
starpu_pthread_mutex_t chunk_mutex;
/*
* used by memory_manager.c
*/
size_t global_size;
size_t used_size;
/* This is used as an optimization to avoid to wake up allocating threads for
* each and every deallocation, only to find that there is still not enough
* room. */
/* Minimum amount being waited for */
size_t waiting_size;
starpu_pthread_mutex_t lock_nodes;
starpu_pthread_cond_t cond_nodes;
char padding[STARPU_CACHELINE_SIZE];
};
struct _starpu_ctx_change_list;
LIST_TYPE(_starpu_worker,
struct _starpu_machine_config *config;
starpu_pthread_mutex_t mutex;
enum starpu_worker_archtype arch;
uint32_t worker_mask;
struct starpu_perfmodel_arch perf_arch;
starpu_pthread_t worker_thread;
unsigned devid;
unsigned devnum;
unsigned subworkerid;
int bindid;
int workerid;
int combined_workerid;
int current_rank;
int worker_size;
starpu_pthread_cond_t started_cond;
starpu_pthread_cond_t ready_cond;
unsigned memory_node;
unsigned numa_memory_node;
starpu_pthread_cond_t sched_cond;
starpu_pthread_mutex_t sched_mutex;
unsigned state_relax_refcnt;
#ifdef STARPU_SPINLOCK_CHECK
const char *relax_on_file;
int relax_on_line;
const char *relax_on_func;
const char *relax_off_file;
int relax_off_line;
const char *relax_off_func;
#endif
unsigned state_sched_op_pending;
unsigned state_changing_ctx_waiting;
unsigned state_changing_ctx_notice;
unsigned state_blocked_in_parallel;
unsigned state_blocked_in_parallel_observed;
unsigned state_block_in_parallel_req;
unsigned state_block_in_parallel_ack;
unsigned state_unblock_in_parallel_req;
unsigned state_unblock_in_parallel_ack;
unsigned block_in_parallel_ref_count;
starpu_pthread_t thread_changing_ctx;
struct _starpu_ctx_change_list ctx_change_list;
struct starpu_task_prio_list local_tasks;
struct starpu_task **local_ordered_tasks;
unsigned local_ordered_tasks_size;
unsigned current_ordered_task;
unsigned current_ordered_task_order;
struct starpu_task *current_task;
struct starpu_task *current_tasks[STARPU_MAX_PIPELINE];
#ifdef STARPU_SIMGRID
starpu_pthread_wait_t wait;
#endif
struct timespec cl_start;
struct timespec cl_expend;
struct timespec cl_end;
unsigned char first_task;
unsigned char ntasks;
unsigned char pipeline_length;
unsigned char pipeline_stuck;
struct _starpu_worker_set *set;
struct _starpu_worker_set *driver_worker_set;
unsigned worker_is_running;
unsigned worker_is_initialized;
unsigned wait_for_worker_initialization;
enum _starpu_worker_status status;
unsigned state_keep_awake;
char name[128];
char short_name[32];
unsigned run_by_starpu;
const struct _starpu_driver_ops *driver_ops;
struct _starpu_sched_ctx_list *sched_ctx_list;
int tmp_sched_ctx;
unsigned nsched_ctxs;
struct _starpu_barrier_counter tasks_barrier;
unsigned has_prev_init;
unsigned removed_from_ctx[STARPU_NMAX_SCHED_CTXS+1];
unsigned spinning_backoff ;
unsigned nb_buffers_transferred;
unsigned nb_buffers_totransfer;
struct starpu_task *task_transferring;
unsigned shares_tasks_lists[STARPU_NMAX_SCHED_CTXS+1];
unsigned poped_in_ctx[STARPU_NMAX_SCHED_CTXS+1];
unsigned reverse_phase[2];
unsigned pop_ctx_priority;
unsigned is_slave_somewhere;
struct _starpu_sched_ctx *stream_ctx;
#ifdef __GLIBC__
cpu_set_t cpu_set;
#endif /* __GLIBC__ */
#ifdef STARPU_HAVE_HWLOC
hwloc_bitmap_t hwloc_cpu_set;
hwloc_obj_t hwloc_obj;
#endif
struct starpu_profiling_worker_info profiling_info;
/* TODO: rather use rwlock? */
starpu_pthread_mutex_t profiling_info_mutex;
/* In case the worker is still sleeping when the user request profiling info,
* we need to account for the time elapsed while sleeping. */
unsigned profiling_registered_start[STATUS_INDEX_NR];
struct timespec profiling_registered_start_date[STATUS_INDEX_NR];
enum _starpu_worker_status profiling_status;
struct timespec profiling_status_start_date;
struct starpu_perf_counter_sample perf_counter_sample;
int64_t __w_total_executed__value;
double __w_cumul_execution_time__value;
int enable_knob;
int bindid_requested;
char padding[STARPU_CACHELINE_SIZE];
);
struct _starpu_combined_worker
{
struct starpu_perfmodel_arch perf_arch;
uint32_t worker_mask;
int worker_size;
unsigned memory_node;
int combined_workerid[STARPU_NMAXWORKERS];
#ifdef STARPU_USE_MP
int count;
starpu_pthread_mutex_t count_mutex;
#endif
#ifdef __GLIBC__
cpu_set_t cpu_set;
#endif /* __GLIBC__ */
#ifdef STARPU_HAVE_HWLOC
hwloc_bitmap_t hwloc_cpu_set;
#endif
char padding[STARPU_CACHELINE_SIZE];
};
struct _starpu_worker_set
{
starpu_pthread_mutex_t mutex;
starpu_pthread_t worker_thread;
unsigned nworkers;
unsigned started;
void *retval;
struct _starpu_worker *workers;
starpu_pthread_cond_t ready_cond;
unsigned set_is_initialized;
unsigned wait_for_set_initialization;
};
struct _starpu_machine_topology
{
unsigned nworkers;
unsigned ncombinedworkers;
unsigned nsched_ctxs;
#ifdef STARPU_HAVE_HWLOC
hwloc_topology_t hwtopology;
hwloc_bitmap_t log_cpuset;
hwloc_bitmap_t log_coreset;
#endif
struct starpu_tree *tree;
unsigned nhwpus;
unsigned firstusedpu;
unsigned nusedpus;
unsigned nhwdevices[STARPU_NARCH];
unsigned nhwworker[STARPU_NARCH][STARPU_NMAXDEVS];
unsigned ndevices[STARPU_NARCH];
unsigned nworker[STARPU_NARCH][STARPU_NMAXDEVS];
int devid[STARPU_NARCH][STARPU_NMAXDEVS];
int cuda_th_per_stream;
int cuda_th_per_dev;
int hip_th_per_stream;
int hip_th_per_dev;
unsigned workers_bindid[STARPU_NMAXWORKERS];
unsigned workers_nbindid;
unsigned workers_devid[STARPU_NARCH][STARPU_NMAXWORKERS];
};
struct _starpu_machine_config
{
struct _starpu_machine_topology topology;
#ifdef STARPU_HAVE_HWLOC
int cpu_depth;
int pu_depth;
#endif
int current_bindid;
char currently_bound[STARPU_NMAXWORKERS];
char currently_shared[STARPU_NMAXWORKERS];
int current_devid[STARPU_NARCH];
int current_tcpip_deviceid;
int arch_nodeid [STARPU_NARCH];
char padding1[STARPU_CACHELINE_SIZE];
struct _starpu_worker workers[STARPU_NMAXWORKERS];
struct _starpu_node nodes[STARPU_MAXNODES];
struct _starpu_combined_worker combined_workers[STARPU_NMAX_COMBINEDWORKERS];
starpu_pthread_mutex_t submitted_mutex;
char padding2[STARPU_CACHELINE_SIZE];
struct
{
int *workerids;
unsigned nworkers;
} *bindid_workers;
unsigned nbindid;
uint32_t worker_mask;
struct starpu_conf conf;
unsigned running;
int disable_kernels;
int pause_depth;
struct _starpu_sched_ctx sched_ctxs[STARPU_NMAX_SCHED_CTXS+1];
unsigned submitting;
int watchdog_ok;
int perf_counter_pause_depth;
};
struct _starpu_machine_topology;
struct _starpu_driver_info
{
const char *name_upper;
const char *name_var;
const char *name_lower;
enum starpu_node_kind memory_kind;
double alpha;
unsigned wait_for_worker_initialization;
const struct _starpu_driver_ops *driver_ops;
void *(*run_worker)(void *);
void (*init_worker_binding)(struct _starpu_machine_config *config, int no_mp_config STARPU_ATTRIBUTE_UNUSED, struct _starpu_worker *workerarg);
void (*init_worker_memory)(struct _starpu_machine_config *config, int no_mp_config STARPU_ATTRIBUTE_UNUSED, struct _starpu_worker *workerarg);
#ifdef STARPU_HAVE_HWLOC
hwloc_obj_t (*get_hwloc_obj)(hwloc_topology_t topology, int devid);
#endif
};
extern struct _starpu_driver_info starpu_driver_info[STARPU_NARCH];
void _starpu_driver_info_register(enum starpu_worker_archtype archtype, const struct _starpu_driver_info *info);
struct _starpu_memory_driver_info
{
const char *name_upper;
enum starpu_worker_archtype worker_archtype;
const struct _starpu_node_ops *ops;
};
extern struct _starpu_memory_driver_info starpu_memory_driver_info[STARPU_MAX_RAM+1];
void _starpu_memory_driver_info_register(enum starpu_node_kind kind, const struct _starpu_memory_driver_info *info);
extern int _starpu_worker_parallel_blocks;
extern struct _starpu_machine_config _starpu_config;
extern int _starpu_keys_initialized;
extern starpu_pthread_key_t _starpu_worker_key;
extern starpu_pthread_key_t _starpu_worker_set_key;
void _starpu_set_catch_signals(int do_catch_signal);
void _starpu_set_argc_argv(int *argc, char ***argv);
int *_starpu_get_argc();
char ***_starpu_get_argv();
void _starpu_conf_check_environment(struct starpu_conf *conf);
void _starpu_may_pause(void);
static inline unsigned _starpu_machine_is_running(void)
{
unsigned ret;
/* running is just protected by a memory barrier */
STARPU_RMB();
ANNOTATE_HAPPENS_AFTER(&_starpu_config.running);
ret = _starpu_config.running;
ANNOTATE_HAPPENS_BEFORE(&_starpu_config.running);
return ret;
}
void _starpu_worker_init(struct _starpu_worker *workerarg, struct _starpu_machine_config *pconfig);
uint32_t _starpu_worker_exists(struct starpu_task *) STARPU_ATTRIBUTE_VISIBILITY_DEFAULT;
uint32_t _starpu_can_submit_ms_task(void);
uint32_t _starpu_can_submit_cuda_task(void);
uint32_t _starpu_can_submit_hip_task(void);
uint32_t _starpu_can_submit_cpu_task(void);
uint32_t _starpu_can_submit_opencl_task(void);
unsigned _starpu_worker_can_block(unsigned memnode, struct _starpu_worker *worker);
void _starpu_driver_start(struct _starpu_worker *worker, enum starpu_worker_archtype archtype, unsigned sync);
void _starpu_worker_start(struct _starpu_worker *worker, enum starpu_worker_archtype archtype, unsigned sync);
static inline unsigned _starpu_worker_get_count(void)
{
return _starpu_config.topology.nworkers;
}
#define starpu_worker_get_count _starpu_worker_get_count
static inline void _starpu_set_local_worker_key(struct _starpu_worker *worker)
{
STARPU_ASSERT(_starpu_keys_initialized);
STARPU_PTHREAD_SETSPECIFIC(_starpu_worker_key, worker);
}
static inline struct _starpu_worker *_starpu_get_local_worker_key(void)
{
if (!_starpu_keys_initialized)
return NULL;
return (struct _starpu_worker *) STARPU_PTHREAD_GETSPECIFIC(_starpu_worker_key);
}
static inline void _starpu_set_local_worker_set_key(struct _starpu_worker_set *worker)
{
STARPU_ASSERT(_starpu_keys_initialized);
STARPU_PTHREAD_SETSPECIFIC(_starpu_worker_set_key, worker);
}
static inline struct _starpu_worker_set *_starpu_get_local_worker_set_key(void)
{
if (!_starpu_keys_initialized)
return NULL;
return (struct _starpu_worker_set *) STARPU_PTHREAD_GETSPECIFIC(_starpu_worker_set_key);
}
static inline struct _starpu_worker *_starpu_get_worker_struct(unsigned id)
{
STARPU_ASSERT(id < STARPU_NMAXWORKERS);
return &_starpu_config.workers[id];
}
static inline struct _starpu_node *_starpu_get_node_struct(unsigned id)
{
STARPU_ASSERT(id < STARPU_MAXNODES);
return &_starpu_config.nodes[id];
}
static inline struct _starpu_sched_ctx *_starpu_get_sched_ctx_struct(unsigned id)
{
return (id > STARPU_NMAX_SCHED_CTXS) ? NULL : &_starpu_config.sched_ctxs[id];
}
struct _starpu_combined_worker *_starpu_get_combined_worker_struct(unsigned id);
static inline struct _starpu_machine_config *_starpu_get_machine_config(void)
{
return &_starpu_config;
}
static inline int _starpu_get_disable_kernels(void)
{
return _starpu_config.disable_kernels;
}
static inline enum _starpu_worker_status _starpu_worker_get_status(int workerid)
{
return _starpu_config.workers[workerid].status;
}
static inline void _starpu_worker_add_status(int workerid, enum _starpu_worker_status_index status)
{
STARPU_ASSERT(!(_starpu_config.workers[workerid].status & (1 << status)));
if (starpu_profiling_status_get())
_starpu_worker_start_state(workerid, status, NULL);
_starpu_config.workers[workerid].status |= (1 << status);
}
static inline void _starpu_worker_clear_status(int workerid, enum _starpu_worker_status_index status)
{
STARPU_ASSERT((_starpu_config.workers[workerid].status & (1 << status)));
if (starpu_profiling_status_get())
_starpu_worker_stop_state(workerid, status, NULL);
_starpu_config.workers[workerid].status &= ~(1 << status);
}
static inline struct _starpu_sched_ctx* _starpu_get_initial_sched_ctx(void)
{
return &_starpu_config.sched_ctxs[STARPU_GLOBAL_SCHED_CTX];
}
int _starpu_worker_get_nids_by_type(enum starpu_worker_archtype type, int *workerids, int maxsize);
int _starpu_worker_get_nids_ctx_free_by_type(enum starpu_worker_archtype type, int *workerids, int maxsize);
static inline unsigned _starpu_worker_mutex_is_sched_mutex(int workerid, starpu_pthread_mutex_t *mutex)
{
struct _starpu_worker *w = _starpu_get_worker_struct(workerid);
return &w->sched_mutex == mutex;
}
static inline int _starpu_worker_get_nsched_ctxs(int workerid)
{
return _starpu_config.workers[workerid].nsched_ctxs;
}
static inline unsigned _starpu_get_nsched_ctxs(void)
{
/* topology.nsched_ctxs may be increased asynchronously in sched_ctx_create */
STARPU_RMB();
return _starpu_config.topology.nsched_ctxs;
}
static inline int _starpu_worker_get_id(void)
{
struct _starpu_worker * worker;
worker = _starpu_get_local_worker_key();
if (worker)
{
return worker->workerid;
}
else
{
/* there is no worker associated to that thread, perhaps it is
* a thread from the application or this is some SPU worker */
return -1;
}
}
#define starpu_worker_get_id _starpu_worker_get_id
static inline unsigned __starpu_worker_get_id_check(const char *f, int l)
{
(void) l;
(void) f;
int id = starpu_worker_get_id();
STARPU_ASSERT_MSG(id>=0, "%s:%d Cannot be called from outside a worker\n", f, l);
return id;
}
#define _starpu_worker_get_id_check(f,l) __starpu_worker_get_id_check(f,l)
void _starpu_worker_set_stream_ctx(unsigned workerid, struct _starpu_sched_ctx *sched_ctx);
struct _starpu_sched_ctx* _starpu_worker_get_ctx_stream(unsigned stream_workerid);
static inline void _starpu_worker_request_blocking_in_parallel(struct _starpu_worker * const worker)
{
_starpu_worker_parallel_blocks = 1;
/* flush pending requests to start on a fresh transaction epoch */
while (worker->state_unblock_in_parallel_req)
STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
/* announce blocking intent */
STARPU_ASSERT(worker->block_in_parallel_ref_count < UINT_MAX);
worker->block_in_parallel_ref_count++;
if (worker->block_in_parallel_ref_count == 1)
{
/* only the transition from 0 to 1 triggers the block_in_parallel_req */
STARPU_ASSERT(!worker->state_blocked_in_parallel);
STARPU_ASSERT(!worker->state_block_in_parallel_req);
STARPU_ASSERT(!worker->state_block_in_parallel_ack);
STARPU_ASSERT(!worker->state_unblock_in_parallel_req);
STARPU_ASSERT(!worker->state_unblock_in_parallel_ack);
/* trigger the block_in_parallel_req */
worker->state_block_in_parallel_req = 1;
STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
#ifdef STARPU_SIMGRID
starpu_pthread_queue_broadcast(&_starpu_simgrid_task_queue[worker->workerid]);
#endif
/* wait for block_in_parallel_req to be processed */
while (!worker->state_block_in_parallel_ack)
STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
STARPU_ASSERT(worker->block_in_parallel_ref_count >= 1);
STARPU_ASSERT(worker->state_block_in_parallel_req);
STARPU_ASSERT(worker->state_blocked_in_parallel);
/* reset block_in_parallel_req state flags */
worker->state_block_in_parallel_req = 0;
worker->state_block_in_parallel_ack = 0;
/* broadcast block_in_parallel_req state flags reset */
STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
}
}
static inline void _starpu_worker_request_unblocking_in_parallel(struct _starpu_worker * const worker)
{
/* flush pending requests to start on a fresh transaction epoch */
while (worker->state_block_in_parallel_req)
STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
/* unblocking may be requested unconditionally
* thus, check is unblocking is really needed */
if (worker->state_blocked_in_parallel)
{
if (worker->block_in_parallel_ref_count == 1)
{
/* only the transition from 1 to 0 triggers the unblock_in_parallel_req */
STARPU_ASSERT(!worker->state_block_in_parallel_req);
STARPU_ASSERT(!worker->state_block_in_parallel_ack);
STARPU_ASSERT(!worker->state_unblock_in_parallel_req);
STARPU_ASSERT(!worker->state_unblock_in_parallel_ack);
/* trigger the unblock_in_parallel_req */
worker->state_unblock_in_parallel_req = 1;
STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
/* wait for the unblock_in_parallel_req to be processed */
while (!worker->state_unblock_in_parallel_ack)
STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
STARPU_ASSERT(worker->state_unblock_in_parallel_req);
STARPU_ASSERT(!worker->state_blocked_in_parallel);
/* reset unblock_in_parallel_req state flags */
worker->state_unblock_in_parallel_req = 0;
worker->state_unblock_in_parallel_ack = 0;
/* broadcast unblock_in_parallel_req state flags reset */
STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
}
/* announce unblocking complete */
STARPU_ASSERT(worker->block_in_parallel_ref_count > 0);
worker->block_in_parallel_ref_count--;
}
}
static inline void _starpu_worker_process_block_in_parallel_requests(struct _starpu_worker * const worker)
{
while (worker->state_block_in_parallel_req)
{
STARPU_ASSERT(!worker->state_blocked_in_parallel);
STARPU_ASSERT(!worker->state_block_in_parallel_ack);
STARPU_ASSERT(!worker->state_unblock_in_parallel_req);
STARPU_ASSERT(!worker->state_unblock_in_parallel_ack);
STARPU_ASSERT(worker->block_in_parallel_ref_count > 0);
/* enter effective blocked state */
worker->state_blocked_in_parallel = 1;
/* notify block_in_parallel_req processing */
worker->state_block_in_parallel_ack = 1;
STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
/* block */
while (!worker->state_unblock_in_parallel_req)
STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
STARPU_ASSERT(worker->state_blocked_in_parallel);
STARPU_ASSERT(!worker->state_block_in_parallel_req);
STARPU_ASSERT(!worker->state_block_in_parallel_ack);
STARPU_ASSERT(!worker->state_unblock_in_parallel_ack);
STARPU_ASSERT(worker->block_in_parallel_ref_count > 0);
/* leave effective blocked state */
worker->state_blocked_in_parallel = 0;
/* notify unblock_in_parallel_req processing */
worker->state_unblock_in_parallel_ack = 1;
STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
}
}
#ifdef STARPU_SPINLOCK_CHECK
#define _starpu_worker_enter_sched_op(worker) __starpu_worker_enter_sched_op((worker), __FILE__, __LINE__, __starpu_func__)
static inline void __starpu_worker_enter_sched_op(struct _starpu_worker * const worker, const char*file, int line, const char* func)
#else
static inline void _starpu_worker_enter_sched_op(struct _starpu_worker * const worker)
#endif
{
STARPU_ASSERT(!worker->state_sched_op_pending);
if (!worker->state_blocked_in_parallel_observed)
{
/* process pending block requests before entering a sched_op region */
_starpu_worker_process_block_in_parallel_requests(worker);
while (worker->state_changing_ctx_notice)
{
STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
/* new block requests may have been triggered during the wait,
* need to check again */
_starpu_worker_process_block_in_parallel_requests(worker);
}
}
else
{
/* if someone observed the worker state since the last call, postpone block request
* processing for one sched_op turn more, because the observer will not have seen
* new block requests between its observation and now.
*
* however, the worker still has to wait for context change operations to complete
* before entering sched_op again*/
while (worker->state_changing_ctx_notice)
{
STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
}
}
/* no block request and no ctx change ahead,
* enter sched_op */
worker->state_sched_op_pending = 1;
worker->state_blocked_in_parallel_observed = 0;
worker->state_relax_refcnt = 0;
#ifdef STARPU_SPINLOCK_CHECK
worker->relax_on_file = file;
worker->relax_on_line = line;
worker->relax_on_func = func;
#endif
}
void _starpu_worker_apply_deferred_ctx_changes(void);
#ifdef STARPU_SPINLOCK_CHECK
#define _starpu_worker_leave_sched_op(worker) __starpu_worker_leave_sched_op((worker), __FILE__, __LINE__, __starpu_func__)
static inline void __starpu_worker_leave_sched_op(struct _starpu_worker * const worker, const char*file, int line, const char* func)
#else
static inline void _starpu_worker_leave_sched_op(struct _starpu_worker * const worker)
#endif
{
STARPU_ASSERT(worker->state_sched_op_pending);
worker->state_relax_refcnt = 1;
#ifdef STARPU_SPINLOCK_CHECK
worker->relax_off_file = file;
worker->relax_off_line = line;
worker->relax_off_func = func;
#endif
worker->state_sched_op_pending = 0;
STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
_starpu_worker_apply_deferred_ctx_changes();
}
static inline int _starpu_worker_sched_op_pending(void)
{
int workerid = starpu_worker_get_id();
if (workerid == -1)
return 0;
struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
STARPU_ASSERT(worker != NULL);
return worker->state_sched_op_pending;
}
static inline void _starpu_worker_enter_changing_ctx_op(struct _starpu_worker * const worker)
{
STARPU_ASSERT(!starpu_pthread_equal(worker->thread_changing_ctx, starpu_pthread_self()));
/* flush pending requests to start on a fresh transaction epoch */
while (worker->state_changing_ctx_notice)
STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
/* announce changing_ctx intent
*
* - an already started sched_op is allowed to complete
* - no new sched_op may be started
*/
worker->state_changing_ctx_notice = 1;
worker->thread_changing_ctx = starpu_pthread_self();
/* allow for an already started sched_op to complete */
if (worker->state_sched_op_pending)
{
/* request sched_op to broadcast when way is cleared */
worker->state_changing_ctx_waiting = 1;
/* wait for sched_op completion */
STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
#ifdef STARPU_SIMGRID
starpu_pthread_queue_broadcast(&_starpu_simgrid_task_queue[worker->workerid]);
#endif
do
{
STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
}
while (worker->state_sched_op_pending);
/* reset flag so other sched_ops won't have to broadcast state */
worker->state_changing_ctx_waiting = 0;
}
}
static inline void _starpu_worker_leave_changing_ctx_op(struct _starpu_worker * const worker)
{
worker->thread_changing_ctx = (starpu_pthread_t)0;
worker->state_changing_ctx_notice = 0;
STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
}
#ifdef STARPU_SPINLOCK_CHECK
#define _starpu_worker_relax_on() __starpu_worker_relax_on(__FILE__, __LINE__, __starpu_func__)
static inline void __starpu_worker_relax_on(const char*file, int line, const char* func)
#else
static inline void _starpu_worker_relax_on(void)
#endif
{
struct _starpu_worker *worker = _starpu_get_local_worker_key();
if (worker == NULL)
return;
if (!worker->state_sched_op_pending)
return;
STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->sched_mutex);
#ifdef STARPU_SPINLOCK_CHECK
STARPU_ASSERT_MSG(worker->state_relax_refcnt<UINT_MAX, "relax last turn on in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
#else
STARPU_ASSERT(worker->state_relax_refcnt<UINT_MAX);
#endif
worker->state_relax_refcnt++;
#ifdef STARPU_SPINLOCK_CHECK
worker->relax_on_file = file;
worker->relax_on_line = line;
worker->relax_on_func = func;
#endif
STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->sched_mutex);
}
#define starpu_worker_relax_on _starpu_worker_relax_on
#ifdef STARPU_SPINLOCK_CHECK
#define _starpu_worker_relax_on_locked(worker) __starpu_worker_relax_on_locked(worker,__FILE__, __LINE__, __starpu_func__)
static inline void __starpu_worker_relax_on_locked(struct _starpu_worker *worker, const char*file, int line, const char* func)
#else
static inline void _starpu_worker_relax_on_locked(struct _starpu_worker *worker)
#endif
{
if (!worker->state_sched_op_pending)
return;
#ifdef STARPU_SPINLOCK_CHECK
STARPU_ASSERT_MSG(worker->state_relax_refcnt<UINT_MAX, "relax last turn on in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
#else
STARPU_ASSERT(worker->state_relax_refcnt<UINT_MAX);
#endif
worker->state_relax_refcnt++;
#ifdef STARPU_SPINLOCK_CHECK
worker->relax_on_file = file;
worker->relax_on_line = line;
worker->relax_on_func = func;
#endif
STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
}
#ifdef STARPU_SPINLOCK_CHECK
#define _starpu_worker_relax_off() __starpu_worker_relax_off(__FILE__, __LINE__, __starpu_func__)
static inline void __starpu_worker_relax_off(const char*file, int line, const char* func)
#else
static inline void _starpu_worker_relax_off(void)
#endif
{
int workerid = starpu_worker_get_id();
if (workerid == -1)
return;
struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
STARPU_ASSERT(worker != NULL);
if (!worker->state_sched_op_pending)
return;
STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->sched_mutex);
#ifdef STARPU_SPINLOCK_CHECK
STARPU_ASSERT_MSG(worker->state_relax_refcnt>0, "relax last turn off in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
#else
STARPU_ASSERT(worker->state_relax_refcnt>0);
#endif
worker->state_relax_refcnt--;
#ifdef STARPU_SPINLOCK_CHECK
worker->relax_off_file = file;
worker->relax_off_line = line;
worker->relax_off_func = func;
#endif
STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->sched_mutex);
}
#define starpu_worker_relax_off _starpu_worker_relax_off
#ifdef STARPU_SPINLOCK_CHECK
#define _starpu_worker_relax_off_locked() __starpu_worker_relax_off_locked(__FILE__, __LINE__, __starpu_func__)
static inline void __starpu_worker_relax_off_locked(const char*file, int line, const char* func)
#else
static inline void _starpu_worker_relax_off_locked(void)
#endif
{
int workerid = starpu_worker_get_id();
if (workerid == -1)
return;
struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
STARPU_ASSERT(worker != NULL);
if (!worker->state_sched_op_pending)
return;
#ifdef STARPU_SPINLOCK_CHECK
STARPU_ASSERT_MSG(worker->state_relax_refcnt>0, "relax last turn off in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
#else
STARPU_ASSERT(worker->state_relax_refcnt>0);
#endif
worker->state_relax_refcnt--;
#ifdef STARPU_SPINLOCK_CHECK
worker->relax_off_file = file;
worker->relax_off_line = line;
worker->relax_off_func = func;
#endif
}
static inline int _starpu_worker_get_relax_state(void)
{
int workerid = starpu_worker_get_id();
if (workerid < 0)
return 1;
struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
STARPU_ASSERT(worker != NULL);
return worker->state_relax_refcnt != 0;
}
#define starpu_worker_get_relax_state _starpu_worker_get_relax_state
static inline void _starpu_worker_lock(int workerid)
{
struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
STARPU_ASSERT(worker != NULL);
int cur_workerid = starpu_worker_get_id();
if (workerid != cur_workerid)
{
starpu_worker_relax_on();
STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->sched_mutex);
while (!worker->state_relax_refcnt)
{
STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
}
}
else
{
STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->sched_mutex);
}
}
#define starpu_worker_lock _starpu_worker_lock
static inline int _starpu_worker_trylock(int workerid)
{
struct _starpu_worker *cur_worker = _starpu_get_local_worker_key();
STARPU_ASSERT(cur_worker != NULL);
int cur_workerid = cur_worker->workerid;
struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
STARPU_ASSERT(worker != NULL);
/* Start with ourself */
int ret = STARPU_PTHREAD_MUTEX_TRYLOCK_SCHED(&cur_worker->sched_mutex);
if (ret)
return ret;
if (workerid == cur_workerid)
/* We only needed to lock ourself */
return 0;
/* Now try to lock the other worker */
ret = STARPU_PTHREAD_MUTEX_TRYLOCK_SCHED(&worker->sched_mutex);
if (!ret)
{
/* Good, check that it is relaxed */
ret = !worker->state_relax_refcnt;
if (ret)
STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->sched_mutex);
}
if (!ret)
_starpu_worker_relax_on_locked(cur_worker);
STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&cur_worker->sched_mutex);
return ret;
}
#define starpu_worker_trylock _starpu_worker_trylock
static inline void _starpu_worker_unlock(int workerid)
{
struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
STARPU_ASSERT(worker != NULL);
STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->sched_mutex);
int cur_workerid = starpu_worker_get_id();
if (workerid != cur_workerid)
{
starpu_worker_relax_off();
}
}
#define starpu_worker_unlock _starpu_worker_unlock
static inline void _starpu_worker_lock_self(void)
{
int workerid = starpu_worker_get_id_check();
struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
STARPU_ASSERT(worker != NULL);
STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->sched_mutex);
}
#define starpu_worker_lock_self _starpu_worker_lock_self
static inline void _starpu_worker_unlock_self(void)
{
int workerid = starpu_worker_get_id_check();
struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
STARPU_ASSERT(worker != NULL);
STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->sched_mutex);
}
#define starpu_worker_unlock_self _starpu_worker_unlock_self
static inline int _starpu_wake_worker_relax(int workerid)
{
_starpu_worker_lock(workerid);
int ret = starpu_wake_worker_locked(workerid);
_starpu_worker_unlock(workerid);
return ret;
}
#define starpu_wake_worker_relax _starpu_wake_worker_relax
int starpu_wake_worker_relax_light(int workerid) STARPU_ATTRIBUTE_VISIBILITY_DEFAULT;
void _starpu_worker_refuse_task(struct _starpu_worker *worker, struct starpu_task *task);
void _starpu_set_catch_signals(int do_catch_signal);
int _starpu_get_catch_signals(void);
static inline int _starpu_perf_counter_paused(void)
{
STARPU_RMB();
return STARPU_UNLIKELY(_starpu_config.perf_counter_pause_depth > 0);
}
void _starpu_crash_add_hook(void (*hook_func)(void));
void _starpu_crash_call_hooks();
uint32_t _starpu_worker_exists(struct starpu_task *task);
/* @}*/
#pragma GCC visibility pop
#endif // __WORKERS_H__
_starpu_barrier_counter
Definition barrier_counter.h:29
datawizard.h
driver_cpu.h
driver_cuda.h
driver_mpi_source.h
driver_opencl.h
drivers.h
errorcheck.h
_starpu_worker_status
_starpu_worker_status
Definition errorcheck.h:44
_starpu_worker_status_index
_starpu_worker_status_index
Definition errorcheck.h:32
fxt.h
starpu_worker_archtype
starpu_worker_archtype
Definition starpu_worker.h:66
starpu_worker_get_id_check
#define starpu_worker_get_id_check()
Definition starpu_worker.h:257
starpu_node_kind
starpu_node_kind
Definition starpu_worker.h:44
STARPU_NARCH
@ STARPU_NARCH
Definition starpu_worker.h:74
STARPU_MAX_RAM
@ STARPU_MAX_RAM
Definition starpu_worker.h:54
jobs.h
knobs.h
starpu_perf_counter_sample
Definition knobs.h:190
malloc.h
perfmodel.h
_starpu_worker_start_state
void _starpu_worker_start_state(int workerid, enum _starpu_worker_status_index index, struct timespec *start_time)
sched_ctx.h
_starpu_worker_apply_deferred_ctx_changes
void _starpu_worker_apply_deferred_ctx_changes(void)
sched_ctx_list.h
_starpu_sched_ctx_list
Definition sched_ctx_list.h:27
sched_policy.h
simgrid.h
_starpu_spinlock
Definition starpu_spinlock.h:82
_starpu_combined_worker
Definition workers.h:311
_starpu_combined_worker::padding
char padding[STARPU_CACHELINE_SIZE]
Definition workers.h:331
_starpu_combined_worker::memory_node
unsigned memory_node
Definition workers.h:315
_starpu_combined_worker::perf_arch
struct starpu_perfmodel_arch perf_arch
Definition workers.h:312
_starpu_combined_worker::worker_mask
uint32_t worker_mask
Definition workers.h:313
_starpu_driver_info
Definition workers.h:523
_starpu_driver_info::get_hwloc_obj
hwloc_obj_t(* get_hwloc_obj)(hwloc_topology_t topology, int devid)
Definition workers.h:535
_starpu_driver_info::init_worker_binding
void(* init_worker_binding)(struct _starpu_machine_config *config, int no_mp_config STARPU_ATTRIBUTE_UNUSED, struct _starpu_worker *workerarg)
Definition workers.h:532
_starpu_driver_info::driver_ops
const struct _starpu_driver_ops * driver_ops
Definition workers.h:530
_starpu_driver_info::wait_for_worker_initialization
unsigned wait_for_worker_initialization
Definition workers.h:529
_starpu_driver_info::init_worker_memory
void(* init_worker_memory)(struct _starpu_machine_config *config, int no_mp_config STARPU_ATTRIBUTE_UNUSED, struct _starpu_worker *workerarg)
Definition workers.h:533
_starpu_driver_info::alpha
double alpha
Definition workers.h:528
_starpu_driver_info::memory_kind
enum starpu_node_kind memory_kind
Definition workers.h:527
_starpu_driver_info::name_upper
const char * name_upper
Definition workers.h:524
_starpu_driver_info::name_var
const char * name_var
Definition workers.h:525
_starpu_driver_info::name_lower
const char * name_lower
Definition workers.h:526
_starpu_driver_ops
Definition drivers.h:26
_starpu_machine_config
Definition workers.h:441
_starpu_machine_config::combined_workers
struct _starpu_combined_worker combined_workers[STARPU_NMAX_COMBINEDWORKERS]
Definition workers.h:475
_starpu_machine_config::bindid_workers
struct _starpu_machine_config::@6 * bindid_workers
_starpu_machine_config::running
unsigned running
Definition workers.h:499
_starpu_machine_config::nbindid
unsigned nbindid
Definition workers.h:488
_starpu_machine_config::pause_depth
int pause_depth
Definition workers.h:505
_starpu_machine_config::current_tcpip_deviceid
int current_tcpip_deviceid
Definition workers.h:458
_starpu_machine_config::sched_ctxs
struct _starpu_sched_ctx sched_ctxs[STARPU_NMAX_SCHED_CTXS+1]
Definition workers.h:508
_starpu_machine_config::arch_nodeid
int arch_nodeid[STARPU_NARCH]
Definition workers.h:461
_starpu_machine_config::nodes
struct _starpu_node nodes[STARPU_MAXNODES]
Definition workers.h:471
_starpu_machine_config::current_bindid
int current_bindid
Definition workers.h:450
_starpu_machine_config::workers
struct _starpu_worker workers[STARPU_NMAXWORKERS]
Definition workers.h:468
_starpu_machine_config::worker_mask
uint32_t worker_mask
Definition workers.h:493
_starpu_machine_config::padding2
char padding2[STARPU_CACHELINE_SIZE]
Definition workers.h:480
_starpu_machine_config::perf_counter_pause_depth
int perf_counter_pause_depth
Definition workers.h:516
_starpu_machine_config::submitting
unsigned submitting
Definition workers.h:511
_starpu_machine_config::current_devid
int current_devid[STARPU_NARCH]
Definition workers.h:455
_starpu_machine_config::padding1
char padding1[STARPU_CACHELINE_SIZE]
Definition workers.h:464
_starpu_machine_config::conf
struct starpu_conf conf
Definition workers.h:496
_starpu_machine_topology
Definition workers.h:352
_starpu_machine_topology::nhwdevices
unsigned nhwdevices[STARPU_NARCH]
Definition workers.h:388
_starpu_machine_topology::tree
struct starpu_tree * tree
Definition workers.h:368
_starpu_machine_topology::hwtopology
hwloc_topology_t hwtopology
Definition workers.h:363
_starpu_machine_topology::hip_th_per_dev
int hip_th_per_dev
Definition workers.h:414
_starpu_machine_topology::devid
int devid[STARPU_NARCH][STARPU_NMAXDEVS]
Definition workers.h:404
_starpu_machine_topology::workers_devid
unsigned workers_devid[STARPU_NARCH][STARPU_NMAXWORKERS]
Definition workers.h:437
_starpu_machine_topology::nworker
unsigned nworker[STARPU_NARCH][STARPU_NMAXDEVS]
Definition workers.h:400
_starpu_machine_topology::cuda_th_per_stream
int cuda_th_per_stream
Definition workers.h:407
_starpu_machine_topology::nworkers
unsigned nworkers
Definition workers.h:354
_starpu_machine_topology::ncombinedworkers
unsigned ncombinedworkers
Definition workers.h:357
_starpu_machine_topology::nhwworker
unsigned nhwworker[STARPU_NARCH][STARPU_NMAXDEVS]
Definition workers.h:392
_starpu_machine_topology::nusedpus
unsigned nusedpus
Definition workers.h:383
_starpu_machine_topology::ndevices
unsigned ndevices[STARPU_NARCH]
Definition workers.h:396
_starpu_machine_topology::workers_nbindid
unsigned workers_nbindid
Definition workers.h:429
_starpu_machine_topology::cuda_th_per_dev
int cuda_th_per_dev
Definition workers.h:409
_starpu_machine_topology::nhwpus
unsigned nhwpus
Definition workers.h:373
_starpu_machine_topology::workers_bindid
unsigned workers_bindid[STARPU_NMAXWORKERS]
Definition workers.h:423
_starpu_machine_topology::hip_th_per_stream
int hip_th_per_stream
Definition workers.h:412
_starpu_machine_topology::firstusedpu
unsigned firstusedpu
Definition workers.h:378
_starpu_memory_driver_info
Definition workers.h:547
_starpu_memory_driver_info::worker_archtype
enum starpu_worker_archtype worker_archtype
Definition workers.h:549
_starpu_memory_driver_info::ops
const struct _starpu_node_ops * ops
Definition workers.h:550
_starpu_memory_driver_info::name_upper
const char * name_upper
Definition workers.h:548
_starpu_node_ops
Definition node_ops.h:92
_starpu_node
Definition workers.h:70
_starpu_node::mc_lock
struct _starpu_spinlock mc_lock
Definition workers.h:76
_starpu_node::evictable
unsigned evictable
Definition workers.h:106
_starpu_node::prefetch_out_of_memory
volatile int prefetch_out_of_memory
Definition workers.h:103
_starpu_node::data_requests
struct _starpu_data_request_prio_list data_requests[STARPU_MAXNODES][2]
Definition workers.h:112
_starpu_node::mc_nb
unsigned mc_nb
Definition workers.h:89
_starpu_node::reclaiming
unsigned reclaiming
Definition workers.h:98
_starpu_node::nfreechunks
int nfreechunks
Definition workers.h:129
_starpu_node::mc_list
struct _starpu_mem_chunk_list mc_list
Definition workers.h:81
_starpu_node::chunk_mutex
starpu_pthread_mutex_t chunk_mutex
Definition workers.h:131
_starpu_node::padding
char padding[STARPU_CACHELINE_SIZE]
Definition workers.h:150
_starpu_node::data_requests_pending
struct _starpu_data_request_prio_list data_requests_pending[STARPU_MAXNODES][2]
Definition workers.h:118
_starpu_node::chunks
struct _starpu_chunk_list chunks
Definition workers.h:127
_starpu_node::mc_dirty_head
struct _starpu_mem_chunk * mc_dirty_head
Definition workers.h:84
_starpu_node::tidying
unsigned tidying
Definition workers.h:96
_starpu_sched_ctx
Definition sched_ctx.h:48
_starpu_sched_ctx::id
unsigned id
Definition sched_ctx.h:50
_starpu_worker_set
Definition workers.h:339
_starpu_worker_set::started
unsigned started
Definition workers.h:343
_starpu_worker_set::worker_thread
starpu_pthread_t worker_thread
Definition workers.h:341
_starpu_worker_set::ready_cond
starpu_pthread_cond_t ready_cond
Definition workers.h:346
_starpu_worker
Definition workers.h:155
_starpu_worker::state_blocked_in_parallel_observed
unsigned state_blocked_in_parallel_observed
Definition workers.h:193
_starpu_worker::current_task
struct starpu_task * current_task
Definition workers.h:220
_starpu_worker::state_block_in_parallel_ack
unsigned state_block_in_parallel_ack
Definition workers.h:195
_starpu_worker::state_relax_refcnt
unsigned state_relax_refcnt
Definition workers.h:180
_starpu_worker::task_transferring
struct starpu_task * task_transferring
Definition workers.h:258
_starpu_worker::state_changing_ctx_waiting
unsigned state_changing_ctx_waiting
Definition workers.h:190
_starpu_worker::subworkerid
unsigned subworkerid
Definition workers.h:164
_starpu_worker::devid
unsigned devid
Definition workers.h:162
_starpu_worker::thread_changing_ctx
starpu_pthread_t thread_changing_ctx
Definition workers.h:206
_starpu_worker::current_rank
int current_rank
Definition workers.h:168
_starpu_worker::state_unblock_in_parallel_req
unsigned state_unblock_in_parallel_req
Definition workers.h:196
_starpu_worker::nb_buffers_transferred
unsigned nb_buffers_transferred
Definition workers.h:256
_starpu_worker::worker_thread
starpu_pthread_t worker_thread
Definition workers.h:161
_starpu_worker::ready_cond
starpu_pthread_cond_t ready_cond
Definition workers.h:171
_starpu_worker::memory_node
unsigned memory_node
Definition workers.h:172
_starpu_worker::workerid
int workerid
Definition workers.h:166
_starpu_worker::nb_buffers_totransfer
unsigned nb_buffers_totransfer
Definition workers.h:257
_starpu_worker::devnum
unsigned devnum
Definition workers.h:163
_starpu_worker::state_keep_awake
unsigned state_keep_awake
Definition workers.h:239
_starpu_worker::set
struct _starpu_worker_set * set
Definition workers.h:233
_starpu_worker::worker_size
int worker_size
Definition workers.h:169
_starpu_worker::pop_ctx_priority
unsigned pop_ctx_priority
Definition workers.h:274
_starpu_worker::state_unblock_in_parallel_ack
unsigned state_unblock_in_parallel_ack
Definition workers.h:197
_starpu_worker::numa_memory_node
unsigned numa_memory_node
Definition workers.h:173
_starpu_worker::pipeline_stuck
unsigned char pipeline_stuck
Definition workers.h:232
_starpu_worker::pipeline_length
unsigned char pipeline_length
Definition workers.h:231
_starpu_worker::nsched_ctxs
unsigned nsched_ctxs
Definition workers.h:247
_starpu_worker::spinning_backoff
unsigned spinning_backoff
Definition workers.h:254
_starpu_worker::state_blocked_in_parallel
unsigned state_blocked_in_parallel
Definition workers.h:192
_starpu_worker::current_ordered_task_order
unsigned current_ordered_task_order
Definition workers.h:219
_starpu_worker::started_cond
starpu_pthread_cond_t started_cond
Definition workers.h:170
_starpu_worker::sched_mutex
starpu_pthread_mutex_t sched_mutex
Definition workers.h:179
_starpu_worker::arch
enum starpu_worker_archtype arch
Definition workers.h:158
_starpu_worker::run_by_starpu
unsigned run_by_starpu
Definition workers.h:242
_starpu_worker::local_ordered_tasks_size
unsigned local_ordered_tasks_size
Definition workers.h:217
_starpu_worker::local_ordered_tasks
struct starpu_task ** local_ordered_tasks
Definition workers.h:216
_starpu_worker::is_slave_somewhere
unsigned is_slave_somewhere
Definition workers.h:275
_starpu_worker::driver_worker_set
struct _starpu_worker_set * driver_worker_set
Definition workers.h:234
_starpu_worker::sched_cond
starpu_pthread_cond_t sched_cond
Definition workers.h:178
_starpu_worker::has_prev_init
unsigned has_prev_init
Definition workers.h:250
_starpu_worker::ntasks
unsigned char ntasks
Definition workers.h:230
_starpu_worker::bindid
int bindid
Definition workers.h:165
_starpu_worker::state_block_in_parallel_req
unsigned state_block_in_parallel_req
Definition workers.h:194
_starpu_worker::state_changing_ctx_notice
unsigned state_changing_ctx_notice
Definition workers.h:191
_starpu_worker::state_sched_op_pending
unsigned state_sched_op_pending
Definition workers.h:189
_starpu_worker::combined_workerid
int combined_workerid
Definition workers.h:167
_starpu_worker::first_task
unsigned char first_task
Definition workers.h:229
_starpu_worker::current_ordered_task
unsigned current_ordered_task
Definition workers.h:218
_starpu_worker::block_in_parallel_ref_count
unsigned block_in_parallel_ref_count
Definition workers.h:205
_starpu_worker::worker_mask
uint32_t worker_mask
Definition workers.h:159
_starpu_worker::status
enum _starpu_worker_status status
Definition workers.h:238
thread.h
timing.h
topology.h
utils.h