Loading...
Searching...
No Matches
coherency.h
Go to the documentation of this file.
/* StarPU --- Runtime system for heterogeneous multicore architectures.
*
* Copyright (C) 2008-2025 University of Bordeaux, CNRS (LaBRI UMR 5800), Inria
*
* StarPU is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation; either version 2.1 of the License, or (at
* your option) any later version.
*
* StarPU is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
*
* See the GNU Lesser General Public License in COPYING.LGPL for more details.
*/
#ifndef __COHERENCY__H__
#define __COHERENCY__H__
#include <starpu.h>
#include <common/config.h>
#include <common/starpu_spinlock.h>
#include <common/rwlock.h>
#include <common/timing.h>
#include <common/fxt.h>
#include <common/list.h>
#include <datawizard/interfaces/data_interface.h>
#include <datawizard/datastats.h>
#include <datawizard/memstats.h>
#include <datawizard/data_request.h>
#pragma GCC visibility push(hidden)
enum _starpu_cache_state
{
STARPU_OWNER,
STARPU_SHARED,
STARPU_INVALID
};
struct _starpu_data_replicate
{
starpu_data_handle_t handle;
void *data_interface;
int refcnt;
char memory_node;
enum _starpu_cache_state state: 2;
unsigned relaxed_coherency:2;
unsigned initialized:1;
unsigned allocated:1;
unsigned automatically_allocated:1;
unsigned map_write:1;
#define STARPU_UNMAPPED -1
int mapped;
uint32_t requested;
struct _starpu_data_request *request[STARPU_MAXNODES];
struct _starpu_data_request *last_request[STARPU_MAXNODES];
/* Which request is loading data here */
struct _starpu_data_request *load_request;
unsigned nb_tasks_prefetch;
struct _starpu_mem_chunk * mc;
};
struct _starpu_data_requester_prio_list;
struct _starpu_jobid_list
{
unsigned long id;
struct _starpu_jobid_list *next;
};
struct _starpu_task_wrapper_list
{
struct starpu_task *task;
struct _starpu_task_wrapper_list *next;
};
struct _starpu_task_wrapper_dlist
{
struct starpu_task *task;
struct _starpu_task_wrapper_dlist *next;
struct _starpu_task_wrapper_dlist *prev;
};
extern int _starpu_has_not_important_data;
typedef void (*_starpu_data_handle_unregister_hook)(starpu_data_handle_t);
struct _starpu_data_state
{
int magic;
struct _starpu_data_requester_prio_list req_list;
unsigned refcnt;
unsigned unlocking_reqs;
enum starpu_data_access_mode current_mode;
struct _starpu_spinlock header_lock;
unsigned busy_count;
unsigned busy_waiting;
starpu_pthread_mutex_t busy_mutex;
starpu_pthread_cond_t busy_cond;
struct _starpu_data_state *root_handle;
struct _starpu_data_state *father_handle;
starpu_data_handle_t *active_children;
unsigned active_nchildren;
starpu_data_handle_t **active_readonly_children;
unsigned *active_readonly_nchildren;
unsigned nactive_readonly_children;
unsigned nsiblings;
starpu_data_handle_t *siblings;
unsigned sibling_index;
unsigned depth;
#ifdef STARPU_BUBBLE
starpu_pthread_mutex_t unpartition_mutex;
#endif
starpu_data_handle_t children;
unsigned nchildren;
unsigned nplans;
struct starpu_codelet *switch_cl;
unsigned switch_cl_nparts;
unsigned partitioned;
unsigned part_readonly:1;
unsigned active:1;
unsigned active_ro:1;
struct _starpu_data_replicate per_node[STARPU_MAXNODES];
struct _starpu_data_replicate *per_worker;
struct starpu_data_interface_ops *ops;
uint32_t footprint;
/* The following bitfields are set from the application initialization */
unsigned is_not_important:1;
unsigned ooc:1;
unsigned sequential_consistency:1;
unsigned readonly:1;
int home_node;
uint32_t wt_mask;
unsigned aliases;
starpu_data_handle_t readonly_dup;
starpu_data_handle_t readonly_dup_of;
/* The following bitfields are set from the application submission thread */
unsigned initialized:1;
#ifdef STARPU_OPENMP
unsigned removed_from_context_hash:1;
#endif
/* The following field is set by StarPU at execution time */
unsigned char lazy_unregister;
starpu_pthread_mutex_t sequential_consistency_mutex;
enum starpu_data_access_mode last_submitted_mode;
struct starpu_task *last_sync_task;
struct _starpu_task_wrapper_dlist last_submitted_accessors;
unsigned last_submitted_ghost_sync_id_is_valid;
unsigned long last_submitted_ghost_sync_id;
struct _starpu_jobid_list *last_submitted_ghost_accessors_id;
struct _starpu_task_wrapper_list *post_sync_tasks;
unsigned post_sync_tasks_cnt;
/*
* Reductions
*/
struct starpu_codelet *redux_cl;
struct starpu_codelet *init_cl;
void *redux_cl_arg;
void *init_cl_arg;
unsigned reduction_refcnt;
struct _starpu_data_requester_prio_list reduction_req_list;
starpu_data_handle_t *reduction_tmp_handles;
struct _starpu_data_request *write_invalidation_req;
void *mpi_data;
_starpu_memory_stats_t memory_stats;
unsigned int mf_node; //XXX
_starpu_data_handle_unregister_hook unregister_hook;
struct starpu_arbiter *arbiter;
struct _starpu_data_requester_prio_list arbitered_req_list;
int last_locality;
unsigned dimensions;
int coordinates[5];
void *user_data;
void *sched_data;
};
int _starpu_fetch_data_on_node(starpu_data_handle_t handle, int node, struct _starpu_data_replicate *replicate,
enum starpu_data_access_mode mode, unsigned detached,
struct starpu_task *task, enum starpu_is_prefetch is_prefetch, unsigned async,
void (*callback_func)(void *), void *callback_arg, int prio, const char *origin);
void _starpu_release_data_on_node(struct _starpu_data_state *state, uint32_t default_wt_mask,
enum starpu_data_access_mode down_to_mode,
struct _starpu_data_replicate *replicate);
void _starpu_update_data_state(starpu_data_handle_t handle,
struct _starpu_data_replicate *requesting_replicate,
enum starpu_data_access_mode mode);
uint32_t _starpu_get_data_refcnt(struct _starpu_data_state *state, unsigned node);
size_t _starpu_data_get_size(starpu_data_handle_t handle);
size_t _starpu_data_get_alloc_size(starpu_data_handle_t handle);
starpu_ssize_t _starpu_data_get_max_size(starpu_data_handle_t handle);
uint32_t _starpu_data_get_footprint(starpu_data_handle_t handle);
void __starpu_push_task_output(struct _starpu_job *j);
void _starpu_push_task_output(struct _starpu_job *j);
struct _starpu_worker;
STARPU_ATTRIBUTE_WARN_UNUSED_RESULT
int _starpu_fetch_task_input(struct starpu_task *task, struct _starpu_job *j, int async);
void _starpu_fetch_task_input_tail(struct starpu_task *task, struct _starpu_job *j, struct _starpu_worker *worker);
void _starpu_fetch_nowhere_task_input(struct _starpu_job *j);
int _starpu_select_src_node(struct _starpu_data_state *state, unsigned destination);
int _starpu_determine_request_path(starpu_data_handle_t handle,
int src_node, int dst_node,
enum starpu_data_access_mode mode, int max_len,
unsigned *src_nodes, unsigned *dst_nodes,
unsigned *handling_nodes, unsigned write_invalidation);
struct _starpu_data_request *_starpu_create_request_to_fetch_data(starpu_data_handle_t handle,
struct _starpu_data_replicate *dst_replicate,
enum starpu_data_access_mode mode,
struct starpu_task *task, enum starpu_is_prefetch is_prefetch,
unsigned async,
void (*callback_func)(void *), void *callback_arg, int prio, const char *origin);
void _starpu_init_data_replicate(starpu_data_handle_t handle, struct _starpu_data_replicate *replicate, int workerid);
void _starpu_data_start_reduction_mode(starpu_data_handle_t handle);
void _starpu_data_end_reduction_mode(starpu_data_handle_t handle, int priority);
void _starpu_data_end_reduction_mode_terminate(starpu_data_handle_t handle);
void _starpu_data_unmap(starpu_data_handle_t handle, unsigned node);
void _starpu_data_set_unregister_hook(starpu_data_handle_t handle, _starpu_data_handle_unregister_hook func) STARPU_ATTRIBUTE_VISIBILITY_DEFAULT;
#pragma GCC visibility pop
#endif // __COHERENCY__H__
_starpu_data_state::busy_count
unsigned busy_count
Definition coherency.h:165
_starpu_data_state::mpi_data
void * mpi_data
Definition coherency.h:320
_starpu_data_state::post_sync_tasks
struct _starpu_task_wrapper_list * post_sync_tasks
Definition coherency.h:288
_starpu_data_state::readonly
unsigned readonly
Definition coherency.h:232
_starpu_data_state::redux_cl
struct starpu_codelet * redux_cl
Definition coherency.h:299
_starpu_create_request_to_fetch_data
struct _starpu_data_request * _starpu_create_request_to_fetch_data(starpu_data_handle_t handle, struct _starpu_data_replicate *dst_replicate, enum starpu_data_access_mode mode, struct starpu_task *task, enum starpu_is_prefetch is_prefetch, unsigned async, void(*callback_func)(void *), void *callback_arg, int prio, const char *origin)
_starpu_data_state::nactive_readonly_children
unsigned nactive_readonly_children
Definition coherency.h:178
_starpu_data_state::switch_cl
struct starpu_codelet * switch_cl
Definition coherency.h:195
_starpu_data_state::arbitered_req_list
struct _starpu_data_requester_prio_list arbitered_req_list
Definition coherency.h:331
_starpu_data_state::unregister_hook
_starpu_data_handle_unregister_hook unregister_hook
Definition coherency.h:327
_starpu_data_replicate::request
struct _starpu_data_request * request[STARPU_MAXNODES]
Definition coherency.h:99
_starpu_data_state::unpartition_mutex
starpu_pthread_mutex_t unpartition_mutex
Definition coherency.h:186
_starpu_data_state::write_invalidation_req
struct _starpu_data_request * write_invalidation_req
Definition coherency.h:317
_starpu_data_state::aliases
unsigned aliases
Definition coherency.h:242
_starpu_data_replicate::state
enum _starpu_cache_state state
Definition coherency.h:59
_starpu_data_state::active
unsigned active
Definition coherency.h:209
_starpu_data_state::current_mode
enum starpu_data_access_mode current_mode
Definition coherency.h:157
_starpu_data_state::unlocking_reqs
unsigned unlocking_reqs
Definition coherency.h:153
_starpu_data_state::siblings
starpu_data_handle_t * siblings
Definition coherency.h:181
_starpu_data_state::switch_cl_nparts
unsigned switch_cl_nparts
Definition coherency.h:197
_starpu_fetch_data_on_node
int _starpu_fetch_data_on_node(starpu_data_handle_t handle, int node, struct _starpu_data_replicate *replicate, enum starpu_data_access_mode mode, unsigned detached, struct starpu_task *task, enum starpu_is_prefetch is_prefetch, unsigned async, void(*callback_func)(void *), void *callback_arg, int prio, const char *origin)
_starpu_data_state::father_handle
struct _starpu_data_state * father_handle
Definition coherency.h:173
_starpu_data_state::children
starpu_data_handle_t children
Definition coherency.h:190
_starpu_data_replicate::automatically_allocated
unsigned automatically_allocated
Definition coherency.h:75
_starpu_data_state::ooc
unsigned ooc
Definition coherency.h:228
_starpu_data_state::sequential_consistency_mutex
starpu_pthread_mutex_t sequential_consistency_mutex
Definition coherency.h:267
_starpu_data_state::nplans
unsigned nplans
Definition coherency.h:193
_starpu_data_replicate::mc
struct _starpu_mem_chunk * mc
Definition coherency.h:113
_starpu_data_replicate::mapped
int mapped
Definition coherency.h:87
_starpu_data_replicate::refcnt
int refcnt
Definition coherency.h:54
_starpu_data_replicate::relaxed_coherency
unsigned relaxed_coherency
Definition coherency.h:63
_starpu_data_state::active_nchildren
unsigned active_nchildren
Definition coherency.h:175
_starpu_data_state::last_submitted_mode
enum starpu_data_access_mode last_submitted_mode
Definition coherency.h:273
_starpu_data_state::last_locality
int last_locality
Definition coherency.h:336
_starpu_fetch_task_input
STARPU_ATTRIBUTE_WARN_UNUSED_RESULT int _starpu_fetch_task_input(struct starpu_task *task, struct _starpu_job *j, int async)
_starpu_data_state::per_node
struct _starpu_data_replicate per_node[STARPU_MAXNODES]
Definition coherency.h:214
_starpu_data_state::header_lock
struct _starpu_spinlock header_lock
Definition coherency.h:159
_starpu_data_state::readonly_dup_of
starpu_data_handle_t readonly_dup_of
Definition coherency.h:248
_starpu_data_state::lazy_unregister
unsigned char lazy_unregister
Definition coherency.h:263
_starpu_data_state::busy_waiting
unsigned busy_waiting
Definition coherency.h:167
_starpu_data_state::sched_data
void * sched_data
Definition coherency.h:349
_starpu_data_state::active_readonly_nchildren
unsigned * active_readonly_nchildren
Definition coherency.h:177
_starpu_data_replicate::map_write
unsigned map_write
Definition coherency.h:82
_starpu_data_state::partitioned
unsigned partitioned
Definition coherency.h:204
_starpu_data_state::initialized
unsigned initialized
Definition coherency.h:254
_starpu_data_replicate::data_interface
void * data_interface
Definition coherency.h:51
_starpu_data_replicate::last_request
struct _starpu_data_request * last_request[STARPU_MAXNODES]
Definition coherency.h:101
_starpu_data_state::home_node
int home_node
Definition coherency.h:235
_starpu_data_replicate::nb_tasks_prefetch
unsigned nb_tasks_prefetch
Definition coherency.h:110
_starpu_data_state::root_handle
struct _starpu_data_state * root_handle
Definition coherency.h:172
_starpu_data_state::last_submitted_ghost_sync_id_is_valid
unsigned last_submitted_ghost_sync_id_is_valid
Definition coherency.h:283
_starpu_release_data_on_node
void _starpu_release_data_on_node(struct _starpu_data_state *state, uint32_t default_wt_mask, enum starpu_data_access_mode down_to_mode, struct _starpu_data_replicate *replicate)
_starpu_push_task_output
void _starpu_push_task_output(struct _starpu_job *j)
_starpu_data_state::readonly_dup
starpu_data_handle_t readonly_dup
Definition coherency.h:245
_starpu_data_state::wt_mask
uint32_t wt_mask
Definition coherency.h:238
_starpu_data_state::depth
unsigned depth
Definition coherency.h:183
_starpu_data_replicate::initialized
unsigned initialized
Definition coherency.h:66
_starpu_data_state::nsiblings
unsigned nsiblings
Definition coherency.h:180
_starpu_data_state::reduction_req_list
struct _starpu_data_requester_prio_list reduction_req_list
Definition coherency.h:312
_starpu_data_state::user_data
void * user_data
Definition coherency.h:345
_starpu_data_state::refcnt
unsigned refcnt
Definition coherency.h:151
_starpu_data_state::dimensions
unsigned dimensions
Definition coherency.h:340
_starpu_data_state::sequential_consistency
unsigned sequential_consistency
Definition coherency.h:230
_starpu_data_state::reduction_refcnt
unsigned reduction_refcnt
Definition coherency.h:307
_starpu_data_replicate::requested
uint32_t requested
Definition coherency.h:96
_starpu_data_state::is_not_important
unsigned is_not_important
Definition coherency.h:226
_starpu_data_replicate::allocated
unsigned allocated
Definition coherency.h:69
_starpu_data_state::active_children
starpu_data_handle_t * active_children
Definition coherency.h:174
_starpu_data_state::footprint
uint32_t footprint
Definition coherency.h:220
_starpu_data_state::part_readonly
unsigned part_readonly
Definition coherency.h:206
_starpu_data_replicate
Definition coherency.h:47
_starpu_data_state
Definition coherency.h:145
_starpu_jobid_list
Definition coherency.h:119
_starpu_task_wrapper_dlist
Definition coherency.h:133
_starpu_task_wrapper_list
Definition coherency.h:126
data_interface.h
data_request.h
datastats.h
fxt.h
_starpu_job
Definition jobs.h:75
list.h
memstats.h
rwlock.h
starpu_spinlock.h
_starpu_spinlock
Definition starpu_spinlock.h:82
_starpu_worker
Definition workers.h:155
timing.h