Loading...
Searching...
No Matches
runtime_atomics.h
Go to the documentation of this file.
#ifndef HALIDE_RUNTIME_RUNTIME_ATOMICS_H
#define HALIDE_RUNTIME_RUNTIME_ATOMICS_H
// This file provides an abstraction layer over the __sync/__atomic builtins
// in Clang; for various reasons, we use __sync for 32-bit targets, and
// __atomic for 64-bit. At some point it may be desirable/necessary to
// migrate 32-bit to __atomic as well, at which time this file can
// likely go away. See https://github.com/halide/Halide/issues/7431 for
// a discussion of the history and issues as to why we work this way.
#include "HalideRuntime.h"
namespace Halide {
namespace Runtime {
namespace Internal {
namespace Synchronization {
namespace {
// TODO: most of these wrappers should do the remove_volatile for secondary arguments;
// I've only put it in place for the locations necessary at this time.
template<class T>
struct remove_volatile {
typedef T type;
};
template<class T>
struct remove_volatile<volatile T> {
typedef T type;
};
#ifdef BITS_32
ALWAYS_INLINE uintptr_t atomic_and_fetch_release(uintptr_t *addr, uintptr_t val) {
return __sync_and_and_fetch(addr, val);
}
template<typename T>
ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
return __sync_fetch_and_add(addr, val);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE TV atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
return __sync_fetch_and_add(addr, val);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE TV atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
return __sync_fetch_and_sub(addr, val);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE TV atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
return __sync_fetch_and_or(addr, val);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE TV atomic_add_fetch_sequentially_consistent(T *addr, TV val) {
return __sync_add_and_fetch(addr, val);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE TV atomic_sub_fetch_sequentially_consistent(T *addr, TV val) {
return __sync_sub_and_fetch(addr, val);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE bool cas_strong_sequentially_consistent_helper(T *addr, TV *expected, TV *desired) {
TV oldval = *expected;
TV gotval = __sync_val_compare_and_swap(addr, oldval, *desired);
*expected = gotval;
return oldval == gotval;
}
ALWAYS_INLINE bool atomic_cas_strong_release_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
return cas_strong_sequentially_consistent_helper(addr, expected, desired);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
return cas_strong_sequentially_consistent_helper(addr, expected, desired);
}
ALWAYS_INLINE bool atomic_cas_weak_release_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
return cas_strong_sequentially_consistent_helper(addr, expected, desired);
}
template<typename T>
ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
return cas_strong_sequentially_consistent_helper(addr, expected, desired);
}
ALWAYS_INLINE bool atomic_cas_weak_relaxed_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
return cas_strong_sequentially_consistent_helper(addr, expected, desired);
}
ALWAYS_INLINE bool atomic_cas_weak_acquire_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
return cas_strong_sequentially_consistent_helper(addr, expected, desired);
}
template<typename T>
ALWAYS_INLINE T atomic_fetch_and_release(T *addr, T val) {
return __sync_fetch_and_and(addr, val);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE TV atomic_fetch_and_sequentially_consistent(T *addr, TV val) {
return __sync_fetch_and_and(addr, val);
}
template<typename T>
ALWAYS_INLINE void atomic_load_relaxed(T *addr, T *val) {
*val = *addr;
}
template<typename T>
ALWAYS_INLINE void atomic_load_acquire(T *addr, T *val) {
__sync_synchronize();
*val = *addr;
}
template<typename T>
ALWAYS_INLINE T atomic_exchange_acquire(T *addr, T val) {
// Despite the name, this is really just an exchange operation with acquire ordering.
return __sync_lock_test_and_set(addr, val);
}
ALWAYS_INLINE uintptr_t atomic_or_fetch_relaxed(uintptr_t *addr, uintptr_t val) {
return __sync_or_and_fetch(addr, val);
}
template<typename T>
ALWAYS_INLINE void atomic_store_relaxed(T *addr, T *val) {
*addr = *val;
}
template<typename T>
ALWAYS_INLINE void atomic_store_release(T *addr, T *val) {
*addr = *val;
__sync_synchronize();
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
*addr = *val;
__sync_synchronize();
}
ALWAYS_INLINE void atomic_thread_fence_acquire() {
__sync_synchronize();
}
ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
__sync_synchronize();
}
#else
ALWAYS_INLINE uintptr_t atomic_and_fetch_release(uintptr_t *addr, uintptr_t val) {
return __atomic_and_fetch(addr, val, __ATOMIC_RELEASE);
}
template<typename T>
ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
return __atomic_fetch_add(addr, val, __ATOMIC_ACQ_REL);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE TV atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE TV atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE TV atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE TV atomic_add_fetch_sequentially_consistent(T *addr, TV val) {
return __atomic_add_fetch(addr, val, __ATOMIC_SEQ_CST);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE TV atomic_sub_fetch_sequentially_consistent(T *addr, TV val) {
return __atomic_sub_fetch(addr, val, __ATOMIC_SEQ_CST);
}
ALWAYS_INLINE bool atomic_cas_strong_release_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
return __atomic_compare_exchange(addr, expected, desired, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
return __atomic_compare_exchange(addr, expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
}
template<typename T>
ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
return __atomic_compare_exchange(addr, expected, desired, true, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
}
ALWAYS_INLINE bool atomic_cas_weak_release_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
return __atomic_compare_exchange(addr, expected, desired, true, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
}
ALWAYS_INLINE bool atomic_cas_weak_relaxed_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
return __atomic_compare_exchange(addr, expected, desired, true, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
}
ALWAYS_INLINE bool atomic_cas_weak_acquire_relaxed(uintptr_t *addr, uintptr_t *expected, uintptr_t *desired) {
return __atomic_compare_exchange(addr, expected, desired, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
}
template<typename T>
ALWAYS_INLINE uintptr_t atomic_fetch_and_release(T *addr, T val) {
return __atomic_fetch_and(addr, val, __ATOMIC_RELEASE);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE uintptr_t atomic_fetch_and_sequentially_consistent(T *addr, TV val) {
return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
}
template<typename T>
ALWAYS_INLINE void atomic_load_relaxed(T *addr, T *val) {
__atomic_load(addr, val, __ATOMIC_RELAXED);
}
template<typename T>
ALWAYS_INLINE void atomic_load_acquire(T *addr, T *val) {
__atomic_load(addr, val, __ATOMIC_ACQUIRE);
__sync_synchronize();
*val = *addr;
}
template<typename T>
ALWAYS_INLINE T atomic_exchange_acquire(T *addr, T val) {
T result;
__atomic_exchange(addr, &val, &result, __ATOMIC_ACQUIRE);
return result;
}
ALWAYS_INLINE uintptr_t atomic_or_fetch_relaxed(uintptr_t *addr, uintptr_t val) {
return __atomic_or_fetch(addr, val, __ATOMIC_RELAXED);
}
template<typename T>
ALWAYS_INLINE void atomic_store_relaxed(T *addr, T *val) {
__atomic_store(addr, val, __ATOMIC_RELAXED);
}
template<typename T>
ALWAYS_INLINE void atomic_store_release(T *addr, T *val) {
__atomic_store(addr, val, __ATOMIC_RELEASE);
}
template<typename T, typename TV = typename remove_volatile<T>::type>
ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
__atomic_store(addr, val, __ATOMIC_SEQ_CST);
}
ALWAYS_INLINE void atomic_thread_fence_acquire() {
__atomic_thread_fence(__ATOMIC_ACQUIRE);
}
ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
__atomic_thread_fence(__ATOMIC_SEQ_CST);
}
#endif
} // namespace
} // namespace Synchronization
} // namespace Internal
} // namespace Runtime
} // namespace Halide
#endif // HALIDE_RUNTIME_RUNTIME_ATOMICS_H
HalideRuntime.h
This file declares the routines used by Halide internally in its runtime.
Halide::Runtime::Internal::Synchronization
Definition runtime_atomics.h:16
Halide::Runtime::Internal
Definition constants.h:13
Halide::Runtime
Definition constants.h:12
Halide
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
Definition AbstractGenerator.h:19
uintptr_t
__UINTPTR_TYPE__ uintptr_t
Definition runtime_internal.h:75
ALWAYS_INLINE
#define ALWAYS_INLINE
Definition runtime_internal.h:57