Loading...
Searching...
No Matches
kmp_atomic.cpp
/*
* kmp_atomic.cpp -- ATOMIC implementation routines
*/
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "kmp_atomic.h"
#include "kmp.h" // TRUE, asm routines prototypes
typedef unsigned char uchar;
typedef unsigned short ushort;
/*
* Global vars
*/
#ifndef KMP_GOMP_COMPAT
int __kmp_atomic_mode = 1; // Intel perf
#else
int __kmp_atomic_mode = 2; // GOMP compatibility
#endif /* KMP_GOMP_COMPAT */
KMP_ALIGN(128)
// Control access to all user coded atomics in Gnu compat mode
kmp_atomic_lock_t __kmp_atomic_lock;
// Control access to all user coded atomics for 1-byte fixed data types
kmp_atomic_lock_t __kmp_atomic_lock_1i;
// Control access to all user coded atomics for 2-byte fixed data types
kmp_atomic_lock_t __kmp_atomic_lock_2i;
// Control access to all user coded atomics for 4-byte fixed data types
kmp_atomic_lock_t __kmp_atomic_lock_4i;
// Control access to all user coded atomics for kmp_real32 data type
kmp_atomic_lock_t __kmp_atomic_lock_4r;
// Control access to all user coded atomics for 8-byte fixed data types
kmp_atomic_lock_t __kmp_atomic_lock_8i;
// Control access to all user coded atomics for kmp_real64 data type
kmp_atomic_lock_t __kmp_atomic_lock_8r;
// Control access to all user coded atomics for complex byte data type
kmp_atomic_lock_t __kmp_atomic_lock_8c;
// Control access to all user coded atomics for long double data type
kmp_atomic_lock_t __kmp_atomic_lock_10r;
// Control access to all user coded atomics for _Quad data type
kmp_atomic_lock_t __kmp_atomic_lock_16r;
// Control access to all user coded atomics for double complex data type
kmp_atomic_lock_t __kmp_atomic_lock_16c;
// Control access to all user coded atomics for long double complex type
kmp_atomic_lock_t __kmp_atomic_lock_20c;
// Control access to all user coded atomics for _Quad complex data type
kmp_atomic_lock_t __kmp_atomic_lock_32c;
/* 2007-03-02:
Without "volatile" specifier in OP_CMPXCHG and MIN_MAX_CMPXCHG we have a bug
on *_32 and *_32e. This is just a temporary workaround for the problem. It
seems the right solution is writing OP_CMPXCHG and MIN_MAX_CMPXCHG routines
in assembler language. */
#define KMP_ATOMIC_VOLATILE volatile
#if (KMP_ARCH_X86) && KMP_HAVE_QUAD
static inline Quad_a4_t operator+(Quad_a4_t &lhs, Quad_a4_t &rhs) {
return lhs.q + rhs.q;
}
static inline Quad_a4_t operator-(Quad_a4_t &lhs, Quad_a4_t &rhs) {
return lhs.q - rhs.q;
}
static inline Quad_a4_t operator*(Quad_a4_t &lhs, Quad_a4_t &rhs) {
return lhs.q * rhs.q;
}
static inline Quad_a4_t operator/(Quad_a4_t &lhs, Quad_a4_t &rhs) {
return lhs.q / rhs.q;
}
static inline bool operator<(Quad_a4_t &lhs, Quad_a4_t &rhs) {
return lhs.q < rhs.q;
}
static inline bool operator>(Quad_a4_t &lhs, Quad_a4_t &rhs) {
return lhs.q > rhs.q;
}
static inline Quad_a16_t operator+(Quad_a16_t &lhs, Quad_a16_t &rhs) {
return lhs.q + rhs.q;
}
static inline Quad_a16_t operator-(Quad_a16_t &lhs, Quad_a16_t &rhs) {
return lhs.q - rhs.q;
}
static inline Quad_a16_t operator*(Quad_a16_t &lhs, Quad_a16_t &rhs) {
return lhs.q * rhs.q;
}
static inline Quad_a16_t operator/(Quad_a16_t &lhs, Quad_a16_t &rhs) {
return lhs.q / rhs.q;
}
static inline bool operator<(Quad_a16_t &lhs, Quad_a16_t &rhs) {
return lhs.q < rhs.q;
}
static inline bool operator>(Quad_a16_t &lhs, Quad_a16_t &rhs) {
return lhs.q > rhs.q;
}
static inline kmp_cmplx128_a4_t operator+(kmp_cmplx128_a4_t &lhs,
kmp_cmplx128_a4_t &rhs) {
return lhs.q + rhs.q;
}
static inline kmp_cmplx128_a4_t operator-(kmp_cmplx128_a4_t &lhs,
kmp_cmplx128_a4_t &rhs) {
return lhs.q - rhs.q;
}
static inline kmp_cmplx128_a4_t operator*(kmp_cmplx128_a4_t &lhs,
kmp_cmplx128_a4_t &rhs) {
return lhs.q * rhs.q;
}
static inline kmp_cmplx128_a4_t operator/(kmp_cmplx128_a4_t &lhs,
kmp_cmplx128_a4_t &rhs) {
return lhs.q / rhs.q;
}
static inline kmp_cmplx128_a16_t operator+(kmp_cmplx128_a16_t &lhs,
kmp_cmplx128_a16_t &rhs) {
return lhs.q + rhs.q;
}
static inline kmp_cmplx128_a16_t operator-(kmp_cmplx128_a16_t &lhs,
kmp_cmplx128_a16_t &rhs) {
return lhs.q - rhs.q;
}
static inline kmp_cmplx128_a16_t operator*(kmp_cmplx128_a16_t &lhs,
kmp_cmplx128_a16_t &rhs) {
return lhs.q * rhs.q;
}
static inline kmp_cmplx128_a16_t operator/(kmp_cmplx128_a16_t &lhs,
kmp_cmplx128_a16_t &rhs) {
return lhs.q / rhs.q;
}
#endif // (KMP_ARCH_X86) && KMP_HAVE_QUAD
// ATOMIC implementation routines -----------------------------------------
// One routine for each operation and operand type.
// All routines declarations looks like
// void __kmpc_atomic_RTYPE_OP( ident_t*, int, TYPE *lhs, TYPE rhs );
#define KMP_CHECK_GTID \
if (gtid == KMP_GTID_UNKNOWN) { \
gtid = __kmp_entry_gtid(); \
} // check and get gtid when needed
// Beginning of a definition (provides name, parameters, gebug trace)
// TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned
// fixed)
// OP_ID - operation identifier (add, sub, mul, ...)
// TYPE - operands' type
#define ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
TYPE *lhs, TYPE rhs) { \
KMP_DEBUG_ASSERT(__kmp_init_serial); \
KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
// ------------------------------------------------------------------------
// Lock variables used for critical sections for various size operands
#define ATOMIC_LOCK0 __kmp_atomic_lock // all types, for Gnu compat
#define ATOMIC_LOCK1i __kmp_atomic_lock_1i // char
#define ATOMIC_LOCK2i __kmp_atomic_lock_2i // short
#define ATOMIC_LOCK4i __kmp_atomic_lock_4i // long int
#define ATOMIC_LOCK4r __kmp_atomic_lock_4r // float
#define ATOMIC_LOCK8i __kmp_atomic_lock_8i // long long int
#define ATOMIC_LOCK8r __kmp_atomic_lock_8r // double
#define ATOMIC_LOCK8c __kmp_atomic_lock_8c // float complex
#define ATOMIC_LOCK10r __kmp_atomic_lock_10r // long double
#define ATOMIC_LOCK16r __kmp_atomic_lock_16r // _Quad
#define ATOMIC_LOCK16c __kmp_atomic_lock_16c // double complex
#define ATOMIC_LOCK20c __kmp_atomic_lock_20c // long double complex
#define ATOMIC_LOCK32c __kmp_atomic_lock_32c // _Quad complex
// ------------------------------------------------------------------------
// Operation on *lhs, rhs bound by critical section
// OP - operator (it's supposed to contain an assignment)
// LCK_ID - lock identifier
// Note: don't check gtid as it should always be valid
// 1, 2-byte - expect valid parameter, other - check before this macro
#define OP_CRITICAL(OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
(*lhs) OP(rhs); \
\
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
#define OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
(*lhs) = (TYPE)((*lhs)OP rhs); \
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
// ------------------------------------------------------------------------
// For GNU compatibility, we may need to use a critical section,
// even though it is not required by the ISA.
//
// On IA-32 architecture, all atomic operations except for fixed 4 byte add,
// sub, and bitwise logical ops, and 1 & 2 byte logical ops use a common
// critical section. On Intel(R) 64, all atomic operations are done with fetch
// and add or compare and exchange. Therefore, the FLAG parameter to this
// macro is either KMP_ARCH_X86 or 0 (or 1, for Intel-specific extension which
// require a critical section, where we predict that they will be implemented
// in the Gnu codegen by calling GOMP_atomic_start() / GOMP_atomic_end()).
//
// When the OP_GOMP_CRITICAL macro is used in a *CRITICAL* macro construct,
// the FLAG parameter should always be 1. If we know that we will be using
// a critical section, then we want to make certain that we use the generic
// lock __kmp_atomic_lock to protect the atomic update, and not of of the
// locks that are specialized based upon the size or type of the data.
//
// If FLAG is 0, then we are relying on dead code elimination by the build
// compiler to get rid of the useless block of code, and save a needless
// branch at runtime.
#ifdef KMP_GOMP_COMPAT
#define OP_GOMP_CRITICAL(OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
OP_CRITICAL(OP, 0); \
return; \
}
#define OP_UPDATE_GOMP_CRITICAL(TYPE, OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
OP_UPDATE_CRITICAL(TYPE, OP, 0); \
return; \
}
#else
#define OP_GOMP_CRITICAL(OP, FLAG)
#define OP_UPDATE_GOMP_CRITICAL(TYPE, OP, FLAG)
#endif /* KMP_GOMP_COMPAT */
#if KMP_MIC
#define KMP_DO_PAUSE _mm_delay_32(1)
#else
#define KMP_DO_PAUSE
#endif /* KMP_MIC */
// ------------------------------------------------------------------------
// Operation on *lhs, rhs using "compare_and_store" routine
// TYPE - operands' type
// BITS - size in bits, used to distinguish low level calls
// OP - operator
#define OP_CMPXCHG(TYPE, BITS, OP) \
{ \
TYPE old_value, new_value; \
old_value = *(TYPE volatile *)lhs; \
new_value = (TYPE)(old_value OP rhs); \
while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
*VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
KMP_DO_PAUSE; \
\
old_value = *(TYPE volatile *)lhs; \
new_value = (TYPE)(old_value OP rhs); \
} \
}
#if USE_CMPXCHG_FIX
// 2007-06-25:
// workaround for C78287 (complex(kind=4) data type). lin_32, lin_32e, win_32
// and win_32e are affected (I verified the asm). Compiler ignores the volatile
// qualifier of the temp_val in the OP_CMPXCHG macro. This is a problem of the
// compiler. Related tracker is C76005, targeted to 11.0. I verified the asm of
// the workaround.
#define OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
{ \
struct _sss { \
TYPE cmp; \
kmp_int##BITS *vvv; \
}; \
struct _sss old_value, new_value; \
old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \
new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \
*old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
new_value.cmp = (TYPE)(old_value.cmp OP rhs); \
while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
*VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \
KMP_DO_PAUSE; \
\
*old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
new_value.cmp = (TYPE)(old_value.cmp OP rhs); \
} \
}
// end of the first part of the workaround for C78287
#endif // USE_CMPXCHG_FIX
#if KMP_OS_WINDOWS && (KMP_ARCH_AARCH64 || KMP_ARCH_ARM)
// Undo explicit type casts to get MSVC ARM64 to build. Uses
// OP_CMPXCHG_WORKAROUND definition for OP_CMPXCHG
#undef OP_CMPXCHG
#define OP_CMPXCHG(TYPE, BITS, OP) \
{ \
struct _sss { \
TYPE cmp; \
kmp_int##BITS *vvv; \
}; \
struct _sss old_value, new_value; \
old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \
new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \
*old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
new_value.cmp = old_value.cmp OP rhs; \
while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
*VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \
KMP_DO_PAUSE; \
\
*old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
new_value.cmp = old_value.cmp OP rhs; \
} \
}
#undef OP_UPDATE_CRITICAL
#define OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
(*lhs) = (*lhs)OP rhs; \
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
#endif // KMP_OS_WINDOWS && (KMP_ARCH_AARCH64 || KMP_ARCH_ARM)
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
// ------------------------------------------------------------------------
// X86 or X86_64: no alignment problems ====================================
#define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
/* OP used as a sign for subtraction: (lhs-rhs) --> (lhs+-rhs) */ \
KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
}
// -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG(TYPE, BITS, OP) \
}
#if USE_CMPXCHG_FIX
// -------------------------------------------------------------------------
// workaround for C78287 (complex(kind=4) data type)
#define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
MASK, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
}
// end of the second part of the workaround for C78287
#endif // USE_CMPXCHG_FIX
#else
// -------------------------------------------------------------------------
// Code for other architectures that don't handle unaligned accesses.
#define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
/* OP used as a sign for subtraction: (lhs-rhs) --> (lhs+-rhs) */ \
KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
} else { \
KMP_CHECK_GTID; \
OP_UPDATE_CRITICAL(TYPE, OP, \
LCK_ID) /* unaligned address - use critical */ \
} \
}
// -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
} else { \
KMP_CHECK_GTID; \
OP_UPDATE_CRITICAL(TYPE, OP, \
LCK_ID) /* unaligned address - use critical */ \
} \
}
#if USE_CMPXCHG_FIX
// -------------------------------------------------------------------------
// workaround for C78287 (complex(kind=4) data type)
#define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
MASK, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
} else { \
KMP_CHECK_GTID; \
OP_UPDATE_CRITICAL(TYPE, OP, \
LCK_ID) /* unaligned address - use critical */ \
} \
}
// end of the second part of the workaround for C78287
#endif // USE_CMPXCHG_FIX
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
// Routines for ATOMIC 4-byte operands addition and subtraction
ATOMIC_FIXED_ADD(fixed4, add, kmp_int32, 32, +, 4i, 3,
0) // __kmpc_atomic_fixed4_add
ATOMIC_FIXED_ADD(fixed4, sub, kmp_int32, 32, -, 4i, 3,
0) // __kmpc_atomic_fixed4_sub
ATOMIC_CMPXCHG(float4, add, kmp_real32, 32, +, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_add
ATOMIC_CMPXCHG(float4, sub, kmp_real32, 32, -, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_sub
// Routines for ATOMIC 8-byte operands addition and subtraction
ATOMIC_FIXED_ADD(fixed8, add, kmp_int64, 64, +, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_add
ATOMIC_FIXED_ADD(fixed8, sub, kmp_int64, 64, -, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_sub
ATOMIC_CMPXCHG(float8, add, kmp_real64, 64, +, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_add
ATOMIC_CMPXCHG(float8, sub, kmp_real64, 64, -, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_sub
// ------------------------------------------------------------------------
// Entries definition for integer operands
// TYPE_ID - operands type and size (fixed4, float4)
// OP_ID - operation identifier (add, sub, mul, ...)
// TYPE - operand type
// BITS - size in bits, used to distinguish low level calls
// OP - operator (used in critical section)
// LCK_ID - lock identifier, used to possibly distinguish lock variable
// MASK - used for alignment check
// TYPE_ID,OP_ID, TYPE, BITS,OP,LCK_ID,MASK,GOMP_FLAG
// ------------------------------------------------------------------------
// Routines for ATOMIC integer operands, other operators
// ------------------------------------------------------------------------
// TYPE_ID,OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG
ATOMIC_CMPXCHG(fixed1, add, kmp_int8, 8, +, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_add
ATOMIC_CMPXCHG(fixed1, andb, kmp_int8, 8, &, 1i, 0,
0) // __kmpc_atomic_fixed1_andb
ATOMIC_CMPXCHG(fixed1, div, kmp_int8, 8, /, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_div
ATOMIC_CMPXCHG(fixed1u, div, kmp_uint8, 8, /, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_div
ATOMIC_CMPXCHG(fixed1, mul, kmp_int8, 8, *, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_mul
ATOMIC_CMPXCHG(fixed1, orb, kmp_int8, 8, |, 1i, 0,
0) // __kmpc_atomic_fixed1_orb
ATOMIC_CMPXCHG(fixed1, shl, kmp_int8, 8, <<, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_shl
ATOMIC_CMPXCHG(fixed1, shr, kmp_int8, 8, >>, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_shr
ATOMIC_CMPXCHG(fixed1u, shr, kmp_uint8, 8, >>, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_shr
ATOMIC_CMPXCHG(fixed1, sub, kmp_int8, 8, -, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_sub
ATOMIC_CMPXCHG(fixed1, xor, kmp_int8, 8, ^, 1i, 0,
0) // __kmpc_atomic_fixed1_xor
ATOMIC_CMPXCHG(fixed2, add, kmp_int16, 16, +, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_add
ATOMIC_CMPXCHG(fixed2, andb, kmp_int16, 16, &, 2i, 1,
0) // __kmpc_atomic_fixed2_andb
ATOMIC_CMPXCHG(fixed2, div, kmp_int16, 16, /, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_div
ATOMIC_CMPXCHG(fixed2u, div, kmp_uint16, 16, /, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_div
ATOMIC_CMPXCHG(fixed2, mul, kmp_int16, 16, *, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_mul
ATOMIC_CMPXCHG(fixed2, orb, kmp_int16, 16, |, 2i, 1,
0) // __kmpc_atomic_fixed2_orb
ATOMIC_CMPXCHG(fixed2, shl, kmp_int16, 16, <<, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_shl
ATOMIC_CMPXCHG(fixed2, shr, kmp_int16, 16, >>, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_shr
ATOMIC_CMPXCHG(fixed2u, shr, kmp_uint16, 16, >>, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_shr
ATOMIC_CMPXCHG(fixed2, sub, kmp_int16, 16, -, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_sub
ATOMIC_CMPXCHG(fixed2, xor, kmp_int16, 16, ^, 2i, 1,
0) // __kmpc_atomic_fixed2_xor
ATOMIC_CMPXCHG(fixed4, andb, kmp_int32, 32, &, 4i, 3,
0) // __kmpc_atomic_fixed4_andb
ATOMIC_CMPXCHG(fixed4, div, kmp_int32, 32, /, 4i, 3,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_div
ATOMIC_CMPXCHG(fixed4u, div, kmp_uint32, 32, /, 4i, 3,
KMP_ARCH_X86) // __kmpc_atomic_fixed4u_div
ATOMIC_CMPXCHG(fixed4, mul, kmp_int32, 32, *, 4i, 3,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_mul
ATOMIC_CMPXCHG(fixed4, orb, kmp_int32, 32, |, 4i, 3,
0) // __kmpc_atomic_fixed4_orb
ATOMIC_CMPXCHG(fixed4, shl, kmp_int32, 32, <<, 4i, 3,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_shl
ATOMIC_CMPXCHG(fixed4, shr, kmp_int32, 32, >>, 4i, 3,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_shr
ATOMIC_CMPXCHG(fixed4u, shr, kmp_uint32, 32, >>, 4i, 3,
KMP_ARCH_X86) // __kmpc_atomic_fixed4u_shr
ATOMIC_CMPXCHG(fixed4, xor, kmp_int32, 32, ^, 4i, 3,
0) // __kmpc_atomic_fixed4_xor
ATOMIC_CMPXCHG(fixed8, andb, kmp_int64, 64, &, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_andb
ATOMIC_CMPXCHG(fixed8, div, kmp_int64, 64, /, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_div
ATOMIC_CMPXCHG(fixed8u, div, kmp_uint64, 64, /, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_div
ATOMIC_CMPXCHG(fixed8, mul, kmp_int64, 64, *, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_mul
ATOMIC_CMPXCHG(fixed8, orb, kmp_int64, 64, |, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_orb
ATOMIC_CMPXCHG(fixed8, shl, kmp_int64, 64, <<, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_shl
ATOMIC_CMPXCHG(fixed8, shr, kmp_int64, 64, >>, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_shr
ATOMIC_CMPXCHG(fixed8u, shr, kmp_uint64, 64, >>, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_shr
ATOMIC_CMPXCHG(fixed8, xor, kmp_int64, 64, ^, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_xor
ATOMIC_CMPXCHG(float4, div, kmp_real32, 32, /, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_div
ATOMIC_CMPXCHG(float4, mul, kmp_real32, 32, *, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_mul
ATOMIC_CMPXCHG(float8, div, kmp_real64, 64, /, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_div
ATOMIC_CMPXCHG(float8, mul, kmp_real64, 64, *, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_mul
// TYPE_ID,OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG
/* ------------------------------------------------------------------------ */
/* Routines for C/C++ Reduction operators && and || */
// ------------------------------------------------------------------------
// Need separate macros for &&, || because there is no combined assignment
// TODO: eliminate ATOMIC_CRIT_{L,EQV} macros as not used
#define ATOMIC_CRIT_L(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
OP_CRITICAL(= *lhs OP, LCK_ID) \
}
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
// ------------------------------------------------------------------------
// X86 or X86_64: no alignment problems ===================================
#define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
OP_CMPXCHG(TYPE, BITS, OP) \
}
#else
// ------------------------------------------------------------------------
// Code for other architectures that don't handle unaligned accesses.
#define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
} else { \
KMP_CHECK_GTID; \
OP_CRITICAL(= *lhs OP, LCK_ID) /* unaligned - use critical */ \
} \
}
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
ATOMIC_CMPX_L(fixed1, andl, char, 8, &&, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_andl
ATOMIC_CMPX_L(fixed1, orl, char, 8, ||, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_orl
ATOMIC_CMPX_L(fixed2, andl, short, 16, &&, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_andl
ATOMIC_CMPX_L(fixed2, orl, short, 16, ||, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_orl
ATOMIC_CMPX_L(fixed4, andl, kmp_int32, 32, &&, 4i, 3,
0) // __kmpc_atomic_fixed4_andl
ATOMIC_CMPX_L(fixed4, orl, kmp_int32, 32, ||, 4i, 3,
0) // __kmpc_atomic_fixed4_orl
ATOMIC_CMPX_L(fixed8, andl, kmp_int64, 64, &&, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_andl
ATOMIC_CMPX_L(fixed8, orl, kmp_int64, 64, ||, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_orl
/* ------------------------------------------------------------------------- */
/* Routines for Fortran operators that matched no one in C: */
/* MAX, MIN, .EQV., .NEQV. */
/* Operators .AND., .OR. are covered by __kmpc_atomic_*_{andl,orl} */
/* Intrinsics IAND, IOR, IEOR are covered by __kmpc_atomic_*_{andb,orb,xor} */
// -------------------------------------------------------------------------
// MIN and MAX need separate macros
// OP - operator to check if we need any actions?
#define MIN_MAX_CRITSECT(OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
if (*lhs OP rhs) { /* still need actions? */ \
*lhs = rhs; \
} \
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
// -------------------------------------------------------------------------
#ifdef KMP_GOMP_COMPAT
#define GOMP_MIN_MAX_CRITSECT(OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
MIN_MAX_CRITSECT(OP, 0); \
return; \
}
#else
#define GOMP_MIN_MAX_CRITSECT(OP, FLAG)
#endif /* KMP_GOMP_COMPAT */
// -------------------------------------------------------------------------
#define MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
{ \
TYPE KMP_ATOMIC_VOLATILE temp_val; \
TYPE old_value; \
temp_val = *lhs; \
old_value = temp_val; \
while (old_value OP rhs && /* still need actions? */ \
!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, \
*VOLATILE_CAST(kmp_int##BITS *) & old_value, \
*VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \
temp_val = *lhs; \
old_value = temp_val; \
} \
}
// -------------------------------------------------------------------------
// 1-byte, 2-byte operands - use critical section
#define MIN_MAX_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
if (*lhs OP rhs) { /* need actions? */ \
GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
MIN_MAX_CRITSECT(OP, LCK_ID) \
} \
}
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
// -------------------------------------------------------------------------
// X86 or X86_64: no alignment problems ====================================
#define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
if (*lhs OP rhs) { \
GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
} \
}
#else
// -------------------------------------------------------------------------
// Code for other architectures that don't handle unaligned accesses.
#define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
if (*lhs OP rhs) { \
GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
MIN_MAX_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
} else { \
KMP_CHECK_GTID; \
MIN_MAX_CRITSECT(OP, LCK_ID) /* unaligned address */ \
} \
} \
}
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
MIN_MAX_COMPXCHG(fixed1, max, char, 8, <, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_max
MIN_MAX_COMPXCHG(fixed1, min, char, 8, >, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_min
MIN_MAX_COMPXCHG(fixed2, max, short, 16, <, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_max
MIN_MAX_COMPXCHG(fixed2, min, short, 16, >, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_min
MIN_MAX_COMPXCHG(fixed4, max, kmp_int32, 32, <, 4i, 3,
0) // __kmpc_atomic_fixed4_max
MIN_MAX_COMPXCHG(fixed4, min, kmp_int32, 32, >, 4i, 3,
0) // __kmpc_atomic_fixed4_min
MIN_MAX_COMPXCHG(fixed8, max, kmp_int64, 64, <, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_max
MIN_MAX_COMPXCHG(fixed8, min, kmp_int64, 64, >, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_min
MIN_MAX_COMPXCHG(float4, max, kmp_real32, 32, <, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_max
MIN_MAX_COMPXCHG(float4, min, kmp_real32, 32, >, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_min
MIN_MAX_COMPXCHG(float8, max, kmp_real64, 64, <, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_max
MIN_MAX_COMPXCHG(float8, min, kmp_real64, 64, >, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_min
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
MIN_MAX_CRITICAL(float10, max, long double, <, 10r,
1) // __kmpc_atomic_float10_max
MIN_MAX_CRITICAL(float10, min, long double, >, 10r,
1) // __kmpc_atomic_float10_min
#endif // KMP_ARCH_X86 || KMP_ARCH_X86_64
#if KMP_HAVE_QUAD
MIN_MAX_CRITICAL(float16, max, QUAD_LEGACY, <, 16r,
1) // __kmpc_atomic_float16_max
MIN_MAX_CRITICAL(float16, min, QUAD_LEGACY, >, 16r,
1) // __kmpc_atomic_float16_min
#if (KMP_ARCH_X86)
MIN_MAX_CRITICAL(float16, max_a16, Quad_a16_t, <, 16r,
1) // __kmpc_atomic_float16_max_a16
MIN_MAX_CRITICAL(float16, min_a16, Quad_a16_t, >, 16r,
1) // __kmpc_atomic_float16_min_a16
#endif // (KMP_ARCH_X86)
#endif // KMP_HAVE_QUAD
// ------------------------------------------------------------------------
// Need separate macros for .EQV. because of the need of complement (~)
// OP ignored for critical sections, ^=~ used instead
#define ATOMIC_CRIT_EQV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) /* send assignment */ \
OP_CRITICAL(^= (TYPE) ~, LCK_ID) /* send assignment and complement */ \
}
// ------------------------------------------------------------------------
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
// ------------------------------------------------------------------------
// X86 or X86_64: no alignment problems ===================================
#define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) /* send assignment */ \
OP_CMPXCHG(TYPE, BITS, OP) \
}
// ------------------------------------------------------------------------
#else
// ------------------------------------------------------------------------
// Code for other architectures that don't handle unaligned accesses.
#define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \
if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
} else { \
KMP_CHECK_GTID; \
OP_CRITICAL(^= (TYPE) ~, LCK_ID) /* unaligned address - use critical */ \
} \
}
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
ATOMIC_CMPXCHG(fixed1, neqv, kmp_int8, 8, ^, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_neqv
ATOMIC_CMPXCHG(fixed2, neqv, kmp_int16, 16, ^, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_neqv
ATOMIC_CMPXCHG(fixed4, neqv, kmp_int32, 32, ^, 4i, 3,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_neqv
ATOMIC_CMPXCHG(fixed8, neqv, kmp_int64, 64, ^, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_neqv
ATOMIC_CMPX_EQV(fixed1, eqv, kmp_int8, 8, ^~, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_eqv
ATOMIC_CMPX_EQV(fixed2, eqv, kmp_int16, 16, ^~, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_eqv
ATOMIC_CMPX_EQV(fixed4, eqv, kmp_int32, 32, ^~, 4i, 3,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_eqv
ATOMIC_CMPX_EQV(fixed8, eqv, kmp_int64, 64, ^~, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_eqv
// ------------------------------------------------------------------------
// Routines for Extended types: long double, _Quad, complex flavours (use
// critical section)
// TYPE_ID, OP_ID, TYPE - detailed above
// OP - operator
// LCK_ID - lock identifier, used to possibly distinguish lock variable
#define ATOMIC_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) /* send assignment */ \
OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) /* send assignment */ \
}
/* ------------------------------------------------------------------------- */
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
// routines for long double type
ATOMIC_CRITICAL(float10, add, long double, +, 10r,
1) // __kmpc_atomic_float10_add
ATOMIC_CRITICAL(float10, sub, long double, -, 10r,
1) // __kmpc_atomic_float10_sub
ATOMIC_CRITICAL(float10, mul, long double, *, 10r,
1) // __kmpc_atomic_float10_mul
ATOMIC_CRITICAL(float10, div, long double, /, 10r,
1) // __kmpc_atomic_float10_div
#endif // KMP_ARCH_X86 || KMP_ARCH_X86_64
#if KMP_HAVE_QUAD
// routines for _Quad type
ATOMIC_CRITICAL(float16, add, QUAD_LEGACY, +, 16r,
1) // __kmpc_atomic_float16_add
ATOMIC_CRITICAL(float16, sub, QUAD_LEGACY, -, 16r,
1) // __kmpc_atomic_float16_sub
ATOMIC_CRITICAL(float16, mul, QUAD_LEGACY, *, 16r,
1) // __kmpc_atomic_float16_mul
ATOMIC_CRITICAL(float16, div, QUAD_LEGACY, /, 16r,
1) // __kmpc_atomic_float16_div
#if (KMP_ARCH_X86)
ATOMIC_CRITICAL(float16, add_a16, Quad_a16_t, +, 16r,
1) // __kmpc_atomic_float16_add_a16
ATOMIC_CRITICAL(float16, sub_a16, Quad_a16_t, -, 16r,
1) // __kmpc_atomic_float16_sub_a16
ATOMIC_CRITICAL(float16, mul_a16, Quad_a16_t, *, 16r,
1) // __kmpc_atomic_float16_mul_a16
ATOMIC_CRITICAL(float16, div_a16, Quad_a16_t, /, 16r,
1) // __kmpc_atomic_float16_div_a16
#endif // (KMP_ARCH_X86)
#endif // KMP_HAVE_QUAD
// routines for complex types
#if USE_CMPXCHG_FIX
// workaround for C78287 (complex(kind=4) data type)
ATOMIC_CMPXCHG_WORKAROUND(cmplx4, add, kmp_cmplx32, 64, +, 8c, 7,
1) // __kmpc_atomic_cmplx4_add
ATOMIC_CMPXCHG_WORKAROUND(cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7,
1) // __kmpc_atomic_cmplx4_sub
ATOMIC_CMPXCHG_WORKAROUND(cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7,
1) // __kmpc_atomic_cmplx4_mul
ATOMIC_CMPXCHG_WORKAROUND(cmplx4, div, kmp_cmplx32, 64, /, 8c, 7,
1) // __kmpc_atomic_cmplx4_div
// end of the workaround for C78287
#else
ATOMIC_CRITICAL(cmplx4, add, kmp_cmplx32, +, 8c, 1) // __kmpc_atomic_cmplx4_add
ATOMIC_CRITICAL(cmplx4, sub, kmp_cmplx32, -, 8c, 1) // __kmpc_atomic_cmplx4_sub
ATOMIC_CRITICAL(cmplx4, mul, kmp_cmplx32, *, 8c, 1) // __kmpc_atomic_cmplx4_mul
ATOMIC_CRITICAL(cmplx4, div, kmp_cmplx32, /, 8c, 1) // __kmpc_atomic_cmplx4_div
#endif // USE_CMPXCHG_FIX
ATOMIC_CRITICAL(cmplx8, add, kmp_cmplx64, +, 16c, 1) // __kmpc_atomic_cmplx8_add
ATOMIC_CRITICAL(cmplx8, sub, kmp_cmplx64, -, 16c, 1) // __kmpc_atomic_cmplx8_sub
ATOMIC_CRITICAL(cmplx8, mul, kmp_cmplx64, *, 16c, 1) // __kmpc_atomic_cmplx8_mul
ATOMIC_CRITICAL(cmplx8, div, kmp_cmplx64, /, 16c, 1) // __kmpc_atomic_cmplx8_div
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
ATOMIC_CRITICAL(cmplx10, add, kmp_cmplx80, +, 20c,
1) // __kmpc_atomic_cmplx10_add
ATOMIC_CRITICAL(cmplx10, sub, kmp_cmplx80, -, 20c,
1) // __kmpc_atomic_cmplx10_sub
ATOMIC_CRITICAL(cmplx10, mul, kmp_cmplx80, *, 20c,
1) // __kmpc_atomic_cmplx10_mul
ATOMIC_CRITICAL(cmplx10, div, kmp_cmplx80, /, 20c,
1) // __kmpc_atomic_cmplx10_div
#endif // KMP_ARCH_X86 || KMP_ARCH_X86_64
#if KMP_HAVE_QUAD
ATOMIC_CRITICAL(cmplx16, add, CPLX128_LEG, +, 32c,
1) // __kmpc_atomic_cmplx16_add
ATOMIC_CRITICAL(cmplx16, sub, CPLX128_LEG, -, 32c,
1) // __kmpc_atomic_cmplx16_sub
ATOMIC_CRITICAL(cmplx16, mul, CPLX128_LEG, *, 32c,
1) // __kmpc_atomic_cmplx16_mul
ATOMIC_CRITICAL(cmplx16, div, CPLX128_LEG, /, 32c,
1) // __kmpc_atomic_cmplx16_div
#if (KMP_ARCH_X86)
ATOMIC_CRITICAL(cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c,
1) // __kmpc_atomic_cmplx16_add_a16
ATOMIC_CRITICAL(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1) // __kmpc_atomic_cmplx16_sub_a16
ATOMIC_CRITICAL(cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c,
1) // __kmpc_atomic_cmplx16_mul_a16
ATOMIC_CRITICAL(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1) // __kmpc_atomic_cmplx16_div_a16
#endif // (KMP_ARCH_X86)
#endif // KMP_HAVE_QUAD
// OpenMP 4.0: x = expr binop x for non-commutative operations.
// Supported only on IA-32 architecture and Intel(R) 64
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
// ------------------------------------------------------------------------
// Operation on *lhs, rhs bound by critical section
// OP - operator (it's supposed to contain an assignment)
// LCK_ID - lock identifier
// Note: don't check gtid as it should always be valid
// 1, 2-byte - expect valid parameter, other - check before this macro
#define OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
(*lhs) = (TYPE)((rhs)OP(*lhs)); \
\
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
#ifdef KMP_GOMP_COMPAT
#define OP_GOMP_CRITICAL_REV(TYPE, OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
OP_CRITICAL_REV(TYPE, OP, 0); \
return; \
}
#else
#define OP_GOMP_CRITICAL_REV(TYPE, OP, FLAG)
#endif /* KMP_GOMP_COMPAT */
// Beginning of a definition (provides name, parameters, gebug trace)
// TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned
// fixed)
// OP_ID - operation identifier (add, sub, mul, ...)
// TYPE - operands' type
#define ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev(ident_t *id_ref, int gtid, \
TYPE *lhs, TYPE rhs) { \
KMP_DEBUG_ASSERT(__kmp_init_serial); \
KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid));
// ------------------------------------------------------------------------
// Operation on *lhs, rhs using "compare_and_store" routine
// TYPE - operands' type
// BITS - size in bits, used to distinguish low level calls
// OP - operator
// Note: temp_val introduced in order to force the compiler to read
// *lhs only once (w/o it the compiler reads *lhs twice)
#define OP_CMPXCHG_REV(TYPE, BITS, OP) \
{ \
TYPE KMP_ATOMIC_VOLATILE temp_val; \
TYPE old_value, new_value; \
temp_val = *lhs; \
old_value = temp_val; \
new_value = (TYPE)(rhs OP old_value); \
while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
*VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
KMP_DO_PAUSE; \
\
temp_val = *lhs; \
old_value = temp_val; \
new_value = (TYPE)(rhs OP old_value); \
} \
}
// -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_REV(TYPE, BITS, OP) \
}
// ------------------------------------------------------------------------
// Entries definition for integer operands
// TYPE_ID - operands type and size (fixed4, float4)
// OP_ID - operation identifier (add, sub, mul, ...)
// TYPE - operand type
// BITS - size in bits, used to distinguish low level calls
// OP - operator (used in critical section)
// LCK_ID - lock identifier, used to possibly distinguish lock variable
// TYPE_ID,OP_ID, TYPE, BITS,OP,LCK_ID,GOMP_FLAG
// ------------------------------------------------------------------------
// Routines for ATOMIC integer operands, other operators
// ------------------------------------------------------------------------
// TYPE_ID,OP_ID, TYPE, BITS, OP, LCK_ID, GOMP_FLAG
ATOMIC_CMPXCHG_REV(fixed1, div, kmp_int8, 8, /, 1i,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_div_rev
ATOMIC_CMPXCHG_REV(fixed1u, div, kmp_uint8, 8, /, 1i,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_div_rev
ATOMIC_CMPXCHG_REV(fixed1, shl, kmp_int8, 8, <<, 1i,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_shl_rev
ATOMIC_CMPXCHG_REV(fixed1, shr, kmp_int8, 8, >>, 1i,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_shr_rev
ATOMIC_CMPXCHG_REV(fixed1u, shr, kmp_uint8, 8, >>, 1i,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_shr_rev
ATOMIC_CMPXCHG_REV(fixed1, sub, kmp_int8, 8, -, 1i,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_sub_rev
ATOMIC_CMPXCHG_REV(fixed2, div, kmp_int16, 16, /, 2i,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_div_rev
ATOMIC_CMPXCHG_REV(fixed2u, div, kmp_uint16, 16, /, 2i,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_div_rev
ATOMIC_CMPXCHG_REV(fixed2, shl, kmp_int16, 16, <<, 2i,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_shl_rev
ATOMIC_CMPXCHG_REV(fixed2, shr, kmp_int16, 16, >>, 2i,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_shr_rev
ATOMIC_CMPXCHG_REV(fixed2u, shr, kmp_uint16, 16, >>, 2i,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_shr_rev
ATOMIC_CMPXCHG_REV(fixed2, sub, kmp_int16, 16, -, 2i,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_sub_rev
ATOMIC_CMPXCHG_REV(fixed4, div, kmp_int32, 32, /, 4i,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_div_rev
ATOMIC_CMPXCHG_REV(fixed4u, div, kmp_uint32, 32, /, 4i,
KMP_ARCH_X86) // __kmpc_atomic_fixed4u_div_rev
ATOMIC_CMPXCHG_REV(fixed4, shl, kmp_int32, 32, <<, 4i,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_shl_rev
ATOMIC_CMPXCHG_REV(fixed4, shr, kmp_int32, 32, >>, 4i,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_shr_rev
ATOMIC_CMPXCHG_REV(fixed4u, shr, kmp_uint32, 32, >>, 4i,
KMP_ARCH_X86) // __kmpc_atomic_fixed4u_shr_rev
ATOMIC_CMPXCHG_REV(fixed4, sub, kmp_int32, 32, -, 4i,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_sub_rev
ATOMIC_CMPXCHG_REV(fixed8, div, kmp_int64, 64, /, 8i,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_div_rev
ATOMIC_CMPXCHG_REV(fixed8u, div, kmp_uint64, 64, /, 8i,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_div_rev
ATOMIC_CMPXCHG_REV(fixed8, shl, kmp_int64, 64, <<, 8i,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_shl_rev
ATOMIC_CMPXCHG_REV(fixed8, shr, kmp_int64, 64, >>, 8i,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_shr_rev
ATOMIC_CMPXCHG_REV(fixed8u, shr, kmp_uint64, 64, >>, 8i,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_shr_rev
ATOMIC_CMPXCHG_REV(fixed8, sub, kmp_int64, 64, -, 8i,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_sub_rev
ATOMIC_CMPXCHG_REV(float4, div, kmp_real32, 32, /, 4r,
KMP_ARCH_X86) // __kmpc_atomic_float4_div_rev
ATOMIC_CMPXCHG_REV(float4, sub, kmp_real32, 32, -, 4r,
KMP_ARCH_X86) // __kmpc_atomic_float4_sub_rev
ATOMIC_CMPXCHG_REV(float8, div, kmp_real64, 64, /, 8r,
KMP_ARCH_X86) // __kmpc_atomic_float8_div_rev
ATOMIC_CMPXCHG_REV(float8, sub, kmp_real64, 64, -, 8r,
KMP_ARCH_X86) // __kmpc_atomic_float8_sub_rev
// TYPE_ID,OP_ID, TYPE, BITS,OP,LCK_ID, GOMP_FLAG
// ------------------------------------------------------------------------
// Routines for Extended types: long double, _Quad, complex flavours (use
// critical section)
// TYPE_ID, OP_ID, TYPE - detailed above
// OP - operator
// LCK_ID - lock identifier, used to possibly distinguish lock variable
#define ATOMIC_CRITICAL_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
}
/* ------------------------------------------------------------------------- */
// routines for long double type
ATOMIC_CRITICAL_REV(float10, sub, long double, -, 10r,
1) // __kmpc_atomic_float10_sub_rev
ATOMIC_CRITICAL_REV(float10, div, long double, /, 10r,
1) // __kmpc_atomic_float10_div_rev
#if KMP_HAVE_QUAD
// routines for _Quad type
ATOMIC_CRITICAL_REV(float16, sub, QUAD_LEGACY, -, 16r,
1) // __kmpc_atomic_float16_sub_rev
ATOMIC_CRITICAL_REV(float16, div, QUAD_LEGACY, /, 16r,
1) // __kmpc_atomic_float16_div_rev
#if (KMP_ARCH_X86)
ATOMIC_CRITICAL_REV(float16, sub_a16, Quad_a16_t, -, 16r,
1) // __kmpc_atomic_float16_sub_a16_rev
ATOMIC_CRITICAL_REV(float16, div_a16, Quad_a16_t, /, 16r,
1) // __kmpc_atomic_float16_div_a16_rev
#endif // KMP_ARCH_X86
#endif // KMP_HAVE_QUAD
// routines for complex types
ATOMIC_CRITICAL_REV(cmplx4, sub, kmp_cmplx32, -, 8c,
1) // __kmpc_atomic_cmplx4_sub_rev
ATOMIC_CRITICAL_REV(cmplx4, div, kmp_cmplx32, /, 8c,
1) // __kmpc_atomic_cmplx4_div_rev
ATOMIC_CRITICAL_REV(cmplx8, sub, kmp_cmplx64, -, 16c,
1) // __kmpc_atomic_cmplx8_sub_rev
ATOMIC_CRITICAL_REV(cmplx8, div, kmp_cmplx64, /, 16c,
1) // __kmpc_atomic_cmplx8_div_rev
ATOMIC_CRITICAL_REV(cmplx10, sub, kmp_cmplx80, -, 20c,
1) // __kmpc_atomic_cmplx10_sub_rev
ATOMIC_CRITICAL_REV(cmplx10, div, kmp_cmplx80, /, 20c,
1) // __kmpc_atomic_cmplx10_div_rev
#if KMP_HAVE_QUAD
ATOMIC_CRITICAL_REV(cmplx16, sub, CPLX128_LEG, -, 32c,
1) // __kmpc_atomic_cmplx16_sub_rev
ATOMIC_CRITICAL_REV(cmplx16, div, CPLX128_LEG, /, 32c,
1) // __kmpc_atomic_cmplx16_div_rev
#if (KMP_ARCH_X86)
ATOMIC_CRITICAL_REV(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1) // __kmpc_atomic_cmplx16_sub_a16_rev
ATOMIC_CRITICAL_REV(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1) // __kmpc_atomic_cmplx16_div_a16_rev
#endif // KMP_ARCH_X86
#endif // KMP_HAVE_QUAD
#endif // KMP_ARCH_X86 || KMP_ARCH_X86_64
// End of OpenMP 4.0: x = expr binop x for non-commutative operations.
/* ------------------------------------------------------------------------ */
/* Routines for mixed types of LHS and RHS, when RHS is "larger" */
/* Note: in order to reduce the total number of types combinations */
/* it is supposed that compiler converts RHS to longest floating type,*/
/* that is _Quad, before call to any of these routines */
/* Conversion to _Quad will be done by the compiler during calculation, */
/* conversion back to TYPE - before the assignment, like: */
/* *lhs = (TYPE)( (_Quad)(*lhs) OP rhs ) */
/* Performance penalty expected because of SW emulation use */
/* ------------------------------------------------------------------------ */
#define ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \
ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs) { \
KMP_DEBUG_ASSERT(__kmp_init_serial); \
KA_TRACE(100, \
("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \
gtid));
// -------------------------------------------------------------------------
#define ATOMIC_CRITICAL_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, LCK_ID, \
GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) /* send assignment */ \
OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) /* send assignment */ \
}
// -------------------------------------------------------------------------
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
// -------------------------------------------------------------------------
// X86 or X86_64: no alignment problems ====================================
#define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG(TYPE, BITS, OP) \
}
// -------------------------------------------------------------------------
#else
// ------------------------------------------------------------------------
// Code for other architectures that don't handle unaligned accesses.
#define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
} else { \
KMP_CHECK_GTID; \
OP_UPDATE_CRITICAL(TYPE, OP, \
LCK_ID) /* unaligned address - use critical */ \
} \
}
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
// -------------------------------------------------------------------------
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
// -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
RTYPE, LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_REV(TYPE, BITS, OP) \
}
#define ATOMIC_CRITICAL_REV_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
}
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
// RHS=float8
ATOMIC_CMPXCHG_MIX(fixed1, char, mul, 8, *, float8, kmp_real64, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_mul_float8
ATOMIC_CMPXCHG_MIX(fixed1, char, div, 8, /, float8, kmp_real64, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_div_float8
ATOMIC_CMPXCHG_MIX(fixed2, short, mul, 16, *, float8, kmp_real64, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_mul_float8
ATOMIC_CMPXCHG_MIX(fixed2, short, div, 16, /, float8, kmp_real64, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_div_float8
ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3,
0) // __kmpc_atomic_fixed4_mul_float8
ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3,
0) // __kmpc_atomic_fixed4_div_float8
ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_mul_float8
ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_div_float8
ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_add_float8
ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_sub_float8
ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_mul_float8
ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_div_float8
// RHS=float16 (deprecated, to be removed when we are sure the compiler does not
// use them)
#if KMP_HAVE_QUAD
ATOMIC_CMPXCHG_MIX(fixed1, char, add, 8, +, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_add_fp
ATOMIC_CMPXCHG_MIX(fixed1u, uchar, add, 8, +, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_add_fp
ATOMIC_CMPXCHG_MIX(fixed1, char, sub, 8, -, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_sub_fp
ATOMIC_CMPXCHG_MIX(fixed1u, uchar, sub, 8, -, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_sub_fp
ATOMIC_CMPXCHG_MIX(fixed1, char, mul, 8, *, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_mul_fp
ATOMIC_CMPXCHG_MIX(fixed1u, uchar, mul, 8, *, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_mul_fp
ATOMIC_CMPXCHG_MIX(fixed1, char, div, 8, /, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_div_fp
ATOMIC_CMPXCHG_MIX(fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_div_fp
ATOMIC_CMPXCHG_MIX(fixed2, short, add, 16, +, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_add_fp
ATOMIC_CMPXCHG_MIX(fixed2u, ushort, add, 16, +, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_add_fp
ATOMIC_CMPXCHG_MIX(fixed2, short, sub, 16, -, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_sub_fp
ATOMIC_CMPXCHG_MIX(fixed2u, ushort, sub, 16, -, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_sub_fp
ATOMIC_CMPXCHG_MIX(fixed2, short, mul, 16, *, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_mul_fp
ATOMIC_CMPXCHG_MIX(fixed2u, ushort, mul, 16, *, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_mul_fp
ATOMIC_CMPXCHG_MIX(fixed2, short, div, 16, /, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_div_fp
ATOMIC_CMPXCHG_MIX(fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_div_fp
ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4_add_fp
ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, add, 32, +, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4u_add_fp
ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4_sub_fp
ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, sub, 32, -, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4u_sub_fp
ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4_mul_fp
ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, mul, 32, *, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4u_mul_fp
ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4_div_fp
ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4u_div_fp
ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_add_fp
ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, add, 64, +, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_add_fp
ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_sub_fp
ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, sub, 64, -, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_sub_fp
ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_mul_fp
ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, mul, 64, *, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_mul_fp
ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_div_fp
ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_div_fp
ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_add_fp
ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_sub_fp
ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_mul_fp
ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_div_fp
ATOMIC_CMPXCHG_MIX(float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_add_fp
ATOMIC_CMPXCHG_MIX(float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_sub_fp
ATOMIC_CMPXCHG_MIX(float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_mul_fp
ATOMIC_CMPXCHG_MIX(float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_div_fp
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
ATOMIC_CRITICAL_FP(float10, long double, add, +, fp, _Quad, 10r,
1) // __kmpc_atomic_float10_add_fp
ATOMIC_CRITICAL_FP(float10, long double, sub, -, fp, _Quad, 10r,
1) // __kmpc_atomic_float10_sub_fp
ATOMIC_CRITICAL_FP(float10, long double, mul, *, fp, _Quad, 10r,
1) // __kmpc_atomic_float10_mul_fp
ATOMIC_CRITICAL_FP(float10, long double, div, /, fp, _Quad, 10r,
1) // __kmpc_atomic_float10_div_fp
// Reverse operations
ATOMIC_CMPXCHG_REV_MIX(fixed1, char, sub_rev, 8, -, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_sub_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, sub_rev, 8, -, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_sub_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed1, char, div_rev, 8, /, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_div_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, div_rev, 8, /, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_div_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed2, short, sub_rev, 16, -, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_sub_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, sub_rev, 16, -, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_sub_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed2, short, div_rev, 16, /, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_div_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, div_rev, 16, /, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_div_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, sub_rev, 32, -, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4_sub_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, sub_rev, 32, -, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4u_sub_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, div_rev, 32, /, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4_div_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, div_rev, 32, /, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4u_div_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, sub_rev, 64, -, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_sub_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, sub_rev, 64, -, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_sub_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, div_rev, 64, /, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_div_rev_fp
ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, div_rev, 64, /, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_div_rev_fp
ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, sub_rev, 32, -, fp, _Quad, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_sub_rev_fp
ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, div_rev, 32, /, fp, _Quad, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_div_rev_fp
ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, sub_rev, 64, -, fp, _Quad, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_sub_rev_fp
ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, div_rev, 64, /, fp, _Quad, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_div_rev_fp
ATOMIC_CRITICAL_REV_FP(float10, long double, sub_rev, -, fp, _Quad, 10r,
1) // __kmpc_atomic_float10_sub_rev_fp
ATOMIC_CRITICAL_REV_FP(float10, long double, div_rev, /, fp, _Quad, 10r,
1) // __kmpc_atomic_float10_div_rev_fp
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
#endif // KMP_HAVE_QUAD
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
// ------------------------------------------------------------------------
// X86 or X86_64: no alignment problems ====================================
#if USE_CMPXCHG_FIX
// workaround for C78287 (complex(kind=4) data type)
#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
}
// end of the second part of the workaround for C78287
#else
#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG(TYPE, BITS, OP) \
}
#endif // USE_CMPXCHG_FIX
#else
// ------------------------------------------------------------------------
// Code for other architectures that don't handle unaligned accesses.
#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
} else { \
KMP_CHECK_GTID; \
OP_UPDATE_CRITICAL(TYPE, OP, \
LCK_ID) /* unaligned address - use critical */ \
} \
}
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c,
7, KMP_ARCH_X86) // __kmpc_atomic_cmplx4_add_cmplx8
ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c,
7, KMP_ARCH_X86) // __kmpc_atomic_cmplx4_sub_cmplx8
ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c,
7, KMP_ARCH_X86) // __kmpc_atomic_cmplx4_mul_cmplx8
ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c,
7, KMP_ARCH_X86) // __kmpc_atomic_cmplx4_div_cmplx8
// READ, WRITE, CAPTURE
// ------------------------------------------------------------------------
// Atomic READ routines
// ------------------------------------------------------------------------
// Beginning of a definition (provides name, parameters, gebug trace)
// TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned
// fixed)
// OP_ID - operation identifier (add, sub, mul, ...)
// TYPE - operands' type
#define ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
TYPE *loc) { \
KMP_DEBUG_ASSERT(__kmp_init_serial); \
KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
// ------------------------------------------------------------------------
// Operation on *lhs, rhs using "compare_and_store_ret" routine
// TYPE - operands' type
// BITS - size in bits, used to distinguish low level calls
// OP - operator
// Note: temp_val introduced in order to force the compiler to read
// *lhs only once (w/o it the compiler reads *lhs twice)
// TODO: check if it is still necessary
// Return old value regardless of the result of "compare & swap# operation
#define OP_CMPXCHG_READ(TYPE, BITS, OP) \
{ \
TYPE KMP_ATOMIC_VOLATILE temp_val; \
union f_i_union { \
TYPE f_val; \
kmp_int##BITS i_val; \
}; \
union f_i_union old_value; \
temp_val = *loc; \
old_value.f_val = temp_val; \
old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( \
(kmp_int##BITS *)loc, \
*VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val, \
*VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val); \
new_value = old_value.f_val; \
return new_value; \
}
// -------------------------------------------------------------------------
// Operation on *lhs, rhs bound by critical section
// OP - operator (it's supposed to contain an assignment)
// LCK_ID - lock identifier
// Note: don't check gtid as it should always be valid
// 1, 2-byte - expect valid parameter, other - check before this macro
#define OP_CRITICAL_READ(OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
new_value = (*loc); \
\
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
// -------------------------------------------------------------------------
#ifdef KMP_GOMP_COMPAT
#define OP_GOMP_CRITICAL_READ(OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
OP_CRITICAL_READ(OP, 0); \
return new_value; \
}
#else
#define OP_GOMP_CRITICAL_READ(OP, FLAG)
#endif /* KMP_GOMP_COMPAT */
// -------------------------------------------------------------------------
#define ATOMIC_FIXED_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value; \
OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
new_value = KMP_TEST_THEN_ADD##BITS(loc, OP 0); \
return new_value; \
}
// -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value; \
OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
OP_CMPXCHG_READ(TYPE, BITS, OP) \
}
// ------------------------------------------------------------------------
// Routines for Extended types: long double, _Quad, complex flavours (use
// critical section)
// TYPE_ID, OP_ID, TYPE - detailed above
// OP - operator
// LCK_ID - lock identifier, used to possibly distinguish lock variable
#define ATOMIC_CRITICAL_READ(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value; \
OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) /* send assignment */ \
OP_CRITICAL_READ(OP, LCK_ID) /* send assignment */ \
return new_value; \
}
// ------------------------------------------------------------------------
// Fix for cmplx4 read (CQ220361) on Windows* OS. Regular routine with return
// value doesn't work.
// Let's return the read value through the additional parameter.
#if (KMP_OS_WINDOWS)
#define OP_CRITICAL_READ_WRK(OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
(*out) = (*loc); \
\
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
// ------------------------------------------------------------------------
#ifdef KMP_GOMP_COMPAT
#define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
OP_CRITICAL_READ_WRK(OP, 0); \
}
#else
#define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG)
#endif /* KMP_GOMP_COMPAT */
// ------------------------------------------------------------------------
#define ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \
void __kmpc_atomic_##TYPE_ID##_##OP_ID(TYPE *out, ident_t *id_ref, int gtid, \
TYPE *loc) { \
KMP_DEBUG_ASSERT(__kmp_init_serial); \
KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
// ------------------------------------------------------------------------
#define ATOMIC_CRITICAL_READ_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \
OP_GOMP_CRITICAL_READ_WRK(OP## =, GOMP_FLAG) /* send assignment */ \
OP_CRITICAL_READ_WRK(OP, LCK_ID) /* send assignment */ \
}
#endif // KMP_OS_WINDOWS
// ------------------------------------------------------------------------
// TYPE_ID,OP_ID, TYPE, OP, GOMP_FLAG
ATOMIC_FIXED_READ(fixed4, rd, kmp_int32, 32, +, 0) // __kmpc_atomic_fixed4_rd
ATOMIC_FIXED_READ(fixed8, rd, kmp_int64, 64, +,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_rd
ATOMIC_CMPXCHG_READ(float4, rd, kmp_real32, 32, +,
KMP_ARCH_X86) // __kmpc_atomic_float4_rd
ATOMIC_CMPXCHG_READ(float8, rd, kmp_real64, 64, +,
KMP_ARCH_X86) // __kmpc_atomic_float8_rd
// !!! TODO: Remove lock operations for "char" since it can't be non-atomic
ATOMIC_CMPXCHG_READ(fixed1, rd, kmp_int8, 8, +,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_rd
ATOMIC_CMPXCHG_READ(fixed2, rd, kmp_int16, 16, +,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_rd
ATOMIC_CRITICAL_READ(float10, rd, long double, +, 10r,
1) // __kmpc_atomic_float10_rd
#if KMP_HAVE_QUAD
ATOMIC_CRITICAL_READ(float16, rd, QUAD_LEGACY, +, 16r,
1) // __kmpc_atomic_float16_rd
#endif // KMP_HAVE_QUAD
// Fix for CQ220361 on Windows* OS
#if (KMP_OS_WINDOWS)
ATOMIC_CRITICAL_READ_WRK(cmplx4, rd, kmp_cmplx32, +, 8c,
1) // __kmpc_atomic_cmplx4_rd
#else
ATOMIC_CRITICAL_READ(cmplx4, rd, kmp_cmplx32, +, 8c,
1) // __kmpc_atomic_cmplx4_rd
#endif // (KMP_OS_WINDOWS)
ATOMIC_CRITICAL_READ(cmplx8, rd, kmp_cmplx64, +, 16c,
1) // __kmpc_atomic_cmplx8_rd
ATOMIC_CRITICAL_READ(cmplx10, rd, kmp_cmplx80, +, 20c,
1) // __kmpc_atomic_cmplx10_rd
#if KMP_HAVE_QUAD
ATOMIC_CRITICAL_READ(cmplx16, rd, CPLX128_LEG, +, 32c,
1) // __kmpc_atomic_cmplx16_rd
#if (KMP_ARCH_X86)
ATOMIC_CRITICAL_READ(float16, a16_rd, Quad_a16_t, +, 16r,
1) // __kmpc_atomic_float16_a16_rd
ATOMIC_CRITICAL_READ(cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c,
1) // __kmpc_atomic_cmplx16_a16_rd
#endif // (KMP_ARCH_X86)
#endif // KMP_HAVE_QUAD
// ------------------------------------------------------------------------
// Atomic WRITE routines
#define ATOMIC_XCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
KMP_XCHG_FIXED##BITS(lhs, rhs); \
}
// ------------------------------------------------------------------------
#define ATOMIC_XCHG_FLOAT_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
KMP_XCHG_REAL##BITS(lhs, rhs); \
}
// ------------------------------------------------------------------------
// Operation on *lhs, rhs using "compare_and_store" routine
// TYPE - operands' type
// BITS - size in bits, used to distinguish low level calls
// OP - operator
// Note: temp_val introduced in order to force the compiler to read
// *lhs only once (w/o it the compiler reads *lhs twice)
#define OP_CMPXCHG_WR(TYPE, BITS, OP) \
{ \
TYPE KMP_ATOMIC_VOLATILE temp_val; \
TYPE old_value, new_value; \
temp_val = *lhs; \
old_value = temp_val; \
new_value = rhs; \
while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
*VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
temp_val = *lhs; \
old_value = temp_val; \
new_value = rhs; \
} \
}
// -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
OP_CMPXCHG_WR(TYPE, BITS, OP) \
}
// ------------------------------------------------------------------------
// Routines for Extended types: long double, _Quad, complex flavours (use
// critical section)
// TYPE_ID, OP_ID, TYPE - detailed above
// OP - operator
// LCK_ID - lock identifier, used to possibly distinguish lock variable
#define ATOMIC_CRITICAL_WR(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
OP_GOMP_CRITICAL(OP, GOMP_FLAG) /* send assignment */ \
OP_CRITICAL(OP, LCK_ID) /* send assignment */ \
}
// -------------------------------------------------------------------------
ATOMIC_XCHG_WR(fixed1, wr, kmp_int8, 8, =,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_wr
ATOMIC_XCHG_WR(fixed2, wr, kmp_int16, 16, =,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_wr
ATOMIC_XCHG_WR(fixed4, wr, kmp_int32, 32, =,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_wr
#if (KMP_ARCH_X86)
ATOMIC_CMPXCHG_WR(fixed8, wr, kmp_int64, 64, =,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_wr
#else
ATOMIC_XCHG_WR(fixed8, wr, kmp_int64, 64, =,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_wr
#endif // (KMP_ARCH_X86)
ATOMIC_XCHG_FLOAT_WR(float4, wr, kmp_real32, 32, =,
KMP_ARCH_X86) // __kmpc_atomic_float4_wr
#if (KMP_ARCH_X86)
ATOMIC_CMPXCHG_WR(float8, wr, kmp_real64, 64, =,
KMP_ARCH_X86) // __kmpc_atomic_float8_wr
#else
ATOMIC_XCHG_FLOAT_WR(float8, wr, kmp_real64, 64, =,
KMP_ARCH_X86) // __kmpc_atomic_float8_wr
#endif // (KMP_ARCH_X86)
ATOMIC_CRITICAL_WR(float10, wr, long double, =, 10r,
1) // __kmpc_atomic_float10_wr
#if KMP_HAVE_QUAD
ATOMIC_CRITICAL_WR(float16, wr, QUAD_LEGACY, =, 16r,
1) // __kmpc_atomic_float16_wr
#endif // KMP_HAVE_QUAD
ATOMIC_CRITICAL_WR(cmplx4, wr, kmp_cmplx32, =, 8c, 1) // __kmpc_atomic_cmplx4_wr
ATOMIC_CRITICAL_WR(cmplx8, wr, kmp_cmplx64, =, 16c,
1) // __kmpc_atomic_cmplx8_wr
ATOMIC_CRITICAL_WR(cmplx10, wr, kmp_cmplx80, =, 20c,
1) // __kmpc_atomic_cmplx10_wr
#if KMP_HAVE_QUAD
ATOMIC_CRITICAL_WR(cmplx16, wr, CPLX128_LEG, =, 32c,
1) // __kmpc_atomic_cmplx16_wr
#if (KMP_ARCH_X86)
ATOMIC_CRITICAL_WR(float16, a16_wr, Quad_a16_t, =, 16r,
1) // __kmpc_atomic_float16_a16_wr
ATOMIC_CRITICAL_WR(cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c,
1) // __kmpc_atomic_cmplx16_a16_wr
#endif // (KMP_ARCH_X86)
#endif // KMP_HAVE_QUAD
// ------------------------------------------------------------------------
// Atomic CAPTURE routines
// Beginning of a definition (provides name, parameters, gebug trace)
// TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned
// fixed)
// OP_ID - operation identifier (add, sub, mul, ...)
// TYPE - operands' type
#define ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
TYPE *lhs, TYPE rhs, int flag) { \
KMP_DEBUG_ASSERT(__kmp_init_serial); \
KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
// -------------------------------------------------------------------------
// Operation on *lhs, rhs bound by critical section
// OP - operator (it's supposed to contain an assignment)
// LCK_ID - lock identifier
// Note: don't check gtid as it should always be valid
// 1, 2-byte - expect valid parameter, other - check before this macro
#define OP_CRITICAL_CPT(OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
if (flag) { \
(*lhs) OP rhs; \
new_value = (*lhs); \
} else { \
new_value = (*lhs); \
(*lhs) OP rhs; \
} \
\
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
return new_value;
#define OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
if (flag) { \
(*lhs) = (TYPE)((*lhs)OP rhs); \
new_value = (*lhs); \
} else { \
new_value = (*lhs); \
(*lhs) = (TYPE)((*lhs)OP rhs); \
} \
\
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
return new_value;
// ------------------------------------------------------------------------
#ifdef KMP_GOMP_COMPAT
#define OP_GOMP_CRITICAL_CPT(TYPE, OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
OP_UPDATE_CRITICAL_CPT(TYPE, OP, 0); \
}
#else
#define OP_GOMP_CRITICAL_CPT(TYPE, OP, FLAG)
#endif /* KMP_GOMP_COMPAT */
// ------------------------------------------------------------------------
// Operation on *lhs, rhs using "compare_and_store" routine
// TYPE - operands' type
// BITS - size in bits, used to distinguish low level calls
// OP - operator
// Note: temp_val introduced in order to force the compiler to read
// *lhs only once (w/o it the compiler reads *lhs twice)
#define OP_CMPXCHG_CPT(TYPE, BITS, OP) \
{ \
TYPE KMP_ATOMIC_VOLATILE temp_val; \
TYPE old_value, new_value; \
temp_val = *lhs; \
old_value = temp_val; \
new_value = (TYPE)(old_value OP rhs); \
while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
*VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
temp_val = *lhs; \
old_value = temp_val; \
new_value = (TYPE)(old_value OP rhs); \
} \
if (flag) { \
return new_value; \
} else \
return old_value; \
}
// -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value; \
(void)new_value; \
OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_CPT(TYPE, BITS, OP) \
}
// -------------------------------------------------------------------------
#define ATOMIC_FIXED_ADD_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE old_value, new_value; \
(void)new_value; \
OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
/* OP used as a sign for subtraction: (lhs-rhs) --> (lhs+-rhs) */ \
old_value = KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
if (flag) { \
return old_value OP rhs; \
} else \
return old_value; \
}
// -------------------------------------------------------------------------
ATOMIC_FIXED_ADD_CPT(fixed4, add_cpt, kmp_int32, 32, +,
0) // __kmpc_atomic_fixed4_add_cpt
ATOMIC_FIXED_ADD_CPT(fixed4, sub_cpt, kmp_int32, 32, -,
0) // __kmpc_atomic_fixed4_sub_cpt
ATOMIC_FIXED_ADD_CPT(fixed8, add_cpt, kmp_int64, 64, +,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_add_cpt
ATOMIC_FIXED_ADD_CPT(fixed8, sub_cpt, kmp_int64, 64, -,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_sub_cpt
ATOMIC_CMPXCHG_CPT(float4, add_cpt, kmp_real32, 32, +,
KMP_ARCH_X86) // __kmpc_atomic_float4_add_cpt
ATOMIC_CMPXCHG_CPT(float4, sub_cpt, kmp_real32, 32, -,
KMP_ARCH_X86) // __kmpc_atomic_float4_sub_cpt
ATOMIC_CMPXCHG_CPT(float8, add_cpt, kmp_real64, 64, +,
KMP_ARCH_X86) // __kmpc_atomic_float8_add_cpt
ATOMIC_CMPXCHG_CPT(float8, sub_cpt, kmp_real64, 64, -,
KMP_ARCH_X86) // __kmpc_atomic_float8_sub_cpt
// ------------------------------------------------------------------------
// Entries definition for integer operands
// TYPE_ID - operands type and size (fixed4, float4)
// OP_ID - operation identifier (add, sub, mul, ...)
// TYPE - operand type
// BITS - size in bits, used to distinguish low level calls
// OP - operator (used in critical section)
// TYPE_ID,OP_ID, TYPE, BITS,OP,GOMP_FLAG
// ------------------------------------------------------------------------
// Routines for ATOMIC integer operands, other operators
// ------------------------------------------------------------------------
// TYPE_ID,OP_ID, TYPE, OP, GOMP_FLAG
ATOMIC_CMPXCHG_CPT(fixed1, add_cpt, kmp_int8, 8, +,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_add_cpt
ATOMIC_CMPXCHG_CPT(fixed1, andb_cpt, kmp_int8, 8, &,
0) // __kmpc_atomic_fixed1_andb_cpt
ATOMIC_CMPXCHG_CPT(fixed1, div_cpt, kmp_int8, 8, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_div_cpt
ATOMIC_CMPXCHG_CPT(fixed1u, div_cpt, kmp_uint8, 8, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_div_cpt
ATOMIC_CMPXCHG_CPT(fixed1, mul_cpt, kmp_int8, 8, *,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_mul_cpt
ATOMIC_CMPXCHG_CPT(fixed1, orb_cpt, kmp_int8, 8, |,
0) // __kmpc_atomic_fixed1_orb_cpt
ATOMIC_CMPXCHG_CPT(fixed1, shl_cpt, kmp_int8, 8, <<,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_shl_cpt
ATOMIC_CMPXCHG_CPT(fixed1, shr_cpt, kmp_int8, 8, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_shr_cpt
ATOMIC_CMPXCHG_CPT(fixed1u, shr_cpt, kmp_uint8, 8, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_shr_cpt
ATOMIC_CMPXCHG_CPT(fixed1, sub_cpt, kmp_int8, 8, -,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_sub_cpt
ATOMIC_CMPXCHG_CPT(fixed1, xor_cpt, kmp_int8, 8, ^,
0) // __kmpc_atomic_fixed1_xor_cpt
ATOMIC_CMPXCHG_CPT(fixed2, add_cpt, kmp_int16, 16, +,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_add_cpt
ATOMIC_CMPXCHG_CPT(fixed2, andb_cpt, kmp_int16, 16, &,
0) // __kmpc_atomic_fixed2_andb_cpt
ATOMIC_CMPXCHG_CPT(fixed2, div_cpt, kmp_int16, 16, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_div_cpt
ATOMIC_CMPXCHG_CPT(fixed2u, div_cpt, kmp_uint16, 16, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_div_cpt
ATOMIC_CMPXCHG_CPT(fixed2, mul_cpt, kmp_int16, 16, *,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_mul_cpt
ATOMIC_CMPXCHG_CPT(fixed2, orb_cpt, kmp_int16, 16, |,
0) // __kmpc_atomic_fixed2_orb_cpt
ATOMIC_CMPXCHG_CPT(fixed2, shl_cpt, kmp_int16, 16, <<,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_shl_cpt
ATOMIC_CMPXCHG_CPT(fixed2, shr_cpt, kmp_int16, 16, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_shr_cpt
ATOMIC_CMPXCHG_CPT(fixed2u, shr_cpt, kmp_uint16, 16, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_shr_cpt
ATOMIC_CMPXCHG_CPT(fixed2, sub_cpt, kmp_int16, 16, -,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_sub_cpt
ATOMIC_CMPXCHG_CPT(fixed2, xor_cpt, kmp_int16, 16, ^,
0) // __kmpc_atomic_fixed2_xor_cpt
ATOMIC_CMPXCHG_CPT(fixed4, andb_cpt, kmp_int32, 32, &,
0) // __kmpc_atomic_fixed4_andb_cpt
ATOMIC_CMPXCHG_CPT(fixed4, div_cpt, kmp_int32, 32, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_div_cpt
ATOMIC_CMPXCHG_CPT(fixed4u, div_cpt, kmp_uint32, 32, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed4u_div_cpt
ATOMIC_CMPXCHG_CPT(fixed4, mul_cpt, kmp_int32, 32, *,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_mul_cpt
ATOMIC_CMPXCHG_CPT(fixed4, orb_cpt, kmp_int32, 32, |,
0) // __kmpc_atomic_fixed4_orb_cpt
ATOMIC_CMPXCHG_CPT(fixed4, shl_cpt, kmp_int32, 32, <<,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_shl_cpt
ATOMIC_CMPXCHG_CPT(fixed4, shr_cpt, kmp_int32, 32, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_shr_cpt
ATOMIC_CMPXCHG_CPT(fixed4u, shr_cpt, kmp_uint32, 32, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed4u_shr_cpt
ATOMIC_CMPXCHG_CPT(fixed4, xor_cpt, kmp_int32, 32, ^,
0) // __kmpc_atomic_fixed4_xor_cpt
ATOMIC_CMPXCHG_CPT(fixed8, andb_cpt, kmp_int64, 64, &,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_andb_cpt
ATOMIC_CMPXCHG_CPT(fixed8, div_cpt, kmp_int64, 64, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_div_cpt
ATOMIC_CMPXCHG_CPT(fixed8u, div_cpt, kmp_uint64, 64, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_div_cpt
ATOMIC_CMPXCHG_CPT(fixed8, mul_cpt, kmp_int64, 64, *,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_mul_cpt
ATOMIC_CMPXCHG_CPT(fixed8, orb_cpt, kmp_int64, 64, |,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_orb_cpt
ATOMIC_CMPXCHG_CPT(fixed8, shl_cpt, kmp_int64, 64, <<,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_shl_cpt
ATOMIC_CMPXCHG_CPT(fixed8, shr_cpt, kmp_int64, 64, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_shr_cpt
ATOMIC_CMPXCHG_CPT(fixed8u, shr_cpt, kmp_uint64, 64, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_shr_cpt
ATOMIC_CMPXCHG_CPT(fixed8, xor_cpt, kmp_int64, 64, ^,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_xor_cpt
ATOMIC_CMPXCHG_CPT(float4, div_cpt, kmp_real32, 32, /,
KMP_ARCH_X86) // __kmpc_atomic_float4_div_cpt
ATOMIC_CMPXCHG_CPT(float4, mul_cpt, kmp_real32, 32, *,
KMP_ARCH_X86) // __kmpc_atomic_float4_mul_cpt
ATOMIC_CMPXCHG_CPT(float8, div_cpt, kmp_real64, 64, /,
KMP_ARCH_X86) // __kmpc_atomic_float8_div_cpt
ATOMIC_CMPXCHG_CPT(float8, mul_cpt, kmp_real64, 64, *,
KMP_ARCH_X86) // __kmpc_atomic_float8_mul_cpt
// TYPE_ID,OP_ID, TYPE, OP, GOMP_FLAG
// CAPTURE routines for mixed types RHS=float16
#if KMP_HAVE_QUAD
// Beginning of a definition (provides name, parameters, gebug trace)
// TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned
// fixed)
// OP_ID - operation identifier (add, sub, mul, ...)
// TYPE - operands' type
#define ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \
ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs, int flag) { \
KMP_DEBUG_ASSERT(__kmp_init_serial); \
KA_TRACE(100, \
("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \
gtid));
// -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
RTYPE, LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
TYPE new_value; \
(void)new_value; \
OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_CPT(TYPE, BITS, OP) \
}
// -------------------------------------------------------------------------
#define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
TYPE new_value; \
(void)new_value; \
OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) /* send assignment */ \
OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) /* send assignment */ \
}
ATOMIC_CMPXCHG_CPT_MIX(fixed1, char, add_cpt, 8, +, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_add_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, add_cpt, 8, +, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_add_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed1, char, sub_cpt, 8, -, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_sub_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, sub_cpt, 8, -, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_sub_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed1, char, mul_cpt, 8, *, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_mul_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, mul_cpt, 8, *, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_mul_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed1, char, div_cpt, 8, /, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_div_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, div_cpt, 8, /, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_div_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed2, short, add_cpt, 16, +, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_add_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, add_cpt, 16, +, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_add_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed2, short, sub_cpt, 16, -, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_sub_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, sub_cpt, 16, -, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_sub_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed2, short, mul_cpt, 16, *, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_mul_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, mul_cpt, 16, *, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_mul_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed2, short, div_cpt, 16, /, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_div_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, div_cpt, 16, /, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_div_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, add_cpt, 32, +, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4_add_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, add_cpt, 32, +, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4u_add_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4_sub_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4u_sub_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4_mul_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4u_mul_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, div_cpt, 32, /, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4_div_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, div_cpt, 32, /, fp, _Quad, 4i, 3,
0) // __kmpc_atomic_fixed4u_div_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, add_cpt, 64, +, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_add_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, add_cpt, 64, +, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_add_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_sub_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_sub_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_mul_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_mul_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, div_cpt, 64, /, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_div_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, div_cpt, 64, /, fp, _Quad, 8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_div_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, add_cpt, 32, +, fp, _Quad, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_add_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, sub_cpt, 32, -, fp, _Quad, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_sub_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, mul_cpt, 32, *, fp, _Quad, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_mul_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, div_cpt, 32, /, fp, _Quad, 4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_div_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, add_cpt, 64, +, fp, _Quad, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_add_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, sub_cpt, 64, -, fp, _Quad, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_sub_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, mul_cpt, 64, *, fp, _Quad, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_mul_cpt_fp
ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, div_cpt, 64, /, fp, _Quad, 8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_div_cpt_fp
ATOMIC_CRITICAL_CPT_MIX(float10, long double, add_cpt, +, fp, _Quad, 10r,
1) // __kmpc_atomic_float10_add_cpt_fp
ATOMIC_CRITICAL_CPT_MIX(float10, long double, sub_cpt, -, fp, _Quad, 10r,
1) // __kmpc_atomic_float10_sub_cpt_fp
ATOMIC_CRITICAL_CPT_MIX(float10, long double, mul_cpt, *, fp, _Quad, 10r,
1) // __kmpc_atomic_float10_mul_cpt_fp
ATOMIC_CRITICAL_CPT_MIX(float10, long double, div_cpt, /, fp, _Quad, 10r,
1) // __kmpc_atomic_float10_div_cpt_fp
#endif // KMP_HAVE_QUAD
// ------------------------------------------------------------------------
// Routines for C/C++ Reduction operators && and ||
// -------------------------------------------------------------------------
// Operation on *lhs, rhs bound by critical section
// OP - operator (it's supposed to contain an assignment)
// LCK_ID - lock identifier
// Note: don't check gtid as it should always be valid
// 1, 2-byte - expect valid parameter, other - check before this macro
#define OP_CRITICAL_L_CPT(OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
if (flag) { \
new_value OP rhs; \
(*lhs) = new_value; \
} else { \
new_value = (*lhs); \
(*lhs) OP rhs; \
} \
\
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
// ------------------------------------------------------------------------
#ifdef KMP_GOMP_COMPAT
#define OP_GOMP_CRITICAL_L_CPT(OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
OP_CRITICAL_L_CPT(OP, 0); \
return new_value; \
}
#else
#define OP_GOMP_CRITICAL_L_CPT(OP, FLAG)
#endif /* KMP_GOMP_COMPAT */
// ------------------------------------------------------------------------
// Need separate macros for &&, || because there is no combined assignment
#define ATOMIC_CMPX_L_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value; \
(void)new_value; \
OP_GOMP_CRITICAL_L_CPT(= *lhs OP, GOMP_FLAG) \
OP_CMPXCHG_CPT(TYPE, BITS, OP) \
}
ATOMIC_CMPX_L_CPT(fixed1, andl_cpt, char, 8, &&,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_andl_cpt
ATOMIC_CMPX_L_CPT(fixed1, orl_cpt, char, 8, ||,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_orl_cpt
ATOMIC_CMPX_L_CPT(fixed2, andl_cpt, short, 16, &&,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_andl_cpt
ATOMIC_CMPX_L_CPT(fixed2, orl_cpt, short, 16, ||,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_orl_cpt
ATOMIC_CMPX_L_CPT(fixed4, andl_cpt, kmp_int32, 32, &&,
0) // __kmpc_atomic_fixed4_andl_cpt
ATOMIC_CMPX_L_CPT(fixed4, orl_cpt, kmp_int32, 32, ||,
0) // __kmpc_atomic_fixed4_orl_cpt
ATOMIC_CMPX_L_CPT(fixed8, andl_cpt, kmp_int64, 64, &&,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_andl_cpt
ATOMIC_CMPX_L_CPT(fixed8, orl_cpt, kmp_int64, 64, ||,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_orl_cpt
// -------------------------------------------------------------------------
// Routines for Fortran operators that matched no one in C:
// MAX, MIN, .EQV., .NEQV.
// Operators .AND., .OR. are covered by __kmpc_atomic_*_{andl,orl}_cpt
// Intrinsics IAND, IOR, IEOR are covered by __kmpc_atomic_*_{andb,orb,xor}_cpt
// -------------------------------------------------------------------------
// MIN and MAX need separate macros
// OP - operator to check if we need any actions?
#define MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
if (*lhs OP rhs) { /* still need actions? */ \
old_value = *lhs; \
*lhs = rhs; \
if (flag) \
new_value = rhs; \
else \
new_value = old_value; \
} else { \
new_value = *lhs; \
} \
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
return new_value;
// -------------------------------------------------------------------------
#ifdef KMP_GOMP_COMPAT
#define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
MIN_MAX_CRITSECT_CPT(OP, 0); \
}
#else
#define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG)
#endif /* KMP_GOMP_COMPAT */
// -------------------------------------------------------------------------
#define MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \
{ \
TYPE KMP_ATOMIC_VOLATILE temp_val; \
/*TYPE old_value; */ \
temp_val = *lhs; \
old_value = temp_val; \
while (old_value OP rhs && /* still need actions? */ \
!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, \
*VOLATILE_CAST(kmp_int##BITS *) & old_value, \
*VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \
temp_val = *lhs; \
old_value = temp_val; \
} \
if (flag) \
return rhs; \
else \
return old_value; \
}
// -------------------------------------------------------------------------
// 1-byte, 2-byte operands - use critical section
#define MIN_MAX_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value, old_value; \
if (*lhs OP rhs) { /* need actions? */ \
GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \
MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \
} \
return *lhs; \
}
#define MIN_MAX_COMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value, old_value; \
(void)new_value; \
if (*lhs OP rhs) { \
GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \
MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \
} \
return *lhs; \
}
MIN_MAX_COMPXCHG_CPT(fixed1, max_cpt, char, 8, <,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_max_cpt
MIN_MAX_COMPXCHG_CPT(fixed1, min_cpt, char, 8, >,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_min_cpt
MIN_MAX_COMPXCHG_CPT(fixed2, max_cpt, short, 16, <,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_max_cpt
MIN_MAX_COMPXCHG_CPT(fixed2, min_cpt, short, 16, >,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_min_cpt
MIN_MAX_COMPXCHG_CPT(fixed4, max_cpt, kmp_int32, 32, <,
0) // __kmpc_atomic_fixed4_max_cpt
MIN_MAX_COMPXCHG_CPT(fixed4, min_cpt, kmp_int32, 32, >,
0) // __kmpc_atomic_fixed4_min_cpt
MIN_MAX_COMPXCHG_CPT(fixed8, max_cpt, kmp_int64, 64, <,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_max_cpt
MIN_MAX_COMPXCHG_CPT(fixed8, min_cpt, kmp_int64, 64, >,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_min_cpt
MIN_MAX_COMPXCHG_CPT(float4, max_cpt, kmp_real32, 32, <,
KMP_ARCH_X86) // __kmpc_atomic_float4_max_cpt
MIN_MAX_COMPXCHG_CPT(float4, min_cpt, kmp_real32, 32, >,
KMP_ARCH_X86) // __kmpc_atomic_float4_min_cpt
MIN_MAX_COMPXCHG_CPT(float8, max_cpt, kmp_real64, 64, <,
KMP_ARCH_X86) // __kmpc_atomic_float8_max_cpt
MIN_MAX_COMPXCHG_CPT(float8, min_cpt, kmp_real64, 64, >,
KMP_ARCH_X86) // __kmpc_atomic_float8_min_cpt
MIN_MAX_CRITICAL_CPT(float10, max_cpt, long double, <, 10r,
1) // __kmpc_atomic_float10_max_cpt
MIN_MAX_CRITICAL_CPT(float10, min_cpt, long double, >, 10r,
1) // __kmpc_atomic_float10_min_cpt
#if KMP_HAVE_QUAD
MIN_MAX_CRITICAL_CPT(float16, max_cpt, QUAD_LEGACY, <, 16r,
1) // __kmpc_atomic_float16_max_cpt
MIN_MAX_CRITICAL_CPT(float16, min_cpt, QUAD_LEGACY, >, 16r,
1) // __kmpc_atomic_float16_min_cpt
#if (KMP_ARCH_X86)
MIN_MAX_CRITICAL_CPT(float16, max_a16_cpt, Quad_a16_t, <, 16r,
1) // __kmpc_atomic_float16_max_a16_cpt
MIN_MAX_CRITICAL_CPT(float16, min_a16_cpt, Quad_a16_t, >, 16r,
1) // __kmpc_atomic_float16_mix_a16_cpt
#endif // (KMP_ARCH_X86)
#endif // KMP_HAVE_QUAD
// ------------------------------------------------------------------------
#ifdef KMP_GOMP_COMPAT
#define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
OP_CRITICAL_CPT(OP, 0); \
}
#else
#define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG)
#endif /* KMP_GOMP_COMPAT */
// ------------------------------------------------------------------------
#define ATOMIC_CMPX_EQV_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value; \
(void)new_value; \
OP_GOMP_CRITICAL_EQV_CPT(^= (TYPE) ~, GOMP_FLAG) /* send assignment */ \
OP_CMPXCHG_CPT(TYPE, BITS, OP) \
}
// ------------------------------------------------------------------------
ATOMIC_CMPXCHG_CPT(fixed1, neqv_cpt, kmp_int8, 8, ^,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_neqv_cpt
ATOMIC_CMPXCHG_CPT(fixed2, neqv_cpt, kmp_int16, 16, ^,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_neqv_cpt
ATOMIC_CMPXCHG_CPT(fixed4, neqv_cpt, kmp_int32, 32, ^,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_neqv_cpt
ATOMIC_CMPXCHG_CPT(fixed8, neqv_cpt, kmp_int64, 64, ^,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_neqv_cpt
ATOMIC_CMPX_EQV_CPT(fixed1, eqv_cpt, kmp_int8, 8, ^~,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_eqv_cpt
ATOMIC_CMPX_EQV_CPT(fixed2, eqv_cpt, kmp_int16, 16, ^~,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_eqv_cpt
ATOMIC_CMPX_EQV_CPT(fixed4, eqv_cpt, kmp_int32, 32, ^~,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_eqv_cpt
ATOMIC_CMPX_EQV_CPT(fixed8, eqv_cpt, kmp_int64, 64, ^~,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_eqv_cpt
// ------------------------------------------------------------------------
// Routines for Extended types: long double, _Quad, complex flavours (use
// critical section)
// TYPE_ID, OP_ID, TYPE - detailed above
// OP - operator
// LCK_ID - lock identifier, used to possibly distinguish lock variable
#define ATOMIC_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value; \
OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) /* send assignment */ \
OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) /* send assignment */ \
}
// ------------------------------------------------------------------------
// Workaround for cmplx4. Regular routines with return value don't work
// on Win_32e. Let's return captured values through the additional parameter.
#define OP_CRITICAL_CPT_WRK(OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
if (flag) { \
(*lhs) OP rhs; \
(*out) = (*lhs); \
} else { \
(*out) = (*lhs); \
(*lhs) OP rhs; \
} \
\
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
return;
// ------------------------------------------------------------------------
#ifdef KMP_GOMP_COMPAT
#define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
OP_CRITICAL_CPT_WRK(OP## =, 0); \
}
#else
#define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG)
#endif /* KMP_GOMP_COMPAT */
// ------------------------------------------------------------------------
#define ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
void __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, TYPE *lhs, \
TYPE rhs, TYPE *out, int flag) { \
KMP_DEBUG_ASSERT(__kmp_init_serial); \
KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
// ------------------------------------------------------------------------
#define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
OP_GOMP_CRITICAL_CPT_WRK(OP, GOMP_FLAG) \
OP_CRITICAL_CPT_WRK(OP## =, LCK_ID) \
}
// The end of workaround for cmplx4
/* ------------------------------------------------------------------------- */
// routines for long double type
ATOMIC_CRITICAL_CPT(float10, add_cpt, long double, +, 10r,
1) // __kmpc_atomic_float10_add_cpt
ATOMIC_CRITICAL_CPT(float10, sub_cpt, long double, -, 10r,
1) // __kmpc_atomic_float10_sub_cpt
ATOMIC_CRITICAL_CPT(float10, mul_cpt, long double, *, 10r,
1) // __kmpc_atomic_float10_mul_cpt
ATOMIC_CRITICAL_CPT(float10, div_cpt, long double, /, 10r,
1) // __kmpc_atomic_float10_div_cpt
#if KMP_HAVE_QUAD
// routines for _Quad type
ATOMIC_CRITICAL_CPT(float16, add_cpt, QUAD_LEGACY, +, 16r,
1) // __kmpc_atomic_float16_add_cpt
ATOMIC_CRITICAL_CPT(float16, sub_cpt, QUAD_LEGACY, -, 16r,
1) // __kmpc_atomic_float16_sub_cpt
ATOMIC_CRITICAL_CPT(float16, mul_cpt, QUAD_LEGACY, *, 16r,
1) // __kmpc_atomic_float16_mul_cpt
ATOMIC_CRITICAL_CPT(float16, div_cpt, QUAD_LEGACY, /, 16r,
1) // __kmpc_atomic_float16_div_cpt
#if (KMP_ARCH_X86)
ATOMIC_CRITICAL_CPT(float16, add_a16_cpt, Quad_a16_t, +, 16r,
1) // __kmpc_atomic_float16_add_a16_cpt
ATOMIC_CRITICAL_CPT(float16, sub_a16_cpt, Quad_a16_t, -, 16r,
1) // __kmpc_atomic_float16_sub_a16_cpt
ATOMIC_CRITICAL_CPT(float16, mul_a16_cpt, Quad_a16_t, *, 16r,
1) // __kmpc_atomic_float16_mul_a16_cpt
ATOMIC_CRITICAL_CPT(float16, div_a16_cpt, Quad_a16_t, /, 16r,
1) // __kmpc_atomic_float16_div_a16_cpt
#endif // (KMP_ARCH_X86)
#endif // KMP_HAVE_QUAD
// routines for complex types
// cmplx4 routines to return void
ATOMIC_CRITICAL_CPT_WRK(cmplx4, add_cpt, kmp_cmplx32, +, 8c,
1) // __kmpc_atomic_cmplx4_add_cpt
ATOMIC_CRITICAL_CPT_WRK(cmplx4, sub_cpt, kmp_cmplx32, -, 8c,
1) // __kmpc_atomic_cmplx4_sub_cpt
ATOMIC_CRITICAL_CPT_WRK(cmplx4, mul_cpt, kmp_cmplx32, *, 8c,
1) // __kmpc_atomic_cmplx4_mul_cpt
ATOMIC_CRITICAL_CPT_WRK(cmplx4, div_cpt, kmp_cmplx32, /, 8c,
1) // __kmpc_atomic_cmplx4_div_cpt
ATOMIC_CRITICAL_CPT(cmplx8, add_cpt, kmp_cmplx64, +, 16c,
1) // __kmpc_atomic_cmplx8_add_cpt
ATOMIC_CRITICAL_CPT(cmplx8, sub_cpt, kmp_cmplx64, -, 16c,
1) // __kmpc_atomic_cmplx8_sub_cpt
ATOMIC_CRITICAL_CPT(cmplx8, mul_cpt, kmp_cmplx64, *, 16c,
1) // __kmpc_atomic_cmplx8_mul_cpt
ATOMIC_CRITICAL_CPT(cmplx8, div_cpt, kmp_cmplx64, /, 16c,
1) // __kmpc_atomic_cmplx8_div_cpt
ATOMIC_CRITICAL_CPT(cmplx10, add_cpt, kmp_cmplx80, +, 20c,
1) // __kmpc_atomic_cmplx10_add_cpt
ATOMIC_CRITICAL_CPT(cmplx10, sub_cpt, kmp_cmplx80, -, 20c,
1) // __kmpc_atomic_cmplx10_sub_cpt
ATOMIC_CRITICAL_CPT(cmplx10, mul_cpt, kmp_cmplx80, *, 20c,
1) // __kmpc_atomic_cmplx10_mul_cpt
ATOMIC_CRITICAL_CPT(cmplx10, div_cpt, kmp_cmplx80, /, 20c,
1) // __kmpc_atomic_cmplx10_div_cpt
#if KMP_HAVE_QUAD
ATOMIC_CRITICAL_CPT(cmplx16, add_cpt, CPLX128_LEG, +, 32c,
1) // __kmpc_atomic_cmplx16_add_cpt
ATOMIC_CRITICAL_CPT(cmplx16, sub_cpt, CPLX128_LEG, -, 32c,
1) // __kmpc_atomic_cmplx16_sub_cpt
ATOMIC_CRITICAL_CPT(cmplx16, mul_cpt, CPLX128_LEG, *, 32c,
1) // __kmpc_atomic_cmplx16_mul_cpt
ATOMIC_CRITICAL_CPT(cmplx16, div_cpt, CPLX128_LEG, /, 32c,
1) // __kmpc_atomic_cmplx16_div_cpt
#if (KMP_ARCH_X86)
ATOMIC_CRITICAL_CPT(cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c,
1) // __kmpc_atomic_cmplx16_add_a16_cpt
ATOMIC_CRITICAL_CPT(cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c,
1) // __kmpc_atomic_cmplx16_sub_a16_cpt
ATOMIC_CRITICAL_CPT(cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c,
1) // __kmpc_atomic_cmplx16_mul_a16_cpt
ATOMIC_CRITICAL_CPT(cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c,
1) // __kmpc_atomic_cmplx16_div_a16_cpt
#endif // (KMP_ARCH_X86)
#endif // KMP_HAVE_QUAD
// OpenMP 4.0: v = x = expr binop x; { v = x; x = expr binop x; } { x = expr
// binop x; v = x; } for non-commutative operations.
// Supported only on IA-32 architecture and Intel(R) 64
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
// -------------------------------------------------------------------------
// Operation on *lhs, rhs bound by critical section
// OP - operator (it's supposed to contain an assignment)
// LCK_ID - lock identifier
// Note: don't check gtid as it should always be valid
// 1, 2-byte - expect valid parameter, other - check before this macro
#define OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
if (flag) { \
/*temp_val = (*lhs);*/ \
(*lhs) = (TYPE)((rhs)OP(*lhs)); \
new_value = (*lhs); \
} else { \
new_value = (*lhs); \
(*lhs) = (TYPE)((rhs)OP(*lhs)); \
} \
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
return new_value;
// ------------------------------------------------------------------------
#ifdef KMP_GOMP_COMPAT
#define OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
OP_CRITICAL_CPT_REV(TYPE, OP, 0); \
}
#else
#define OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, FLAG)
#endif /* KMP_GOMP_COMPAT */
// ------------------------------------------------------------------------
// Operation on *lhs, rhs using "compare_and_store" routine
// TYPE - operands' type
// BITS - size in bits, used to distinguish low level calls
// OP - operator
// Note: temp_val introduced in order to force the compiler to read
// *lhs only once (w/o it the compiler reads *lhs twice)
#define OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
{ \
TYPE KMP_ATOMIC_VOLATILE temp_val; \
TYPE old_value, new_value; \
temp_val = *lhs; \
old_value = temp_val; \
new_value = (TYPE)(rhs OP old_value); \
while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
*VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
temp_val = *lhs; \
old_value = temp_val; \
new_value = (TYPE)(rhs OP old_value); \
} \
if (flag) { \
return new_value; \
} else \
return old_value; \
}
// -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value; \
(void)new_value; \
OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
}
ATOMIC_CMPXCHG_CPT_REV(fixed1, div_cpt_rev, kmp_int8, 8, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_div_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed1u, div_cpt_rev, kmp_uint8, 8, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_div_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed1, shl_cpt_rev, kmp_int8, 8, <<,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_shl_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed1, shr_cpt_rev, kmp_int8, 8, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_shr_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed1u, shr_cpt_rev, kmp_uint8, 8, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_shr_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed1, sub_cpt_rev, kmp_int8, 8, -,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_sub_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed2, div_cpt_rev, kmp_int16, 16, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_div_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed2u, div_cpt_rev, kmp_uint16, 16, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_div_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed2, shl_cpt_rev, kmp_int16, 16, <<,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_shl_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed2, shr_cpt_rev, kmp_int16, 16, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_shr_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed2u, shr_cpt_rev, kmp_uint16, 16, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_shr_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed2, sub_cpt_rev, kmp_int16, 16, -,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_sub_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed4, div_cpt_rev, kmp_int32, 32, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_div_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed4u, div_cpt_rev, kmp_uint32, 32, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed4u_div_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed4, shl_cpt_rev, kmp_int32, 32, <<,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_shl_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed4, shr_cpt_rev, kmp_int32, 32, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_shr_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed4u, shr_cpt_rev, kmp_uint32, 32, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed4u_shr_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed4, sub_cpt_rev, kmp_int32, 32, -,
KMP_ARCH_X86) // __kmpc_atomic_fixed4_sub_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed8, div_cpt_rev, kmp_int64, 64, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_div_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed8u, div_cpt_rev, kmp_uint64, 64, /,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_div_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed8, shl_cpt_rev, kmp_int64, 64, <<,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_shl_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed8, shr_cpt_rev, kmp_int64, 64, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_shr_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed8u, shr_cpt_rev, kmp_uint64, 64, >>,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_shr_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(fixed8, sub_cpt_rev, kmp_int64, 64, -,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_sub_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(float4, div_cpt_rev, kmp_real32, 32, /,
KMP_ARCH_X86) // __kmpc_atomic_float4_div_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(float4, sub_cpt_rev, kmp_real32, 32, -,
KMP_ARCH_X86) // __kmpc_atomic_float4_sub_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(float8, div_cpt_rev, kmp_real64, 64, /,
KMP_ARCH_X86) // __kmpc_atomic_float8_div_cpt_rev
ATOMIC_CMPXCHG_CPT_REV(float8, sub_cpt_rev, kmp_real64, 64, -,
KMP_ARCH_X86) // __kmpc_atomic_float8_sub_cpt_rev
// TYPE_ID,OP_ID, TYPE, OP, GOMP_FLAG
// ------------------------------------------------------------------------
// Routines for Extended types: long double, _Quad, complex flavours (use
// critical section)
// TYPE_ID, OP_ID, TYPE - detailed above
// OP - operator
// LCK_ID - lock identifier, used to possibly distinguish lock variable
#define ATOMIC_CRITICAL_CPT_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
TYPE new_value; \
/*printf("__kmp_atomic_mode = %d\n", __kmp_atomic_mode);*/ \
OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \
}
/* ------------------------------------------------------------------------- */
// routines for long double type
ATOMIC_CRITICAL_CPT_REV(float10, sub_cpt_rev, long double, -, 10r,
1) // __kmpc_atomic_float10_sub_cpt_rev
ATOMIC_CRITICAL_CPT_REV(float10, div_cpt_rev, long double, /, 10r,
1) // __kmpc_atomic_float10_div_cpt_rev
#if KMP_HAVE_QUAD
// routines for _Quad type
ATOMIC_CRITICAL_CPT_REV(float16, sub_cpt_rev, QUAD_LEGACY, -, 16r,
1) // __kmpc_atomic_float16_sub_cpt_rev
ATOMIC_CRITICAL_CPT_REV(float16, div_cpt_rev, QUAD_LEGACY, /, 16r,
1) // __kmpc_atomic_float16_div_cpt_rev
#if (KMP_ARCH_X86)
ATOMIC_CRITICAL_CPT_REV(float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r,
1) // __kmpc_atomic_float16_sub_a16_cpt_rev
ATOMIC_CRITICAL_CPT_REV(float16, div_a16_cpt_rev, Quad_a16_t, /, 16r,
1) // __kmpc_atomic_float16_div_a16_cpt_rev
#endif // (KMP_ARCH_X86)
#endif // KMP_HAVE_QUAD
// routines for complex types
// ------------------------------------------------------------------------
// Workaround for cmplx4. Regular routines with return value don't work
// on Win_32e. Let's return captured values through the additional parameter.
#define OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
if (flag) { \
(*lhs) = (rhs)OP(*lhs); \
(*out) = (*lhs); \
} else { \
(*out) = (*lhs); \
(*lhs) = (rhs)OP(*lhs); \
} \
\
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
return;
// ------------------------------------------------------------------------
#ifdef KMP_GOMP_COMPAT
#define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
OP_CRITICAL_CPT_REV_WRK(OP, 0); \
}
#else
#define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG)
#endif /* KMP_GOMP_COMPAT */
// ------------------------------------------------------------------------
#define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, \
GOMP_FLAG) \
ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
OP_GOMP_CRITICAL_CPT_REV_WRK(OP, GOMP_FLAG) \
OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \
}
// The end of workaround for cmplx4
// !!! TODO: check if we need to return void for cmplx4 routines
// cmplx4 routines to return void
ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c,
1) // __kmpc_atomic_cmplx4_sub_cpt_rev
ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c,
1) // __kmpc_atomic_cmplx4_div_cpt_rev
ATOMIC_CRITICAL_CPT_REV(cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c,
1) // __kmpc_atomic_cmplx8_sub_cpt_rev
ATOMIC_CRITICAL_CPT_REV(cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c,
1) // __kmpc_atomic_cmplx8_div_cpt_rev
ATOMIC_CRITICAL_CPT_REV(cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c,
1) // __kmpc_atomic_cmplx10_sub_cpt_rev
ATOMIC_CRITICAL_CPT_REV(cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c,
1) // __kmpc_atomic_cmplx10_div_cpt_rev
#if KMP_HAVE_QUAD
ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c,
1) // __kmpc_atomic_cmplx16_sub_cpt_rev
ATOMIC_CRITICAL_CPT_REV(cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c,
1) // __kmpc_atomic_cmplx16_div_cpt_rev
#if (KMP_ARCH_X86)
ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c,
1) // __kmpc_atomic_cmplx16_sub_a16_cpt_rev
ATOMIC_CRITICAL_CPT_REV(cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c,
1) // __kmpc_atomic_cmplx16_div_a16_cpt_rev
#endif // (KMP_ARCH_X86)
#endif // KMP_HAVE_QUAD
// Capture reverse for mixed type: RHS=float16
#if KMP_HAVE_QUAD
// Beginning of a definition (provides name, parameters, gebug trace)
// TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned
// fixed)
// OP_ID - operation identifier (add, sub, mul, ...)
// TYPE - operands' type
// -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
RTYPE, LCK_ID, MASK, GOMP_FLAG) \
ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
TYPE new_value; \
(void)new_value; \
OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
}
// -------------------------------------------------------------------------
#define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
TYPE new_value; \
(void)new_value; \
OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) /* send assignment */ \
OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) /* send assignment */ \
}
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1, char, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_sub_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_sub_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1, char, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1_div_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
KMP_ARCH_X86) // __kmpc_atomic_fixed1u_div_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2, short, sub_cpt_rev, 16, -, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_sub_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, sub_cpt_rev, 16, -, fp, _Quad, 2i,
1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_sub_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2, short, div_cpt_rev, 16, /, fp, _Quad, 2i, 1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2_div_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, div_cpt_rev, 16, /, fp, _Quad, 2i,
1,
KMP_ARCH_X86) // __kmpc_atomic_fixed2u_div_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, sub_cpt_rev, 32, -, fp, _Quad, 4i,
3, 0) // __kmpc_atomic_fixed4_sub_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, sub_cpt_rev, 32, -, fp, _Quad,
4i, 3, 0) // __kmpc_atomic_fixed4u_sub_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, div_cpt_rev, 32, /, fp, _Quad, 4i,
3, 0) // __kmpc_atomic_fixed4_div_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, div_cpt_rev, 32, /, fp, _Quad,
4i, 3, 0) // __kmpc_atomic_fixed4u_div_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, sub_cpt_rev, 64, -, fp, _Quad, 8i,
7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_sub_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, sub_cpt_rev, 64, -, fp, _Quad,
8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_sub_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, div_cpt_rev, 64, /, fp, _Quad, 8i,
7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_div_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, div_cpt_rev, 64, /, fp, _Quad,
8i, 7,
KMP_ARCH_X86) // __kmpc_atomic_fixed8u_div_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, sub_cpt_rev, 32, -, fp, _Quad,
4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_sub_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, div_cpt_rev, 32, /, fp, _Quad,
4r, 3,
KMP_ARCH_X86) // __kmpc_atomic_float4_div_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, sub_cpt_rev, 64, -, fp, _Quad,
8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_sub_cpt_rev_fp
ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, div_cpt_rev, 64, /, fp, _Quad,
8r, 7,
KMP_ARCH_X86) // __kmpc_atomic_float8_div_cpt_rev_fp
ATOMIC_CRITICAL_CPT_REV_MIX(float10, long double, sub_cpt_rev, -, fp, _Quad,
10r, 1) // __kmpc_atomic_float10_sub_cpt_rev_fp
ATOMIC_CRITICAL_CPT_REV_MIX(float10, long double, div_cpt_rev, /, fp, _Quad,
10r, 1) // __kmpc_atomic_float10_div_cpt_rev_fp
#endif // KMP_HAVE_QUAD
// OpenMP 4.0 Capture-write (swap): {v = x; x = expr;}
#define ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
TYPE __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \
TYPE rhs) { \
KMP_DEBUG_ASSERT(__kmp_init_serial); \
KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid));
#define CRITICAL_SWP(LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
old_value = (*lhs); \
(*lhs) = rhs; \
\
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
return old_value;
// ------------------------------------------------------------------------
#ifdef KMP_GOMP_COMPAT
#define GOMP_CRITICAL_SWP(FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
CRITICAL_SWP(0); \
}
#else
#define GOMP_CRITICAL_SWP(FLAG)
#endif /* KMP_GOMP_COMPAT */
#define ATOMIC_XCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
TYPE old_value; \
GOMP_CRITICAL_SWP(GOMP_FLAG) \
old_value = KMP_XCHG_FIXED##BITS(lhs, rhs); \
return old_value; \
}
// ------------------------------------------------------------------------
#define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
TYPE old_value; \
GOMP_CRITICAL_SWP(GOMP_FLAG) \
old_value = KMP_XCHG_REAL##BITS(lhs, rhs); \
return old_value; \
}
// ------------------------------------------------------------------------
#define CMPXCHG_SWP(TYPE, BITS) \
{ \
TYPE KMP_ATOMIC_VOLATILE temp_val; \
TYPE old_value, new_value; \
temp_val = *lhs; \
old_value = temp_val; \
new_value = rhs; \
while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
(kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
*VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
temp_val = *lhs; \
old_value = temp_val; \
new_value = rhs; \
} \
return old_value; \
}
// -------------------------------------------------------------------------
#define ATOMIC_CMPXCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
TYPE old_value; \
(void)old_value; \
GOMP_CRITICAL_SWP(GOMP_FLAG) \
CMPXCHG_SWP(TYPE, BITS) \
}
ATOMIC_XCHG_SWP(fixed1, kmp_int8, 8, KMP_ARCH_X86) // __kmpc_atomic_fixed1_swp
ATOMIC_XCHG_SWP(fixed2, kmp_int16, 16, KMP_ARCH_X86) // __kmpc_atomic_fixed2_swp
ATOMIC_XCHG_SWP(fixed4, kmp_int32, 32, KMP_ARCH_X86) // __kmpc_atomic_fixed4_swp
ATOMIC_XCHG_FLOAT_SWP(float4, kmp_real32, 32,
KMP_ARCH_X86) // __kmpc_atomic_float4_swp
#if (KMP_ARCH_X86)
ATOMIC_CMPXCHG_SWP(fixed8, kmp_int64, 64,
KMP_ARCH_X86) // __kmpc_atomic_fixed8_swp
ATOMIC_CMPXCHG_SWP(float8, kmp_real64, 64,
KMP_ARCH_X86) // __kmpc_atomic_float8_swp
#else
ATOMIC_XCHG_SWP(fixed8, kmp_int64, 64, KMP_ARCH_X86) // __kmpc_atomic_fixed8_swp
ATOMIC_XCHG_FLOAT_SWP(float8, kmp_real64, 64,
KMP_ARCH_X86) // __kmpc_atomic_float8_swp
#endif // (KMP_ARCH_X86)
// ------------------------------------------------------------------------
// Routines for Extended types: long double, _Quad, complex flavours (use
// critical section)
#define ATOMIC_CRITICAL_SWP(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
TYPE old_value; \
GOMP_CRITICAL_SWP(GOMP_FLAG) \
CRITICAL_SWP(LCK_ID) \
}
// ------------------------------------------------------------------------
// !!! TODO: check if we need to return void for cmplx4 routines
// Workaround for cmplx4. Regular routines with return value don't work
// on Win_32e. Let's return captured values through the additional parameter.
#define ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \
void __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \
TYPE rhs, TYPE *out) { \
KMP_DEBUG_ASSERT(__kmp_init_serial); \
KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid));
#define CRITICAL_SWP_WRK(LCK_ID) \
__kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
\
tmp = (*lhs); \
(*lhs) = (rhs); \
(*out) = tmp; \
__kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
return;
// ------------------------------------------------------------------------
#ifdef KMP_GOMP_COMPAT
#define GOMP_CRITICAL_SWP_WRK(FLAG) \
if ((FLAG) && (__kmp_atomic_mode == 2)) { \
KMP_CHECK_GTID; \
CRITICAL_SWP_WRK(0); \
}
#else
#define GOMP_CRITICAL_SWP_WRK(FLAG)
#endif /* KMP_GOMP_COMPAT */
// ------------------------------------------------------------------------
#define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \
ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \
TYPE tmp; \
GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \
CRITICAL_SWP_WRK(LCK_ID) \
}
// The end of workaround for cmplx4
ATOMIC_CRITICAL_SWP(float10, long double, 10r, 1) // __kmpc_atomic_float10_swp
#if KMP_HAVE_QUAD
ATOMIC_CRITICAL_SWP(float16, QUAD_LEGACY, 16r, 1) // __kmpc_atomic_float16_swp
#endif // KMP_HAVE_QUAD
// cmplx4 routine to return void
ATOMIC_CRITICAL_SWP_WRK(cmplx4, kmp_cmplx32, 8c, 1) // __kmpc_atomic_cmplx4_swp
// ATOMIC_CRITICAL_SWP( cmplx4, kmp_cmplx32, 8c, 1 ) //
// __kmpc_atomic_cmplx4_swp
ATOMIC_CRITICAL_SWP(cmplx8, kmp_cmplx64, 16c, 1) // __kmpc_atomic_cmplx8_swp
ATOMIC_CRITICAL_SWP(cmplx10, kmp_cmplx80, 20c, 1) // __kmpc_atomic_cmplx10_swp
#if KMP_HAVE_QUAD
ATOMIC_CRITICAL_SWP(cmplx16, CPLX128_LEG, 32c, 1) // __kmpc_atomic_cmplx16_swp
#if (KMP_ARCH_X86)
ATOMIC_CRITICAL_SWP(float16_a16, Quad_a16_t, 16r,
1) // __kmpc_atomic_float16_a16_swp
ATOMIC_CRITICAL_SWP(cmplx16_a16, kmp_cmplx128_a16_t, 32c,
1) // __kmpc_atomic_cmplx16_a16_swp
#endif // (KMP_ARCH_X86)
#endif // KMP_HAVE_QUAD
// End of OpenMP 4.0 Capture
#endif // KMP_ARCH_X86 || KMP_ARCH_X86_64
#undef OP_CRITICAL
/* ------------------------------------------------------------------------ */
/* Generic atomic routines */
void __kmpc_atomic_1(ident_t *id_ref, int gtid, void *lhs, void *rhs,
void (*f)(void *, void *, void *)) {
KMP_DEBUG_ASSERT(__kmp_init_serial);
if (
#if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
FALSE /* must use lock */
#else
TRUE
#endif // KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
) {
kmp_int8 old_value, new_value;
old_value = *(kmp_int8 *)lhs;
(*f)(&new_value, &old_value, rhs);
/* TODO: Should this be acquire or release? */
while (!KMP_COMPARE_AND_STORE_ACQ8((kmp_int8 *)lhs, *(kmp_int8 *)&old_value,
*(kmp_int8 *)&new_value)) {
KMP_CPU_PAUSE();
old_value = *(kmp_int8 *)lhs;
(*f)(&new_value, &old_value, rhs);
}
return;
} else {
// All 1-byte data is of integer data type.
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_acquire_atomic_lock(&__kmp_atomic_lock_1i, gtid);
(*f)(lhs, lhs, rhs);
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_release_atomic_lock(&__kmp_atomic_lock_1i, gtid);
}
}
void __kmpc_atomic_2(ident_t *id_ref, int gtid, void *lhs, void *rhs,
void (*f)(void *, void *, void *)) {
if (
#if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
FALSE /* must use lock */
#elif KMP_ARCH_X86 || KMP_ARCH_X86_64
TRUE /* no alignment problems */
#else
!((kmp_uintptr_t)lhs & 0x1) /* make sure address is 2-byte aligned */
#endif // KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
) {
kmp_int16 old_value, new_value;
old_value = *(kmp_int16 *)lhs;
(*f)(&new_value, &old_value, rhs);
/* TODO: Should this be acquire or release? */
while (!KMP_COMPARE_AND_STORE_ACQ16(
(kmp_int16 *)lhs, *(kmp_int16 *)&old_value, *(kmp_int16 *)&new_value)) {
KMP_CPU_PAUSE();
old_value = *(kmp_int16 *)lhs;
(*f)(&new_value, &old_value, rhs);
}
return;
} else {
// All 2-byte data is of integer data type.
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_acquire_atomic_lock(&__kmp_atomic_lock_2i, gtid);
(*f)(lhs, lhs, rhs);
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_release_atomic_lock(&__kmp_atomic_lock_2i, gtid);
}
}
void __kmpc_atomic_4(ident_t *id_ref, int gtid, void *lhs, void *rhs,
void (*f)(void *, void *, void *)) {
KMP_DEBUG_ASSERT(__kmp_init_serial);
if (
// FIXME: On IA-32 architecture, gcc uses cmpxchg only for 4-byte ints.
// Gomp compatibility is broken if this routine is called for floats.
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
TRUE /* no alignment problems */
#else
!((kmp_uintptr_t)lhs & 0x3) /* make sure address is 4-byte aligned */
#endif // KMP_ARCH_X86 || KMP_ARCH_X86_64
) {
kmp_int32 old_value, new_value;
old_value = *(kmp_int32 *)lhs;
(*f)(&new_value, &old_value, rhs);
/* TODO: Should this be acquire or release? */
while (!KMP_COMPARE_AND_STORE_ACQ32(
(kmp_int32 *)lhs, *(kmp_int32 *)&old_value, *(kmp_int32 *)&new_value)) {
KMP_CPU_PAUSE();
old_value = *(kmp_int32 *)lhs;
(*f)(&new_value, &old_value, rhs);
}
return;
} else {
// Use __kmp_atomic_lock_4i for all 4-byte data,
// even if it isn't of integer data type.
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_acquire_atomic_lock(&__kmp_atomic_lock_4i, gtid);
(*f)(lhs, lhs, rhs);
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_release_atomic_lock(&__kmp_atomic_lock_4i, gtid);
}
}
void __kmpc_atomic_8(ident_t *id_ref, int gtid, void *lhs, void *rhs,
void (*f)(void *, void *, void *)) {
KMP_DEBUG_ASSERT(__kmp_init_serial);
if (
#if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
FALSE /* must use lock */
#elif KMP_ARCH_X86 || KMP_ARCH_X86_64
TRUE /* no alignment problems */
#else
!((kmp_uintptr_t)lhs & 0x7) /* make sure address is 8-byte aligned */
#endif // KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
) {
kmp_int64 old_value, new_value;
old_value = *(kmp_int64 *)lhs;
(*f)(&new_value, &old_value, rhs);
/* TODO: Should this be acquire or release? */
while (!KMP_COMPARE_AND_STORE_ACQ64(
(kmp_int64 *)lhs, *(kmp_int64 *)&old_value, *(kmp_int64 *)&new_value)) {
KMP_CPU_PAUSE();
old_value = *(kmp_int64 *)lhs;
(*f)(&new_value, &old_value, rhs);
}
return;
} else {
// Use __kmp_atomic_lock_8i for all 8-byte data,
// even if it isn't of integer data type.
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_acquire_atomic_lock(&__kmp_atomic_lock_8i, gtid);
(*f)(lhs, lhs, rhs);
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_release_atomic_lock(&__kmp_atomic_lock_8i, gtid);
}
}
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
void __kmpc_atomic_10(ident_t *id_ref, int gtid, void *lhs, void *rhs,
void (*f)(void *, void *, void *)) {
KMP_DEBUG_ASSERT(__kmp_init_serial);
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_acquire_atomic_lock(&__kmp_atomic_lock_10r, gtid);
(*f)(lhs, lhs, rhs);
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_release_atomic_lock(&__kmp_atomic_lock_10r, gtid);
}
#endif // KMP_ARCH_X86 || KMP_ARCH_X86_64
void __kmpc_atomic_16(ident_t *id_ref, int gtid, void *lhs, void *rhs,
void (*f)(void *, void *, void *)) {
KMP_DEBUG_ASSERT(__kmp_init_serial);
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_acquire_atomic_lock(&__kmp_atomic_lock_16c, gtid);
(*f)(lhs, lhs, rhs);
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_release_atomic_lock(&__kmp_atomic_lock_16c, gtid);
}
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
void __kmpc_atomic_20(ident_t *id_ref, int gtid, void *lhs, void *rhs,
void (*f)(void *, void *, void *)) {
KMP_DEBUG_ASSERT(__kmp_init_serial);
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_acquire_atomic_lock(&__kmp_atomic_lock_20c, gtid);
(*f)(lhs, lhs, rhs);
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_release_atomic_lock(&__kmp_atomic_lock_20c, gtid);
}
#endif // KMP_ARCH_X86 || KMP_ARCH_X86_64
void __kmpc_atomic_32(ident_t *id_ref, int gtid, void *lhs, void *rhs,
void (*f)(void *, void *, void *)) {
KMP_DEBUG_ASSERT(__kmp_init_serial);
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_acquire_atomic_lock(&__kmp_atomic_lock_32c, gtid);
(*f)(lhs, lhs, rhs);
#ifdef KMP_GOMP_COMPAT
if (__kmp_atomic_mode == 2) {
__kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
} else
#endif /* KMP_GOMP_COMPAT */
__kmp_release_atomic_lock(&__kmp_atomic_lock_32c, gtid);
}
// AC: same two routines as GOMP_atomic_start/end, but will be called by our
// compiler; duplicated in order to not use 3-party names in pure Intel code
// TODO: consider adding GTID parameter after consultation with Ernesto/Xinmin.
void __kmpc_atomic_start(void) {
int gtid = __kmp_entry_gtid();
KA_TRACE(20, ("__kmpc_atomic_start: T#%d\n", gtid));
__kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
}
void __kmpc_atomic_end(void) {
int gtid = __kmp_get_gtid();
KA_TRACE(20, ("__kmpc_atomic_end: T#%d\n", gtid));
__kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
}
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
// OpenMP 5.1 compare and swap
bool __kmpc_atomic_bool_1_cas(ident_t *loc, int gtid, char *x, char e, char d) {
return KMP_COMPARE_AND_STORE_ACQ8(x, e, d);
}
bool __kmpc_atomic_bool_2_cas(ident_t *loc, int gtid, short *x, short e,
short d) {
return KMP_COMPARE_AND_STORE_ACQ16(x, e, d);
}
bool __kmpc_atomic_bool_4_cas(ident_t *loc, int gtid, kmp_int32 *x, kmp_int32 e,
kmp_int32 d) {
return KMP_COMPARE_AND_STORE_ACQ32(x, e, d);
}
bool __kmpc_atomic_bool_8_cas(ident_t *loc, int gtid, kmp_int64 *x, kmp_int64 e,
kmp_int64 d) {
return KMP_COMPARE_AND_STORE_ACQ64(x, e, d);
}
char __kmpc_atomic_val_1_cas(ident_t *loc, int gtid, char *x, char e, char d) {
return KMP_COMPARE_AND_STORE_RET8(x, e, d);
}
short __kmpc_atomic_val_2_cas(ident_t *loc, int gtid, short *x, short e,
short d) {
return KMP_COMPARE_AND_STORE_RET16(x, e, d);
}
kmp_int32 __kmpc_atomic_val_4_cas(ident_t *loc, int gtid, kmp_int32 *x,
kmp_int32 e, kmp_int32 d) {
return KMP_COMPARE_AND_STORE_RET32(x, e, d);
}
kmp_int64 __kmpc_atomic_val_8_cas(ident_t *loc, int gtid, kmp_int64 *x,
kmp_int64 e, kmp_int64 d) {
return KMP_COMPARE_AND_STORE_RET64(x, e, d);
}
bool __kmpc_atomic_bool_1_cas_cpt(ident_t *loc, int gtid, char *x, char e,
char d, char *pv) {
char old = KMP_COMPARE_AND_STORE_RET8(x, e, d);
if (old == e)
return true;
KMP_ASSERT(pv != NULL);
*pv = old;
return false;
}
bool __kmpc_atomic_bool_2_cas_cpt(ident_t *loc, int gtid, short *x, short e,
short d, short *pv) {
short old = KMP_COMPARE_AND_STORE_RET16(x, e, d);
if (old == e)
return true;
KMP_ASSERT(pv != NULL);
*pv = old;
return false;
}
bool __kmpc_atomic_bool_4_cas_cpt(ident_t *loc, int gtid, kmp_int32 *x,
kmp_int32 e, kmp_int32 d, kmp_int32 *pv) {
kmp_int32 old = KMP_COMPARE_AND_STORE_RET32(x, e, d);
if (old == e)
return true;
KMP_ASSERT(pv != NULL);
*pv = old;
return false;
}
bool __kmpc_atomic_bool_8_cas_cpt(ident_t *loc, int gtid, kmp_int64 *x,
kmp_int64 e, kmp_int64 d, kmp_int64 *pv) {
kmp_int64 old = KMP_COMPARE_AND_STORE_RET64(x, e, d);
if (old == e)
return true;
KMP_ASSERT(pv != NULL);
*pv = old;
return false;
}
char __kmpc_atomic_val_1_cas_cpt(ident_t *loc, int gtid, char *x, char e,
char d, char *pv) {
char old = KMP_COMPARE_AND_STORE_RET8(x, e, d);
KMP_ASSERT(pv != NULL);
*pv = old == e ? d : old;
return old;
}
short __kmpc_atomic_val_2_cas_cpt(ident_t *loc, int gtid, short *x, short e,
short d, short *pv) {
short old = KMP_COMPARE_AND_STORE_RET16(x, e, d);
KMP_ASSERT(pv != NULL);
*pv = old == e ? d : old;
return old;
}
kmp_int32 __kmpc_atomic_val_4_cas_cpt(ident_t *loc, int gtid, kmp_int32 *x,
kmp_int32 e, kmp_int32 d, kmp_int32 *pv) {
kmp_int32 old = KMP_COMPARE_AND_STORE_RET32(x, e, d);
KMP_ASSERT(pv != NULL);
*pv = old == e ? d : old;
return old;
}
kmp_int64 __kmpc_atomic_val_8_cas_cpt(ident_t *loc, int gtid, kmp_int64 *x,
kmp_int64 e, kmp_int64 d, kmp_int64 *pv) {
kmp_int64 old = KMP_COMPARE_AND_STORE_RET64(x, e, d);
KMP_ASSERT(pv != NULL);
*pv = old == e ? d : old;
return old;
}
// End OpenMP 5.1 compare + capture
#endif // KMP_ARCH_X86 || KMP_ARCH_X86_64
// end of file
ident
Definition kmp.h:246