scalar-inl.h
Go to the documentation of this file.
// Copyright 2019 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Single-element vectors and operations.
// External include guard in highway.h - see comment there.
#include <stddef.h>
#include <stdint.h>
#include "hwy/base.h"
#include "hwy/ops/shared-inl.h"
HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
// Single instruction, single data.
template <typename T>
using Sisd = Simd<T, 1, 0>;
// (Wrapper class required for overloading comparison operators.)
template <typename T>
struct Vec1 {
using PrivateT = T; // only for DFromV
static constexpr size_t kPrivateN = 1; // only for DFromV
HWY_INLINE Vec1() = default;
Vec1(const Vec1&) = default;
Vec1& operator=(const Vec1&) = default;
HWY_INLINE explicit Vec1(const T t) : raw(t) {}
HWY_INLINE Vec1& operator*=(const Vec1 other) {
return *this = (*this * other);
}
HWY_INLINE Vec1& operator/=(const Vec1 other) {
return *this = (*this / other);
}
HWY_INLINE Vec1& operator+=(const Vec1 other) {
return *this = (*this + other);
}
HWY_INLINE Vec1& operator-=(const Vec1 other) {
return *this = (*this - other);
}
HWY_INLINE Vec1& operator&=(const Vec1 other) {
return *this = (*this & other);
}
HWY_INLINE Vec1& operator|=(const Vec1 other) {
return *this = (*this | other);
}
HWY_INLINE Vec1& operator^=(const Vec1 other) {
return *this = (*this ^ other);
}
T raw;
};
// 0 or FF..FF, same size as Vec1.
template <typename T>
class Mask1 {
using Raw = hwy::MakeUnsigned<T>;
public:
static HWY_INLINE Mask1<T> FromBool(bool b) {
Mask1<T> mask;
mask.bits = b ? static_cast<Raw>(~Raw{0}) : 0;
return mask;
}
Raw bits;
};
template <class V>
using DFromV = Simd<typename V::PrivateT, V::kPrivateN, 0>;
template <class V>
using TFromV = typename V::PrivateT;
// ------------------------------ BitCast
template <typename T, typename FromT>
HWY_API Vec1<T> BitCast(Sisd<T> /* tag */, Vec1<FromT> v) {
static_assert(sizeof(T) <= sizeof(FromT), "Promoting is undefined");
T to;
CopyBytes<sizeof(FromT)>(&v.raw, &to); // not same size - ok to shrink
return Vec1<T>(to);
}
// ------------------------------ Set
template <typename T>
HWY_API Vec1<T> Zero(Sisd<T> /* tag */) {
return Vec1<T>(T(0));
}
template <typename T, typename T2>
HWY_API Vec1<T> Set(Sisd<T> /* tag */, const T2 t) {
return Vec1<T>(static_cast<T>(t));
}
template <typename T>
HWY_API Vec1<T> Undefined(Sisd<T> d) {
return Zero(d);
}
template <typename T, typename T2>
HWY_API Vec1<T> Iota(const Sisd<T> /* tag */, const T2 first) {
return Vec1<T>(static_cast<T>(first));
}
template <class D>
using VFromD = decltype(Zero(D()));
// ================================================== LOGICAL
// ------------------------------ Not
template <typename T>
HWY_API Vec1<T> Not(const Vec1<T> v) {
using TU = MakeUnsigned<T>;
const Sisd<TU> du;
return BitCast(Sisd<T>(), Vec1<TU>(static_cast<TU>(~BitCast(du, v).raw)));
}
// ------------------------------ And
template <typename T>
HWY_API Vec1<T> And(const Vec1<T> a, const Vec1<T> b) {
using TU = MakeUnsigned<T>;
const Sisd<TU> du;
return BitCast(Sisd<T>(), Vec1<TU>(BitCast(du, a).raw & BitCast(du, b).raw));
}
template <typename T>
HWY_API Vec1<T> operator&(const Vec1<T> a, const Vec1<T> b) {
return And(a, b);
}
// ------------------------------ AndNot
template <typename T>
HWY_API Vec1<T> AndNot(const Vec1<T> a, const Vec1<T> b) {
using TU = MakeUnsigned<T>;
const Sisd<TU> du;
return BitCast(Sisd<T>(), Vec1<TU>(static_cast<TU>(~BitCast(du, a).raw &
BitCast(du, b).raw)));
}
// ------------------------------ Or
template <typename T>
HWY_API Vec1<T> Or(const Vec1<T> a, const Vec1<T> b) {
using TU = MakeUnsigned<T>;
const Sisd<TU> du;
return BitCast(Sisd<T>(), Vec1<TU>(BitCast(du, a).raw | BitCast(du, b).raw));
}
template <typename T>
HWY_API Vec1<T> operator|(const Vec1<T> a, const Vec1<T> b) {
return Or(a, b);
}
// ------------------------------ Xor
template <typename T>
HWY_API Vec1<T> Xor(const Vec1<T> a, const Vec1<T> b) {
using TU = MakeUnsigned<T>;
const Sisd<TU> du;
return BitCast(Sisd<T>(), Vec1<TU>(BitCast(du, a).raw ^ BitCast(du, b).raw));
}
template <typename T>
HWY_API Vec1<T> operator^(const Vec1<T> a, const Vec1<T> b) {
return Xor(a, b);
}
// ------------------------------ Xor3
template <typename T>
HWY_API Vec1<T> Xor3(Vec1<T> x1, Vec1<T> x2, Vec1<T> x3) {
return Xor(x1, Xor(x2, x3));
}
// ------------------------------ Or3
template <typename T>
HWY_API Vec1<T> Or3(Vec1<T> o1, Vec1<T> o2, Vec1<T> o3) {
return Or(o1, Or(o2, o3));
}
// ------------------------------ OrAnd
template <typename T>
HWY_API Vec1<T> OrAnd(const Vec1<T> o, const Vec1<T> a1, const Vec1<T> a2) {
return Or(o, And(a1, a2));
}
// ------------------------------ IfVecThenElse
template <typename T>
HWY_API Vec1<T> IfVecThenElse(Vec1<T> mask, Vec1<T> yes, Vec1<T> no) {
return IfThenElse(MaskFromVec(mask), yes, no);
}
// ------------------------------ CopySign
template <typename T>
HWY_API Vec1<T> CopySign(const Vec1<T> magn, const Vec1<T> sign) {
static_assert(IsFloat<T>(), "Only makes sense for floating-point");
const auto msb = SignBit(Sisd<T>());
return Or(AndNot(msb, magn), And(msb, sign));
}
template <typename T>
HWY_API Vec1<T> CopySignToAbs(const Vec1<T> abs, const Vec1<T> sign) {
static_assert(IsFloat<T>(), "Only makes sense for floating-point");
return Or(abs, And(SignBit(Sisd<T>()), sign));
}
// ------------------------------ BroadcastSignBit
template <typename T>
HWY_API Vec1<T> BroadcastSignBit(const Vec1<T> v) {
// This is used inside ShiftRight, so we cannot implement in terms of it.
return v.raw < 0 ? Vec1<T>(T(-1)) : Vec1<T>(0);
}
// ------------------------------ PopulationCount
#ifdef HWY_NATIVE_POPCNT
#undef HWY_NATIVE_POPCNT
#else
#define HWY_NATIVE_POPCNT
#endif
template <typename T>
HWY_API Vec1<T> PopulationCount(Vec1<T> v) {
return Vec1<T>(static_cast<T>(PopCount(v.raw)));
}
// ------------------------------ Mask
template <typename TFrom, typename TTo>
HWY_API Mask1<TTo> RebindMask(Sisd<TTo> /*tag*/, Mask1<TFrom> m) {
static_assert(sizeof(TFrom) == sizeof(TTo), "Must have same size");
return Mask1<TTo>{m.bits};
}
// v must be 0 or FF..FF.
template <typename T>
HWY_API Mask1<T> MaskFromVec(const Vec1<T> v) {
Mask1<T> mask;
CopySameSize(&v, &mask);
return mask;
}
template <typename T>
Vec1<T> VecFromMask(const Mask1<T> mask) {
Vec1<T> v;
CopySameSize(&mask, &v);
return v;
}
template <typename T>
Vec1<T> VecFromMask(Sisd<T> /* tag */, const Mask1<T> mask) {
Vec1<T> v;
CopySameSize(&mask, &v);
return v;
}
template <typename T>
HWY_API Mask1<T> FirstN(Sisd<T> /*tag*/, size_t n) {
return Mask1<T>::FromBool(n != 0);
}
// Returns mask ? yes : no.
template <typename T>
HWY_API Vec1<T> IfThenElse(const Mask1<T> mask, const Vec1<T> yes,
const Vec1<T> no) {
return mask.bits ? yes : no;
}
template <typename T>
HWY_API Vec1<T> IfThenElseZero(const Mask1<T> mask, const Vec1<T> yes) {
return mask.bits ? yes : Vec1<T>(0);
}
template <typename T>
HWY_API Vec1<T> IfThenZeroElse(const Mask1<T> mask, const Vec1<T> no) {
return mask.bits ? Vec1<T>(0) : no;
}
template <typename T>
HWY_API Vec1<T> IfNegativeThenElse(Vec1<T> v, Vec1<T> yes, Vec1<T> no) {
return v.raw < 0 ? yes : no;
}
template <typename T>
HWY_API Vec1<T> ZeroIfNegative(const Vec1<T> v) {
return v.raw < 0 ? Vec1<T>(0) : v;
}
// ------------------------------ Mask logical
template <typename T>
HWY_API Mask1<T> Not(const Mask1<T> m) {
return MaskFromVec(Not(VecFromMask(Sisd<T>(), m)));
}
template <typename T>
HWY_API Mask1<T> And(const Mask1<T> a, Mask1<T> b) {
const Sisd<T> d;
return MaskFromVec(And(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T>
HWY_API Mask1<T> AndNot(const Mask1<T> a, Mask1<T> b) {
const Sisd<T> d;
return MaskFromVec(AndNot(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T>
HWY_API Mask1<T> Or(const Mask1<T> a, Mask1<T> b) {
const Sisd<T> d;
return MaskFromVec(Or(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T>
HWY_API Mask1<T> Xor(const Mask1<T> a, Mask1<T> b) {
const Sisd<T> d;
return MaskFromVec(Xor(VecFromMask(d, a), VecFromMask(d, b)));
}
template <typename T>
HWY_API Mask1<T> ExclusiveNeither(const Mask1<T> a, Mask1<T> b) {
const Sisd<T> d;
return MaskFromVec(AndNot(VecFromMask(d, a), Not(VecFromMask(d, b))));
}
// ================================================== SHIFTS
// ------------------------------ ShiftLeft/ShiftRight (BroadcastSignBit)
template <int kBits, typename T>
HWY_API Vec1<T> ShiftLeft(const Vec1<T> v) {
static_assert(0 <= kBits && kBits < sizeof(T) * 8, "Invalid shift");
return Vec1<T>(
static_cast<T>(static_cast<hwy::MakeUnsigned<T>>(v.raw) << kBits));
}
template <int kBits, typename T>
HWY_API Vec1<T> ShiftRight(const Vec1<T> v) {
static_assert(0 <= kBits && kBits < sizeof(T) * 8, "Invalid shift");
#if __cplusplus >= 202002L
// Signed right shift is now guaranteed to be arithmetic (rounding toward
// negative infinity, i.e. shifting in the sign bit).
return Vec1<T>(static_cast<T>(v.raw >> kBits));
#else
if (IsSigned<T>()) {
// Emulate arithmetic shift using only logical (unsigned) shifts, because
// signed shifts are still implementation-defined.
using TU = hwy::MakeUnsigned<T>;
const Sisd<TU> du;
const TU shifted = static_cast<TU>(BitCast(du, v).raw >> kBits);
const TU sign = BitCast(du, BroadcastSignBit(v)).raw;
const size_t sign_shift =
static_cast<size_t>(static_cast<int>(sizeof(TU)) * 8 - 1 - kBits);
const TU upper = static_cast<TU>(sign << sign_shift);
return BitCast(Sisd<T>(), Vec1<TU>(shifted | upper));
} else { // T is unsigned
return Vec1<T>(static_cast<T>(v.raw >> kBits));
}
#endif
}
// ------------------------------ RotateRight (ShiftRight)
namespace detail {
// For partial specialization: kBits == 0 results in an invalid shift count
template <int kBits>
struct RotateRight {
template <typename T>
HWY_INLINE Vec1<T> operator()(const Vec1<T> v) const {
return Or(ShiftRight<kBits>(v), ShiftLeft<sizeof(T) * 8 - kBits>(v));
}
};
template <>
struct RotateRight<0> {
template <typename T>
HWY_INLINE Vec1<T> operator()(const Vec1<T> v) const {
return v;
}
};
} // namespace detail
template <int kBits, typename T>
HWY_API Vec1<T> RotateRight(const Vec1<T> v) {
static_assert(0 <= kBits && kBits < sizeof(T) * 8, "Invalid shift");
return detail::RotateRight<kBits>()(v);
}
// ------------------------------ ShiftLeftSame (BroadcastSignBit)
template <typename T>
HWY_API Vec1<T> ShiftLeftSame(const Vec1<T> v, int bits) {
return Vec1<T>(
static_cast<T>(static_cast<hwy::MakeUnsigned<T>>(v.raw) << bits));
}
template <typename T>
HWY_API Vec1<T> ShiftRightSame(const Vec1<T> v, int bits) {
#if __cplusplus >= 202002L
// Signed right shift is now guaranteed to be arithmetic (rounding toward
// negative infinity, i.e. shifting in the sign bit).
return Vec1<T>(static_cast<T>(v.raw >> bits));
#else
if (IsSigned<T>()) {
// Emulate arithmetic shift using only logical (unsigned) shifts, because
// signed shifts are still implementation-defined.
using TU = hwy::MakeUnsigned<T>;
const Sisd<TU> du;
const TU shifted = static_cast<TU>(BitCast(du, v).raw >> bits);
const TU sign = BitCast(du, BroadcastSignBit(v)).raw;
const size_t sign_shift =
static_cast<size_t>(static_cast<int>(sizeof(TU)) * 8 - 1 - bits);
const TU upper = static_cast<TU>(sign << sign_shift);
return BitCast(Sisd<T>(), Vec1<TU>(shifted | upper));
} else { // T is unsigned
return Vec1<T>(static_cast<T>(v.raw >> bits));
}
#endif
}
// ------------------------------ Shl
// Single-lane => same as ShiftLeftSame except for the argument type.
template <typename T>
HWY_API Vec1<T> operator<<(const Vec1<T> v, const Vec1<T> bits) {
return ShiftLeftSame(v, static_cast<int>(bits.raw));
}
template <typename T>
HWY_API Vec1<T> operator>>(const Vec1<T> v, const Vec1<T> bits) {
return ShiftRightSame(v, static_cast<int>(bits.raw));
}
// ================================================== ARITHMETIC
template <typename T>
HWY_API Vec1<T> operator+(Vec1<T> a, Vec1<T> b) {
const uint64_t a64 = static_cast<uint64_t>(a.raw);
const uint64_t b64 = static_cast<uint64_t>(b.raw);
return Vec1<T>(static_cast<T>((a64 + b64) & static_cast<uint64_t>(~T(0))));
}
HWY_API Vec1<float> operator+(const Vec1<float> a, const Vec1<float> b) {
return Vec1<float>(a.raw + b.raw);
}
HWY_API Vec1<double> operator+(const Vec1<double> a, const Vec1<double> b) {
return Vec1<double>(a.raw + b.raw);
}
template <typename T>
HWY_API Vec1<T> operator-(Vec1<T> a, Vec1<T> b) {
const uint64_t a64 = static_cast<uint64_t>(a.raw);
const uint64_t b64 = static_cast<uint64_t>(b.raw);
return Vec1<T>(static_cast<T>((a64 - b64) & static_cast<uint64_t>(~T(0))));
}
HWY_API Vec1<float> operator-(const Vec1<float> a, const Vec1<float> b) {
return Vec1<float>(a.raw - b.raw);
}
HWY_API Vec1<double> operator-(const Vec1<double> a, const Vec1<double> b) {
return Vec1<double>(a.raw - b.raw);
}
// ------------------------------ SumsOf8
HWY_API Vec1<uint64_t> SumsOf8(const Vec1<uint8_t> v) {
return Vec1<uint64_t>(v.raw);
}
// ------------------------------ SaturatedAdd
// Returns a + b clamped to the destination range.
// Unsigned
HWY_API Vec1<uint8_t> SaturatedAdd(const Vec1<uint8_t> a,
const Vec1<uint8_t> b) {
return Vec1<uint8_t>(
static_cast<uint8_t>(HWY_MIN(HWY_MAX(0, a.raw + b.raw), 255)));
}
HWY_API Vec1<uint16_t> SaturatedAdd(const Vec1<uint16_t> a,
const Vec1<uint16_t> b) {
return Vec1<uint16_t>(
static_cast<uint16_t>(HWY_MIN(HWY_MAX(0, a.raw + b.raw), 65535)));
}
// Signed
HWY_API Vec1<int8_t> SaturatedAdd(const Vec1<int8_t> a, const Vec1<int8_t> b) {
return Vec1<int8_t>(
static_cast<int8_t>(HWY_MIN(HWY_MAX(-128, a.raw + b.raw), 127)));
}
HWY_API Vec1<int16_t> SaturatedAdd(const Vec1<int16_t> a,
const Vec1<int16_t> b) {
return Vec1<int16_t>(
static_cast<int16_t>(HWY_MIN(HWY_MAX(-32768, a.raw + b.raw), 32767)));
}
// ------------------------------ Saturating subtraction
// Returns a - b clamped to the destination range.
// Unsigned
HWY_API Vec1<uint8_t> SaturatedSub(const Vec1<uint8_t> a,
const Vec1<uint8_t> b) {
return Vec1<uint8_t>(
static_cast<uint8_t>(HWY_MIN(HWY_MAX(0, a.raw - b.raw), 255)));
}
HWY_API Vec1<uint16_t> SaturatedSub(const Vec1<uint16_t> a,
const Vec1<uint16_t> b) {
return Vec1<uint16_t>(
static_cast<uint16_t>(HWY_MIN(HWY_MAX(0, a.raw - b.raw), 65535)));
}
// Signed
HWY_API Vec1<int8_t> SaturatedSub(const Vec1<int8_t> a, const Vec1<int8_t> b) {
return Vec1<int8_t>(
static_cast<int8_t>(HWY_MIN(HWY_MAX(-128, a.raw - b.raw), 127)));
}
HWY_API Vec1<int16_t> SaturatedSub(const Vec1<int16_t> a,
const Vec1<int16_t> b) {
return Vec1<int16_t>(
static_cast<int16_t>(HWY_MIN(HWY_MAX(-32768, a.raw - b.raw), 32767)));
}
// ------------------------------ Average
// Returns (a + b + 1) / 2
HWY_API Vec1<uint8_t> AverageRound(const Vec1<uint8_t> a,
const Vec1<uint8_t> b) {
return Vec1<uint8_t>(static_cast<uint8_t>((a.raw + b.raw + 1) / 2));
}
HWY_API Vec1<uint16_t> AverageRound(const Vec1<uint16_t> a,
const Vec1<uint16_t> b) {
return Vec1<uint16_t>(static_cast<uint16_t>((a.raw + b.raw + 1) / 2));
}
// ------------------------------ Absolute value
template <typename T>
HWY_API Vec1<T> Abs(const Vec1<T> a) {
const T i = a.raw;
return (i >= 0 || i == hwy::LimitsMin<T>()) ? a : Vec1<T>(static_cast<T>(-i));
}
HWY_API Vec1<float> Abs(Vec1<float> a) {
int32_t i;
CopyBytes<sizeof(i)>(&a.raw, &i);
i &= 0x7FFFFFFF;
CopyBytes<sizeof(i)>(&i, &a.raw);
return a;
}
HWY_API Vec1<double> Abs(Vec1<double> a) {
int64_t i;
CopyBytes<sizeof(i)>(&a.raw, &i);
i &= 0x7FFFFFFFFFFFFFFFL;
CopyBytes<sizeof(i)>(&i, &a.raw);
return a;
}
// ------------------------------ Min/Max
// <cmath> may be unavailable, so implement our own.
namespace detail {
static inline float Abs(float f) {
uint32_t i;
CopyBytes<4>(&f, &i);
i &= 0x7FFFFFFFu;
CopyBytes<4>(&i, &f);
return f;
}
static inline double Abs(double f) {
uint64_t i;
CopyBytes<8>(&f, &i);
i &= 0x7FFFFFFFFFFFFFFFull;
CopyBytes<8>(&i, &f);
return f;
}
static inline bool SignBit(float f) {
uint32_t i;
CopyBytes<4>(&f, &i);
return (i >> 31) != 0;
}
static inline bool SignBit(double f) {
uint64_t i;
CopyBytes<8>(&f, &i);
return (i >> 63) != 0;
}
} // namespace detail
template <typename T, HWY_IF_NOT_FLOAT(T)>
HWY_API Vec1<T> Min(const Vec1<T> a, const Vec1<T> b) {
return Vec1<T>(HWY_MIN(a.raw, b.raw));
}
template <typename T, HWY_IF_FLOAT(T)>
HWY_API Vec1<T> Min(const Vec1<T> a, const Vec1<T> b) {
if (isnan(a.raw)) return b;
if (isnan(b.raw)) return a;
return Vec1<T>(HWY_MIN(a.raw, b.raw));
}
template <typename T, HWY_IF_NOT_FLOAT(T)>
HWY_API Vec1<T> Max(const Vec1<T> a, const Vec1<T> b) {
return Vec1<T>(HWY_MAX(a.raw, b.raw));
}
template <typename T, HWY_IF_FLOAT(T)>
HWY_API Vec1<T> Max(const Vec1<T> a, const Vec1<T> b) {
if (isnan(a.raw)) return b;
if (isnan(b.raw)) return a;
return Vec1<T>(HWY_MAX(a.raw, b.raw));
}
// ------------------------------ Floating-point negate
template <typename T, HWY_IF_FLOAT(T)>
HWY_API Vec1<T> Neg(const Vec1<T> v) {
return Xor(v, SignBit(Sisd<T>()));
}
template <typename T, HWY_IF_NOT_FLOAT(T)>
HWY_API Vec1<T> Neg(const Vec1<T> v) {
return Zero(Sisd<T>()) - v;
}
// ------------------------------ mul/div
template <typename T, HWY_IF_FLOAT(T)>
HWY_API Vec1<T> operator*(const Vec1<T> a, const Vec1<T> b) {
return Vec1<T>(static_cast<T>(double{a.raw} * b.raw));
}
template <typename T, HWY_IF_SIGNED(T)>
HWY_API Vec1<T> operator*(const Vec1<T> a, const Vec1<T> b) {
return Vec1<T>(static_cast<T>(static_cast<uint64_t>(a.raw) *
static_cast<uint64_t>(b.raw)));
}
template <typename T, HWY_IF_UNSIGNED(T)>
HWY_API Vec1<T> operator*(const Vec1<T> a, const Vec1<T> b) {
return Vec1<T>(static_cast<T>(static_cast<uint64_t>(a.raw) *
static_cast<uint64_t>(b.raw)));
}
template <typename T>
HWY_API Vec1<T> operator/(const Vec1<T> a, const Vec1<T> b) {
return Vec1<T>(a.raw / b.raw);
}
// Returns the upper 16 bits of a * b in each lane.
HWY_API Vec1<int16_t> MulHigh(const Vec1<int16_t> a, const Vec1<int16_t> b) {
return Vec1<int16_t>(static_cast<int16_t>((a.raw * b.raw) >> 16));
}
HWY_API Vec1<uint16_t> MulHigh(const Vec1<uint16_t> a, const Vec1<uint16_t> b) {
// Cast to uint32_t first to prevent overflow. Otherwise the result of
// uint16_t * uint16_t is in "int" which may overflow. In practice the result
// is the same but this way it is also defined.
return Vec1<uint16_t>(static_cast<uint16_t>(
(static_cast<uint32_t>(a.raw) * static_cast<uint32_t>(b.raw)) >> 16));
}
HWY_API Vec1<int16_t> MulFixedPoint15(Vec1<int16_t> a, Vec1<int16_t> b) {
return Vec1<int16_t>(static_cast<int16_t>((2 * a.raw * b.raw + 32768) >> 16));
}
// Multiplies even lanes (0, 2 ..) and returns the double-wide result.
HWY_API Vec1<int64_t> MulEven(const Vec1<int32_t> a, const Vec1<int32_t> b) {
const int64_t a64 = a.raw;
return Vec1<int64_t>(a64 * b.raw);
}
HWY_API Vec1<uint64_t> MulEven(const Vec1<uint32_t> a, const Vec1<uint32_t> b) {
const uint64_t a64 = a.raw;
return Vec1<uint64_t>(a64 * b.raw);
}
// Approximate reciprocal
HWY_API Vec1<float> ApproximateReciprocal(const Vec1<float> v) {
// Zero inputs are allowed, but callers are responsible for replacing the
// return value with something else (typically using IfThenElse). This check
// avoids a ubsan error. The return value is arbitrary.
if (v.raw == 0.0f) return Vec1<float>(0.0f);
return Vec1<float>(1.0f / v.raw);
}
// Absolute value of difference.
HWY_API Vec1<float> AbsDiff(const Vec1<float> a, const Vec1<float> b) {
return Abs(a - b);
}
// ------------------------------ Floating-point multiply-add variants
template <typename T>
HWY_API Vec1<T> MulAdd(const Vec1<T> mul, const Vec1<T> x, const Vec1<T> add) {
return mul * x + add;
}
template <typename T>
HWY_API Vec1<T> NegMulAdd(const Vec1<T> mul, const Vec1<T> x,
const Vec1<T> add) {
return add - mul * x;
}
template <typename T>
HWY_API Vec1<T> MulSub(const Vec1<T> mul, const Vec1<T> x, const Vec1<T> sub) {
return mul * x - sub;
}
template <typename T>
HWY_API Vec1<T> NegMulSub(const Vec1<T> mul, const Vec1<T> x,
const Vec1<T> sub) {
return Neg(mul) * x - sub;
}
// ------------------------------ Floating-point square root
// Approximate reciprocal square root
HWY_API Vec1<float> ApproximateReciprocalSqrt(const Vec1<float> v) {
float f = v.raw;
const float half = f * 0.5f;
uint32_t bits;
CopySameSize(&f, &bits);
// Initial guess based on log2(f)
bits = 0x5F3759DF - (bits >> 1);
CopySameSize(&bits, &f);
// One Newton-Raphson iteration
return Vec1<float>(f * (1.5f - (half * f * f)));
}
// Square root
HWY_API Vec1<float> Sqrt(const Vec1<float> v) {
#if HWY_COMPILER_GCC && defined(HWY_NO_LIBCXX)
return Vec1<float>(__builtin_sqrt(v.raw));
#else
return Vec1<float>(sqrtf(v.raw));
#endif
}
HWY_API Vec1<double> Sqrt(const Vec1<double> v) {
#if HWY_COMPILER_GCC && defined(HWY_NO_LIBCXX)
return Vec1<float>(__builtin_sqrt(v.raw));
#else
return Vec1<double>(sqrt(v.raw));
#endif
}
// ------------------------------ Floating-point rounding
template <typename T>
HWY_API Vec1<T> Round(const Vec1<T> v) {
using TI = MakeSigned<T>;
if (!(Abs(v).raw < MantissaEnd<T>())) { // Huge or NaN
return v;
}
const T bias = v.raw < T(0.0) ? T(-0.5) : T(0.5);
const TI rounded = static_cast<TI>(v.raw + bias);
if (rounded == 0) return CopySignToAbs(Vec1<T>(0), v);
// Round to even
if ((rounded & 1) && detail::Abs(static_cast<T>(rounded) - v.raw) == T(0.5)) {
return Vec1<T>(static_cast<T>(rounded - (v.raw < T(0) ? -1 : 1)));
}
return Vec1<T>(static_cast<T>(rounded));
}
// Round-to-nearest even.
HWY_API Vec1<int32_t> NearestInt(const Vec1<float> v) {
using T = float;
using TI = int32_t;
const T abs = Abs(v).raw;
const bool is_sign = detail::SignBit(v.raw);
if (!(abs < MantissaEnd<T>())) { // Huge or NaN
// Check if too large to cast or NaN
if (!(abs <= static_cast<T>(LimitsMax<TI>()))) {
return Vec1<TI>(is_sign ? LimitsMin<TI>() : LimitsMax<TI>());
}
return Vec1<int32_t>(static_cast<TI>(v.raw));
}
const T bias = v.raw < T(0.0) ? T(-0.5) : T(0.5);
const TI rounded = static_cast<TI>(v.raw + bias);
if (rounded == 0) return Vec1<int32_t>(0);
// Round to even
if ((rounded & 1) && detail::Abs(static_cast<T>(rounded) - v.raw) == T(0.5)) {
return Vec1<TI>(rounded - (is_sign ? -1 : 1));
}
return Vec1<TI>(rounded);
}
template <typename T>
HWY_API Vec1<T> Trunc(const Vec1<T> v) {
using TI = MakeSigned<T>;
if (!(Abs(v).raw <= MantissaEnd<T>())) { // Huge or NaN
return v;
}
const TI truncated = static_cast<TI>(v.raw);
if (truncated == 0) return CopySignToAbs(Vec1<T>(0), v);
return Vec1<T>(static_cast<T>(truncated));
}
template <typename Float, typename Bits, int kMantissaBits, int kExponentBits,
class V>
V Ceiling(const V v) {
const Bits kExponentMask = (1ull << kExponentBits) - 1;
const Bits kMantissaMask = (1ull << kMantissaBits) - 1;
const Bits kBias = kExponentMask / 2;
Float f = v.raw;
const bool positive = f > Float(0.0);
Bits bits;
CopySameSize(&v, &bits);
const int exponent =
static_cast<int>(((bits >> kMantissaBits) & kExponentMask) - kBias);
// Already an integer.
if (exponent >= kMantissaBits) return v;
// |v| <= 1 => 0 or 1.
if (exponent < 0) return positive ? V(1) : V(-0.0);
const Bits mantissa_mask = kMantissaMask >> exponent;
// Already an integer
if ((bits & mantissa_mask) == 0) return v;
// Clear fractional bits and round up
if (positive) bits += (kMantissaMask + 1) >> exponent;
bits &= ~mantissa_mask;
CopySameSize(&bits, &f);
return V(f);
}
template <typename Float, typename Bits, int kMantissaBits, int kExponentBits,
class V>
V Floor(const V v) {
const Bits kExponentMask = (1ull << kExponentBits) - 1;
const Bits kMantissaMask = (1ull << kMantissaBits) - 1;
const Bits kBias = kExponentMask / 2;
Float f = v.raw;
const bool negative = f < Float(0.0);
Bits bits;
CopySameSize(&v, &bits);
const int exponent =
static_cast<int>(((bits >> kMantissaBits) & kExponentMask) - kBias);
// Already an integer.
if (exponent >= kMantissaBits) return v;
// |v| <= 1 => -1 or 0.
if (exponent < 0) return V(negative ? Float(-1.0) : Float(0.0));
const Bits mantissa_mask = kMantissaMask >> exponent;
// Already an integer
if ((bits & mantissa_mask) == 0) return v;
// Clear fractional bits and round down
if (negative) bits += (kMantissaMask + 1) >> exponent;
bits &= ~mantissa_mask;
CopySameSize(&bits, &f);
return V(f);
}
// Toward +infinity, aka ceiling
HWY_API Vec1<float> Ceil(const Vec1<float> v) {
return Ceiling<float, uint32_t, 23, 8>(v);
}
HWY_API Vec1<double> Ceil(const Vec1<double> v) {
return Ceiling<double, uint64_t, 52, 11>(v);
}
// Toward -infinity, aka floor
HWY_API Vec1<float> Floor(const Vec1<float> v) {
return Floor<float, uint32_t, 23, 8>(v);
}
HWY_API Vec1<double> Floor(const Vec1<double> v) {
return Floor<double, uint64_t, 52, 11>(v);
}
// ================================================== COMPARE
template <typename T>
HWY_API Mask1<T> operator==(const Vec1<T> a, const Vec1<T> b) {
return Mask1<T>::FromBool(a.raw == b.raw);
}
template <typename T>
HWY_API Mask1<T> operator!=(const Vec1<T> a, const Vec1<T> b) {
return Mask1<T>::FromBool(a.raw != b.raw);
}
template <typename T>
HWY_API Mask1<T> TestBit(const Vec1<T> v, const Vec1<T> bit) {
static_assert(!hwy::IsFloat<T>(), "Only integer vectors supported");
return (v & bit) == bit;
}
template <typename T>
HWY_API Mask1<T> operator<(const Vec1<T> a, const Vec1<T> b) {
return Mask1<T>::FromBool(a.raw < b.raw);
}
template <typename T>
HWY_API Mask1<T> operator>(const Vec1<T> a, const Vec1<T> b) {
return Mask1<T>::FromBool(a.raw > b.raw);
}
template <typename T>
HWY_API Mask1<T> operator<=(const Vec1<T> a, const Vec1<T> b) {
return Mask1<T>::FromBool(a.raw <= b.raw);
}
template <typename T>
HWY_API Mask1<T> operator>=(const Vec1<T> a, const Vec1<T> b) {
return Mask1<T>::FromBool(a.raw >= b.raw);
}
// ------------------------------ Floating-point classification (==)
template <typename T>
HWY_API Mask1<T> IsNaN(const Vec1<T> v) {
// std::isnan returns false for 0x7F..FF in clang AVX3 builds, so DIY.
MakeUnsigned<T> bits;
CopySameSize(&v, &bits);
bits += bits;
bits >>= 1; // clear sign bit
// NaN if all exponent bits are set and the mantissa is not zero.
return Mask1<T>::FromBool(bits > ExponentMask<T>());
}
HWY_API Mask1<float> IsInf(const Vec1<float> v) {
const Sisd<float> d;
const RebindToUnsigned<decltype(d)> du;
const Vec1<uint32_t> vu = BitCast(du, v);
// 'Shift left' to clear the sign bit, check for exponent=max and mantissa=0.
return RebindMask(d, (vu + vu) == Set(du, 0xFF000000u));
}
HWY_API Mask1<double> IsInf(const Vec1<double> v) {
const Sisd<double> d;
const RebindToUnsigned<decltype(d)> du;
const Vec1<uint64_t> vu = BitCast(du, v);
// 'Shift left' to clear the sign bit, check for exponent=max and mantissa=0.
return RebindMask(d, (vu + vu) == Set(du, 0xFFE0000000000000ull));
}
HWY_API Mask1<float> IsFinite(const Vec1<float> v) {
const Vec1<uint32_t> vu = BitCast(Sisd<uint32_t>(), v);
// Shift left to clear the sign bit, check whether exponent != max value.
return Mask1<float>::FromBool((vu.raw << 1) < 0xFF000000u);
}
HWY_API Mask1<double> IsFinite(const Vec1<double> v) {
const Vec1<uint64_t> vu = BitCast(Sisd<uint64_t>(), v);
// Shift left to clear the sign bit, check whether exponent != max value.
return Mask1<double>::FromBool((vu.raw << 1) < 0xFFE0000000000000ull);
}
// ================================================== MEMORY
// ------------------------------ Load
template <typename T>
HWY_API Vec1<T> Load(Sisd<T> /* tag */, const T* HWY_RESTRICT aligned) {
T t;
CopySameSize(aligned, &t);
return Vec1<T>(t);
}
template <typename T>
HWY_API Vec1<T> MaskedLoad(Mask1<T> m, Sisd<T> d,
const T* HWY_RESTRICT aligned) {
return IfThenElseZero(m, Load(d, aligned));
}
template <typename T>
HWY_API Vec1<T> LoadU(Sisd<T> d, const T* HWY_RESTRICT p) {
return Load(d, p);
}
// In some use cases, "load single lane" is sufficient; otherwise avoid this.
template <typename T>
HWY_API Vec1<T> LoadDup128(Sisd<T> d, const T* HWY_RESTRICT aligned) {
return Load(d, aligned);
}
// ------------------------------ Store
template <typename T>
HWY_API void Store(const Vec1<T> v, Sisd<T> /* tag */,
T* HWY_RESTRICT aligned) {
CopySameSize(&v.raw, aligned);
}
template <typename T>
HWY_API void StoreU(const Vec1<T> v, Sisd<T> d, T* HWY_RESTRICT p) {
return Store(v, d, p);
}
template <typename T>
HWY_API void BlendedStore(const Vec1<T> v, Mask1<T> m, Sisd<T> d,
T* HWY_RESTRICT p) {
if (!m.bits) return;
StoreU(v, d, p);
}
// ------------------------------ LoadInterleaved2/3/4
// Per-target flag to prevent generic_ops-inl.h from defining StoreInterleaved2.
#ifdef HWY_NATIVE_LOAD_STORE_INTERLEAVED
#undef HWY_NATIVE_LOAD_STORE_INTERLEAVED
#else
#define HWY_NATIVE_LOAD_STORE_INTERLEAVED
#endif
template <typename T>
HWY_API void LoadInterleaved2(Sisd<T> d, const T* HWY_RESTRICT unaligned,
Vec1<T>& v0, Vec1<T>& v1) {
v0 = LoadU(d, unaligned + 0);
v1 = LoadU(d, unaligned + 1);
}
template <typename T>
HWY_API void LoadInterleaved3(Sisd<T> d, const T* HWY_RESTRICT unaligned,
Vec1<T>& v0, Vec1<T>& v1, Vec1<T>& v2) {
v0 = LoadU(d, unaligned + 0);
v1 = LoadU(d, unaligned + 1);
v2 = LoadU(d, unaligned + 2);
}
template <typename T>
HWY_API void LoadInterleaved4(Sisd<T> d, const T* HWY_RESTRICT unaligned,
Vec1<T>& v0, Vec1<T>& v1, Vec1<T>& v2,
Vec1<T>& v3) {
v0 = LoadU(d, unaligned + 0);
v1 = LoadU(d, unaligned + 1);
v2 = LoadU(d, unaligned + 2);
v3 = LoadU(d, unaligned + 3);
}
// ------------------------------ StoreInterleaved2/3/4
template <typename T>
HWY_API void StoreInterleaved2(const Vec1<T> v0, const Vec1<T> v1, Sisd<T> d,
T* HWY_RESTRICT unaligned) {
StoreU(v0, d, unaligned + 0);
StoreU(v1, d, unaligned + 1);
}
template <typename T>
HWY_API void StoreInterleaved3(const Vec1<T> v0, const Vec1<T> v1,
const Vec1<T> v2, Sisd<T> d,
T* HWY_RESTRICT unaligned) {
StoreU(v0, d, unaligned + 0);
StoreU(v1, d, unaligned + 1);
StoreU(v2, d, unaligned + 2);
}
template <typename T>
HWY_API void StoreInterleaved4(const Vec1<T> v0, const Vec1<T> v1,
const Vec1<T> v2, const Vec1<T> v3, Sisd<T> d,
T* HWY_RESTRICT unaligned) {
StoreU(v0, d, unaligned + 0);
StoreU(v1, d, unaligned + 1);
StoreU(v2, d, unaligned + 2);
StoreU(v3, d, unaligned + 3);
}
// ------------------------------ Stream
template <typename T>
HWY_API void Stream(const Vec1<T> v, Sisd<T> d, T* HWY_RESTRICT aligned) {
return Store(v, d, aligned);
}
// ------------------------------ Scatter
template <typename T, typename Offset>
HWY_API void ScatterOffset(Vec1<T> v, Sisd<T> d, T* base,
const Vec1<Offset> offset) {
static_assert(sizeof(T) == sizeof(Offset), "Must match for portability");
uint8_t* const base8 = reinterpret_cast<uint8_t*>(base) + offset.raw;
return Store(v, d, reinterpret_cast<T*>(base8));
}
template <typename T, typename Index>
HWY_API void ScatterIndex(Vec1<T> v, Sisd<T> d, T* HWY_RESTRICT base,
const Vec1<Index> index) {
static_assert(sizeof(T) == sizeof(Index), "Must match for portability");
return Store(v, d, base + index.raw);
}
// ------------------------------ Gather
template <typename T, typename Offset>
HWY_API Vec1<T> GatherOffset(Sisd<T> d, const T* base,
const Vec1<Offset> offset) {
static_assert(sizeof(T) == sizeof(Offset), "Must match for portability");
const intptr_t addr =
reinterpret_cast<intptr_t>(base) + static_cast<intptr_t>(offset.raw);
return Load(d, reinterpret_cast<const T*>(addr));
}
template <typename T, typename Index>
HWY_API Vec1<T> GatherIndex(Sisd<T> d, const T* HWY_RESTRICT base,
const Vec1<Index> index) {
static_assert(sizeof(T) == sizeof(Index), "Must match for portability");
return Load(d, base + index.raw);
}
// ================================================== CONVERT
// ConvertTo and DemoteTo with floating-point input and integer output truncate
// (rounding toward zero).
template <typename FromT, typename ToT>
HWY_API Vec1<ToT> PromoteTo(Sisd<ToT> /* tag */, Vec1<FromT> from) {
static_assert(sizeof(ToT) > sizeof(FromT), "Not promoting");
// For bits Y > X, floatX->floatY and intX->intY are always representable.
return Vec1<ToT>(static_cast<ToT>(from.raw));
}
// MSVC 19.10 cannot deduce the argument type if HWY_IF_FLOAT(FromT) is here,
// so we overload for FromT=double and ToT={float,int32_t}.
HWY_API Vec1<float> DemoteTo(Sisd<float> /* tag */, Vec1<double> from) {
// Prevent ubsan errors when converting float to narrower integer/float
if (IsInf(from).bits ||
Abs(from).raw > static_cast<double>(HighestValue<float>())) {
return Vec1<float>(detail::SignBit(from.raw) ? LowestValue<float>()
: HighestValue<float>());
}
return Vec1<float>(static_cast<float>(from.raw));
}
HWY_API Vec1<int32_t> DemoteTo(Sisd<int32_t> /* tag */, Vec1<double> from) {
// Prevent ubsan errors when converting int32_t to narrower integer/int32_t
if (IsInf(from).bits ||
Abs(from).raw > static_cast<double>(HighestValue<int32_t>())) {
return Vec1<int32_t>(detail::SignBit(from.raw) ? LowestValue<int32_t>()
: HighestValue<int32_t>());
}
return Vec1<int32_t>(static_cast<int32_t>(from.raw));
}
template <typename FromT, typename ToT>
HWY_API Vec1<ToT> DemoteTo(Sisd<ToT> /* tag */, Vec1<FromT> from) {
static_assert(!IsFloat<FromT>(), "FromT=double are handled above");
static_assert(sizeof(ToT) < sizeof(FromT), "Not demoting");
// Int to int: choose closest value in ToT to `from` (avoids UB)
from.raw = HWY_MIN(HWY_MAX(LimitsMin<ToT>(), from.raw), LimitsMax<ToT>());
return Vec1<ToT>(static_cast<ToT>(from.raw));
}
HWY_API Vec1<float> PromoteTo(Sisd<float> /* tag */, const Vec1<float16_t> v) {
uint16_t bits16;
CopySameSize(&v.raw, &bits16);
const uint32_t sign = static_cast<uint32_t>(bits16 >> 15);
const uint32_t biased_exp = (bits16 >> 10) & 0x1F;
const uint32_t mantissa = bits16 & 0x3FF;
// Subnormal or zero
if (biased_exp == 0) {
const float subnormal =
(1.0f / 16384) * (static_cast<float>(mantissa) * (1.0f / 1024));
return Vec1<float>(sign ? -subnormal : subnormal);
}
// Normalized: convert the representation directly (faster than ldexp/tables).
const uint32_t biased_exp32 = biased_exp + (127 - 15);
const uint32_t mantissa32 = mantissa << (23 - 10);
const uint32_t bits32 = (sign << 31) | (biased_exp32 << 23) | mantissa32;
float out;
CopySameSize(&bits32, &out);
return Vec1<float>(out);
}
HWY_API Vec1<float> PromoteTo(Sisd<float> d, const Vec1<bfloat16_t> v) {
return Set(d, F32FromBF16(v.raw));
}
HWY_API Vec1<float16_t> DemoteTo(Sisd<float16_t> /* tag */,
const Vec1<float> v) {
uint32_t bits32;
CopySameSize(&v.raw, &bits32);
const uint32_t sign = bits32 >> 31;
const uint32_t biased_exp32 = (bits32 >> 23) & 0xFF;
const uint32_t mantissa32 = bits32 & 0x7FFFFF;
const int32_t exp = HWY_MIN(static_cast<int32_t>(biased_exp32) - 127, 15);
// Tiny or zero => zero.
Vec1<float16_t> out;
if (exp < -24) {
const uint16_t zero = 0;
CopySameSize(&zero, &out.raw);
return out;
}
uint32_t biased_exp16, mantissa16;
// exp = [-24, -15] => subnormal
if (exp < -14) {
biased_exp16 = 0;
const uint32_t sub_exp = static_cast<uint32_t>(-14 - exp);
HWY_DASSERT(1 <= sub_exp && sub_exp < 11);
mantissa16 = static_cast<uint32_t>((1u << (10 - sub_exp)) +
(mantissa32 >> (13 + sub_exp)));
} else {
// exp = [-14, 15]
biased_exp16 = static_cast<uint32_t>(exp + 15);
HWY_DASSERT(1 <= biased_exp16 && biased_exp16 < 31);
mantissa16 = mantissa32 >> 13;
}
HWY_DASSERT(mantissa16 < 1024);
const uint32_t bits16 = (sign << 15) | (biased_exp16 << 10) | mantissa16;
HWY_DASSERT(bits16 < 0x10000);
const uint16_t narrowed = static_cast<uint16_t>(bits16); // big-endian safe
CopySameSize(&narrowed, &out.raw);
return out;
}
HWY_API Vec1<bfloat16_t> DemoteTo(Sisd<bfloat16_t> d, const Vec1<float> v) {
return Set(d, BF16FromF32(v.raw));
}
template <typename FromT, typename ToT, HWY_IF_FLOAT(FromT)>
HWY_API Vec1<ToT> ConvertTo(Sisd<ToT> /* tag */, Vec1<FromT> from) {
static_assert(sizeof(ToT) == sizeof(FromT), "Should have same size");
// float## -> int##: return closest representable value. We cannot exactly
// represent LimitsMax<ToT> in FromT, so use double.
const double f = static_cast<double>(from.raw);
if (IsInf(from).bits ||
Abs(Vec1<double>(f)).raw > static_cast<double>(LimitsMax<ToT>())) {
return Vec1<ToT>(detail::SignBit(from.raw) ? LimitsMin<ToT>()
: LimitsMax<ToT>());
}
return Vec1<ToT>(static_cast<ToT>(from.raw));
}
template <typename FromT, typename ToT, HWY_IF_NOT_FLOAT(FromT)>
HWY_API Vec1<ToT> ConvertTo(Sisd<ToT> /* tag */, Vec1<FromT> from) {
static_assert(sizeof(ToT) == sizeof(FromT), "Should have same size");
// int## -> float##: no check needed
return Vec1<ToT>(static_cast<ToT>(from.raw));
}
HWY_API Vec1<uint8_t> U8FromU32(const Vec1<uint32_t> v) {
return DemoteTo(Sisd<uint8_t>(), v);
}
// ------------------------------ Truncations
HWY_API Vec1<uint8_t> TruncateTo(Sisd<uint8_t> /* tag */,
const Vec1<uint64_t> v) {
return Vec1<uint8_t>{static_cast<uint8_t>(v.raw & 0xFF)};
}
HWY_API Vec1<uint16_t> TruncateTo(Sisd<uint16_t> /* tag */,
const Vec1<uint64_t> v) {
return Vec1<uint16_t>{static_cast<uint16_t>(v.raw & 0xFFFF)};
}
HWY_API Vec1<uint32_t> TruncateTo(Sisd<uint32_t> /* tag */,
const Vec1<uint64_t> v) {
return Vec1<uint32_t>{static_cast<uint32_t>(v.raw & 0xFFFFFFFFu)};
}
HWY_API Vec1<uint8_t> TruncateTo(Sisd<uint8_t> /* tag */,
const Vec1<uint32_t> v) {
return Vec1<uint8_t>{static_cast<uint8_t>(v.raw & 0xFF)};
}
HWY_API Vec1<uint16_t> TruncateTo(Sisd<uint16_t> /* tag */,
const Vec1<uint32_t> v) {
return Vec1<uint16_t>{static_cast<uint16_t>(v.raw & 0xFFFF)};
}
HWY_API Vec1<uint8_t> TruncateTo(Sisd<uint8_t> /* tag */,
const Vec1<uint16_t> v) {
return Vec1<uint8_t>{static_cast<uint8_t>(v.raw & 0xFF)};
}
// ================================================== COMBINE
// UpperHalf, ZeroExtendVector, Combine, Concat* are unsupported.
template <typename T>
HWY_API Vec1<T> LowerHalf(Vec1<T> v) {
return v;
}
template <typename T>
HWY_API Vec1<T> LowerHalf(Sisd<T> /* tag */, Vec1<T> v) {
return v;
}
// ================================================== SWIZZLE
template <typename T>
HWY_API T GetLane(const Vec1<T> v) {
return v.raw;
}
template <typename T>
HWY_API T ExtractLane(const Vec1<T> v, size_t i) {
HWY_DASSERT(i == 0);
(void)i;
return v.raw;
}
template <typename T>
HWY_API Vec1<T> InsertLane(Vec1<T> v, size_t i, T t) {
HWY_DASSERT(i == 0);
(void)i;
v.raw = t;
return v;
}
template <typename T>
HWY_API Vec1<T> DupEven(Vec1<T> v) {
return v;
}
// DupOdd is unsupported.
template <typename T>
HWY_API Vec1<T> OddEven(Vec1<T> /* odd */, Vec1<T> even) {
return even;
}
template <typename T>
HWY_API Vec1<T> OddEvenBlocks(Vec1<T> /* odd */, Vec1<T> even) {
return even;
}
// ------------------------------ SwapAdjacentBlocks
template <typename T>
HWY_API Vec1<T> SwapAdjacentBlocks(Vec1<T> v) {
return v;
}
// ------------------------------ TableLookupLanes
// Returned by SetTableIndices for use by TableLookupLanes.
template <typename T>
struct Indices1 {
MakeSigned<T> raw;
};
template <typename T, typename TI>
HWY_API Indices1<T> IndicesFromVec(Sisd<T>, Vec1<TI> vec) {
static_assert(sizeof(T) == sizeof(TI), "Index size must match lane size");
HWY_DASSERT(vec.raw == 0);
return Indices1<T>{vec.raw};
}
template <typename T, typename TI>
HWY_API Indices1<T> SetTableIndices(Sisd<T> d, const TI* idx) {
return IndicesFromVec(d, LoadU(Sisd<TI>(), idx));
}
template <typename T>
HWY_API Vec1<T> TableLookupLanes(const Vec1<T> v, const Indices1<T> /* idx */) {
return v;
}
// ------------------------------ ReverseBlocks
// Single block: no change
template <typename T>
HWY_API Vec1<T> ReverseBlocks(Sisd<T> /* tag */, const Vec1<T> v) {
return v;
}
// ------------------------------ Reverse
template <typename T>
HWY_API Vec1<T> Reverse(Sisd<T> /* tag */, const Vec1<T> v) {
return v;
}
// Must not be called:
template <typename T>
HWY_API Vec1<T> Reverse2(Sisd<T> /* tag */, const Vec1<T> v) {
return v;
}
template <typename T>
HWY_API Vec1<T> Reverse4(Sisd<T> /* tag */, const Vec1<T> v) {
return v;
}
template <typename T>
HWY_API Vec1<T> Reverse8(Sisd<T> /* tag */, const Vec1<T> v) {
return v;
}
// ================================================== BLOCKWISE
// Shift*Bytes, CombineShiftRightBytes, Interleave*, Shuffle* are unsupported.
// ------------------------------ Broadcast/splat any lane
template <int kLane, typename T>
HWY_API Vec1<T> Broadcast(const Vec1<T> v) {
static_assert(kLane == 0, "Scalar only has one lane");
return v;
}
// ------------------------------ TableLookupBytes, TableLookupBytesOr0
template <typename T, typename TI>
HWY_API Vec1<TI> TableLookupBytes(const Vec1<T> in, const Vec1<TI> indices) {
uint8_t in_bytes[sizeof(T)];
uint8_t idx_bytes[sizeof(T)];
uint8_t out_bytes[sizeof(T)];
CopyBytes<sizeof(T)>(&in, &in_bytes); // copy to bytes
CopyBytes<sizeof(T)>(&indices, &idx_bytes);
for (size_t i = 0; i < sizeof(T); ++i) {
out_bytes[i] = in_bytes[idx_bytes[i]];
}
TI out;
CopyBytes<sizeof(TI)>(&out_bytes, &out);
return Vec1<TI>{out};
}
template <typename T, typename TI>
HWY_API Vec1<TI> TableLookupBytesOr0(const Vec1<T> in, const Vec1<TI> indices) {
uint8_t in_bytes[sizeof(T)];
uint8_t idx_bytes[sizeof(T)];
uint8_t out_bytes[sizeof(T)];
CopyBytes<sizeof(T)>(&in, &in_bytes); // copy to bytes
CopyBytes<sizeof(T)>(&indices, &idx_bytes);
for (size_t i = 0; i < sizeof(T); ++i) {
out_bytes[i] = idx_bytes[i] & 0x80 ? 0 : in_bytes[idx_bytes[i]];
}
TI out;
CopyBytes<sizeof(TI)>(&out_bytes, &out);
return Vec1<TI>{out};
}
// ------------------------------ ZipLower
HWY_API Vec1<uint16_t> ZipLower(const Vec1<uint8_t> a, const Vec1<uint8_t> b) {
return Vec1<uint16_t>(static_cast<uint16_t>((uint32_t{b.raw} << 8) + a.raw));
}
HWY_API Vec1<uint32_t> ZipLower(const Vec1<uint16_t> a,
const Vec1<uint16_t> b) {
return Vec1<uint32_t>((uint32_t{b.raw} << 16) + a.raw);
}
HWY_API Vec1<uint64_t> ZipLower(const Vec1<uint32_t> a,
const Vec1<uint32_t> b) {
return Vec1<uint64_t>((uint64_t{b.raw} << 32) + a.raw);
}
HWY_API Vec1<int16_t> ZipLower(const Vec1<int8_t> a, const Vec1<int8_t> b) {
return Vec1<int16_t>(static_cast<int16_t>((int32_t{b.raw} << 8) + a.raw));
}
HWY_API Vec1<int32_t> ZipLower(const Vec1<int16_t> a, const Vec1<int16_t> b) {
return Vec1<int32_t>((int32_t{b.raw} << 16) + a.raw);
}
HWY_API Vec1<int64_t> ZipLower(const Vec1<int32_t> a, const Vec1<int32_t> b) {
return Vec1<int64_t>((int64_t{b.raw} << 32) + a.raw);
}
template <typename T, typename TW = MakeWide<T>, class VW = Vec1<TW>>
HWY_API VW ZipLower(Sisd<TW> /* tag */, Vec1<T> a, Vec1<T> b) {
return VW(static_cast<TW>((TW{b.raw} << (sizeof(T) * 8)) + a.raw));
}
// ================================================== MASK
template <typename T>
HWY_API bool AllFalse(Sisd<T> /* tag */, const Mask1<T> mask) {
return mask.bits == 0;
}
template <typename T>
HWY_API bool AllTrue(Sisd<T> /* tag */, const Mask1<T> mask) {
return mask.bits != 0;
}
// `p` points to at least 8 readable bytes, not all of which need be valid.
template <typename T>
HWY_API Mask1<T> LoadMaskBits(Sisd<T> /* tag */,
const uint8_t* HWY_RESTRICT bits) {
return Mask1<T>::FromBool((bits[0] & 1) != 0);
}
// `p` points to at least 8 writable bytes.
template <typename T>
HWY_API size_t StoreMaskBits(Sisd<T> d, const Mask1<T> mask, uint8_t* bits) {
*bits = AllTrue(d, mask);
return 1;
}
template <typename T>
HWY_API size_t CountTrue(Sisd<T> /* tag */, const Mask1<T> mask) {
return mask.bits == 0 ? 0 : 1;
}
template <typename T>
HWY_API intptr_t FindFirstTrue(Sisd<T> /* tag */, const Mask1<T> mask) {
return mask.bits == 0 ? -1 : 0;
}
template <typename T>
HWY_API size_t FindKnownFirstTrue(Sisd<T> /* tag */, const Mask1<T> /* m */) {
return 0; // There is only one lane and we know it is true.
}
// ------------------------------ Compress, CompressBits
template <typename T>
struct CompressIsPartition {
enum { value = 1 };
};
template <typename T>
HWY_API Vec1<T> Compress(Vec1<T> v, const Mask1<T> /* mask */) {
// A single lane is already partitioned by definition.
return v;
}
template <typename T>
HWY_API Vec1<T> CompressNot(Vec1<T> v, const Mask1<T> /* mask */) {
// A single lane is already partitioned by definition.
return v;
}
// ------------------------------ CompressStore
template <typename T>
HWY_API size_t CompressStore(Vec1<T> v, const Mask1<T> mask, Sisd<T> d,
T* HWY_RESTRICT unaligned) {
StoreU(Compress(v, mask), d, unaligned);
return CountTrue(d, mask);
}
// ------------------------------ CompressBlendedStore
template <typename T>
HWY_API size_t CompressBlendedStore(Vec1<T> v, const Mask1<T> mask, Sisd<T> d,
T* HWY_RESTRICT unaligned) {
if (!mask.bits) return 0;
StoreU(v, d, unaligned);
return 1;
}
// ------------------------------ CompressBits
template <typename T>
HWY_API Vec1<T> CompressBits(Vec1<T> v, const uint8_t* HWY_RESTRICT /*bits*/) {
return v;
}
// ------------------------------ CompressBitsStore
template <typename T>
HWY_API size_t CompressBitsStore(Vec1<T> v, const uint8_t* HWY_RESTRICT bits,
Sisd<T> d, T* HWY_RESTRICT unaligned) {
const Mask1<T> mask = LoadMaskBits(d, bits);
StoreU(Compress(v, mask), d, unaligned);
return CountTrue(d, mask);
}
// ------------------------------ ReorderWidenMulAccumulate (MulAdd, ZipLower)
HWY_API Vec1<float> ReorderWidenMulAccumulate(Sisd<float> /* tag */,
Vec1<bfloat16_t> a,
Vec1<bfloat16_t> b,
const Vec1<float> sum0,
Vec1<float>& /* sum1 */) {
return MulAdd(Vec1<float>(F32FromBF16(a.raw)),
Vec1<float>(F32FromBF16(b.raw)), sum0);
}
HWY_API Vec1<int32_t> ReorderWidenMulAccumulate(Sisd<int32_t> /* tag */,
Vec1<int16_t> a,
Vec1<int16_t> b,
const Vec1<int32_t> sum0,
Vec1<int32_t>& /* sum1 */) {
return Vec1<int32_t>(a.raw * b.raw + sum0.raw);
}
// ------------------------------ RearrangeToOddPlusEven
template <typename TW>
HWY_API Vec1<TW> RearrangeToOddPlusEven(const Vec1<TW> sum0,
Vec1<TW> /* sum1 */) {
return sum0; // invariant already holds
}
// ================================================== REDUCTIONS
// Sum of all lanes, i.e. the only one.
template <typename T>
HWY_API Vec1<T> SumOfLanes(Sisd<T> /* tag */, const Vec1<T> v) {
return v;
}
template <typename T>
HWY_API Vec1<T> MinOfLanes(Sisd<T> /* tag */, const Vec1<T> v) {
return v;
}
template <typename T>
HWY_API Vec1<T> MaxOfLanes(Sisd<T> /* tag */, const Vec1<T> v) {
return v;
}
// NOLINTNEXTLINE(google-readability-namespace-comments)
} // namespace HWY_NAMESPACE
} // namespace hwy
HWY_AFTER_NAMESPACE();
base.h
HWY_MAX
#define HWY_MAX(a, b)
Definition base.h:135
HWY_RESTRICT
#define HWY_RESTRICT
Definition base.h:64
HWY_API
#define HWY_API
Definition base.h:129
HWY_MIN
#define HWY_MIN(a, b)
Definition base.h:134
HWY_INLINE
#define HWY_INLINE
Definition base.h:70
HWY_DASSERT
#define HWY_DASSERT(condition)
Definition base.h:238
hwy::HWY_NAMESPACE::Mask1
Definition scalar-inl.h:71
hwy::HWY_NAMESPACE::Mask1::bits
Raw bits
Definition scalar-inl.h:81
hwy::HWY_NAMESPACE::Mask1::Raw
hwy::MakeUnsigned< T > Raw
Definition scalar-inl.h:72
hwy::HWY_NAMESPACE::Mask1::FromBool
static HWY_INLINE Mask1< T > FromBool(bool b)
Definition scalar-inl.h:75
hwy::HWY_NAMESPACE::detail::Abs
HWY_INLINE Vec128< T, N > Abs(SignedTag, Vec128< T, N > a)
Definition emu128-inl.h:633
hwy::HWY_NAMESPACE::detail::Or
HWY_INLINE Mask128< T, N > Or(hwy::SizeTag< 1 >, const Mask128< T, N > a, const Mask128< T, N > b)
Definition x86_128-inl.h:889
hwy::HWY_NAMESPACE::detail::SignBit
static bool SignBit(float f)
Definition scalar-inl.h:601
hwy::HWY_NAMESPACE::d
d
Definition rvv-inl.h:1998
hwy::HWY_NAMESPACE::ShiftLeftSame
HWY_API Vec128< T, N > ShiftLeftSame(const Vec128< T, N > v, int bits)
Definition arm_neon-inl.h:1631
hwy::HWY_NAMESPACE::AverageRound
HWY_API Vec128< T, N > AverageRound(Vec128< T, N > a, const Vec128< T, N > b)
Definition emu128-inl.h:619
hwy::HWY_NAMESPACE::CopySign
HWY_API Vec128< T, N > CopySign(const Vec128< T, N > magn, const Vec128< T, N > sign)
Definition arm_neon-inl.h:2190
hwy::HWY_NAMESPACE::OddEvenBlocks
HWY_API Vec128< T, N > OddEvenBlocks(Vec128< T, N >, Vec128< T, N > even)
Definition arm_neon-inl.h:4697
hwy::HWY_NAMESPACE::operator>
HWY_API Mask128< T, N > operator>(Vec128< T, N > a, Vec128< T, N > b)
Definition arm_neon-inl.h:2445
hwy::HWY_NAMESPACE::operator-
HWY_API Vec128< T, N > operator-(Vec128< T, N > a, const Vec128< T, N > b)
Definition emu128-inl.h:576
hwy::HWY_NAMESPACE::RebindMask
HWY_API Mask128< TTo, N > RebindMask(Simd< TTo, N, 0 > dto, Mask128< TFrom, N > m)
Definition arm_neon-inl.h:2230
hwy::HWY_NAMESPACE::operator==
HWY_API Mask128< T, N > operator==(const Vec128< T, N > a, const Vec128< T, N > b)
Definition emu128-inl.h:1139
hwy::HWY_NAMESPACE::ZipLower
HWY_API VFromD< DW > ZipLower(V a, V b)
Definition arm_neon-inl.h:4272
hwy::HWY_NAMESPACE::AllTrue
HWY_API bool AllTrue(const Full128< T > d, const Mask128< T > m)
Definition arm_neon-inl.h:5716
hwy::HWY_NAMESPACE::LoadInterleaved2
HWY_API void LoadInterleaved2(Simd< T, N, 0 >, const T *HWY_RESTRICT unaligned, Vec128< T, N > &v0, Vec128< T, N > &v1)
Definition arm_neon-inl.h:6349
hwy::HWY_NAMESPACE::StoreInterleaved4
HWY_API void StoreInterleaved4(const Vec128< T, N > v0, const Vec128< T, N > v1, const Vec128< T, N > v2, const Vec128< T, N > v3, Simd< T, N, 0 >, T *HWY_RESTRICT unaligned)
Definition arm_neon-inl.h:6584
hwy::HWY_NAMESPACE::MulHigh
HWY_API Vec128< int16_t > MulHigh(const Vec128< int16_t > a, const Vec128< int16_t > b)
Definition arm_neon-inl.h:1684
hwy::HWY_NAMESPACE::Round
HWY_API Vec128< float, N > Round(const Vec128< float, N > v)
Definition arm_neon-inl.h:3436
hwy::HWY_NAMESPACE::IsNaN
HWY_API Mask128< T, N > IsNaN(const Vec128< T, N > v)
Definition arm_neon-inl.h:3506
hwy::HWY_NAMESPACE::FindFirstTrue
HWY_API intptr_t FindFirstTrue(const Simd< T, N, 0 > d, const Mask128< T, N > mask)
Definition arm_neon-inl.h:5691
hwy::HWY_NAMESPACE::FirstN
HWY_API Mask128< T, N > FirstN(const Simd< T, N, 0 > d, size_t num)
Definition arm_neon-inl.h:2456
hwy::HWY_NAMESPACE::StoreMaskBits
HWY_API size_t StoreMaskBits(Simd< T, N, 0 >, const Mask128< T, N > mask, uint8_t *bits)
Definition arm_neon-inl.h:5701
hwy::HWY_NAMESPACE::MulAdd
HWY_API Vec128< float, N > MulAdd(const Vec128< float, N > mul, const Vec128< float, N > x, const Vec128< float, N > add)
Definition arm_neon-inl.h:1799
hwy::HWY_NAMESPACE::Stream
HWY_API void Stream(const Vec128< T, N > v, Simd< T, N, 0 > d, T *HWY_RESTRICT aligned)
Definition arm_neon-inl.h:2955
hwy::HWY_NAMESPACE::Xor3
HWY_API Vec128< T, N > Xor3(Vec128< T, N > x1, Vec128< T, N > x2, Vec128< T, N > x3)
Definition arm_neon-inl.h:2025
hwy::HWY_NAMESPACE::And
HWY_API Vec128< T, N > And(const Vec128< T, N > a, const Vec128< T, N > b)
Definition arm_neon-inl.h:1949
hwy::HWY_NAMESPACE::SumOfLanes
HWY_API Vec128< T, N > SumOfLanes(Simd< T, N, 0 >, const Vec128< T, N > v)
Definition arm_neon-inl.h:5334
hwy::HWY_NAMESPACE::BroadcastSignBit
HWY_API Vec128< T, N > BroadcastSignBit(const Vec128< T, N > v)
Definition arm_neon-inl.h:2207
hwy::HWY_NAMESPACE::TruncateTo
HWY_API Vec128< To, 1 > TruncateTo(Simd< To, 1, 0 >, const Vec128< From, 1 > v)
Definition arm_neon-inl.h:4806
hwy::HWY_NAMESPACE::Min
HWY_API Vec128< uint64_t, N > Min(const Vec128< uint64_t, N > a, const Vec128< uint64_t, N > b)
Definition arm_neon-inl.h:2517
hwy::HWY_NAMESPACE::PopulationCount
HWY_API Vec128< T, N > PopulationCount(Vec128< T, N > v)
Definition arm_neon-inl.h:2137
hwy::HWY_NAMESPACE::Max
HWY_API Vec128< uint64_t, N > Max(const Vec128< uint64_t, N > a, const Vec128< uint64_t, N > b)
Definition arm_neon-inl.h:2555
hwy::HWY_NAMESPACE::MaskFromVec
HWY_API Mask128< T, N > MaskFromVec(const Vec128< T, N > v)
Definition arm_neon-inl.h:2217
hwy::HWY_NAMESPACE::Neg
HWY_API Vec64< int64_t > Neg(const Vec64< int64_t > v)
Definition arm_neon-inl.h:1405
hwy::HWY_NAMESPACE::RebindToUnsigned
Rebind< MakeUnsigned< TFromD< D > >, D > RebindToUnsigned
Definition ops/shared-inl.h:212
hwy::HWY_NAMESPACE::SaturatedAdd
HWY_API Vec128< T, N > SaturatedAdd(Vec128< T, N > a, const Vec128< T, N > b)
Definition emu128-inl.h:597
hwy::HWY_NAMESPACE::GatherIndex
HWY_API Vec128< T, N > GatherIndex(const Simd< T, N, 0 > d, const T *HWY_RESTRICT base, const Vec128< Index, N > index)
Definition arm_neon-inl.h:5037
hwy::HWY_NAMESPACE::Not
HWY_API Vec128< T > Not(const Vec128< T > v)
Definition arm_neon-inl.h:1931
hwy::HWY_NAMESPACE::IsInf
HWY_API Mask128< T, N > IsInf(const Vec128< T, N > v)
Definition arm_neon-inl.h:3511
hwy::HWY_NAMESPACE::LowerHalf
HWY_API Vec128< T, N/2 > LowerHalf(const Vec128< T, N > v)
Definition arm_neon-inl.h:3540
hwy::HWY_NAMESPACE::operator&
HWY_API Vec128< T, N > operator&(const Vec128< T, N > a, const Vec128< T, N > b)
Definition arm_neon-inl.h:2055
hwy::HWY_NAMESPACE::operator|
HWY_API Vec128< T, N > operator|(const Vec128< T, N > a, const Vec128< T, N > b)
Definition arm_neon-inl.h:2060
hwy::HWY_NAMESPACE::MulEven
HWY_API Vec128< int64_t > MulEven(Vec128< int32_t > a, Vec128< int32_t > b)
Definition arm_neon-inl.h:4872
hwy::HWY_NAMESPACE::CompressNot
HWY_API Vec128< T, 1 > CompressNot(Vec128< T, 1 > v, Mask128< T, 1 >)
Definition arm_neon-inl.h:6198
hwy::HWY_NAMESPACE::MaskedLoad
HWY_API Vec128< T, N > MaskedLoad(Mask128< T, N > m, Simd< T, N, 0 > d, const T *HWY_RESTRICT aligned)
Definition arm_neon-inl.h:2758
hwy::HWY_NAMESPACE::operator<
HWY_API Mask128< T, N > operator<(const Vec128< T, N > a, const Vec128< T, N > b)
Definition emu128-inl.h:1163
hwy::HWY_NAMESPACE::ReorderWidenMulAccumulate
HWY_API Vec128< float, N > ReorderWidenMulAccumulate(Simd< float, N, 0 > df32, Vec128< bfloat16_t, 2 *N > a, Vec128< bfloat16_t, 2 *N > b, const Vec128< float, N > sum0, Vec128< float, N > &sum1)
Definition arm_neon-inl.h:4288
hwy::HWY_NAMESPACE::IfVecThenElse
HWY_API Vec128< T, N > IfVecThenElse(Vec128< T, N > mask, Vec128< T, N > yes, Vec128< T, N > no)
Definition arm_neon-inl.h:2047
hwy::HWY_NAMESPACE::operator^
HWY_API Vec128< T, N > operator^(const Vec128< T, N > a, const Vec128< T, N > b)
Definition arm_neon-inl.h:2065
hwy::HWY_NAMESPACE::BlendedStore
HWY_API void BlendedStore(Vec128< T, N > v, Mask128< T, N > m, Simd< T, N, 0 > d, T *HWY_RESTRICT p)
Definition arm_neon-inl.h:2941
hwy::HWY_NAMESPACE::CountTrue
HWY_API size_t CountTrue(Full128< T >, const Mask128< T > mask)
Definition arm_neon-inl.h:5671
hwy::HWY_NAMESPACE::VecFromMask
HWY_API Vec128< T, N > VecFromMask(Simd< T, N, 0 > d, const Mask128< T, N > v)
Definition arm_neon-inl.h:2223
hwy::HWY_NAMESPACE::DupEven
HWY_API Vec128< T, N > DupEven(Vec128< T, N > v)
Definition arm_neon-inl.h:4646
hwy::HWY_NAMESPACE::IfThenElseZero
HWY_API Vec128< T, N > IfThenElseZero(const Mask128< T, N > mask, const Vec128< T, N > yes)
Definition arm_neon-inl.h:2253
hwy::HWY_NAMESPACE::TestBit
HWY_API Mask128< uint64_t, N > TestBit(Vec128< uint64_t, N > v, Vec128< uint64_t, N > bit)
Definition arm_neon-inl.h:2477
hwy::HWY_NAMESPACE::Load
HWY_API Vec128< T, N > Load(Simd< T, N, 0 > d, const T *HWY_RESTRICT p)
Definition arm_neon-inl.h:2753
hwy::HWY_NAMESPACE::TableLookupBytes
HWY_API Vec128< TI > TableLookupBytes(const Vec128< T > bytes, const Vec128< TI > from)
Definition arm_neon-inl.h:4922
hwy::HWY_NAMESPACE::IfThenElse
HWY_API Vec128< T, N > IfThenElse(const Mask128< T, N > mask, const Vec128< T, N > yes, const Vec128< T, N > no)
Definition emu128-inl.h:303
hwy::HWY_NAMESPACE::TableLookupLanes
HWY_API Vec128< T, N > TableLookupLanes(Vec128< T, N > v, Indices128< T, N > idx)
Definition arm_neon-inl.h:4019
hwy::HWY_NAMESPACE::Xor
HWY_API Vec128< T, N > Xor(const Vec128< T, N > a, const Vec128< T, N > b)
Definition arm_neon-inl.h:1998
hwy::HWY_NAMESPACE::Floor
HWY_API Vec128< float, N > Floor(const Vec128< float, N > v)
Definition arm_neon-inl.h:3467
hwy::HWY_NAMESPACE::MulSub
HWY_API Vec128< float, N > MulSub(const Vec128< float, N > mul, const Vec128< float, N > x, const Vec128< float, N > sub)
Definition arm_neon-inl.h:1853
hwy::HWY_NAMESPACE::CopySignToAbs
HWY_API Vec128< T, N > CopySignToAbs(const Vec128< T, N > abs, const Vec128< T, N > sign)
Definition arm_neon-inl.h:2198
hwy::HWY_NAMESPACE::StoreU
HWY_API void StoreU(const Vec128< uint8_t > v, Full128< uint8_t >, uint8_t *HWY_RESTRICT unaligned)
Definition arm_neon-inl.h:2772
hwy::HWY_NAMESPACE::Ceil
HWY_API Vec128< float, N > Ceil(const Vec128< float, N > v)
Definition arm_neon-inl.h:3453
hwy::HWY_NAMESPACE::IndicesFromVec
HWY_API Indices128< T, N > IndicesFromVec(Simd< T, N, 0 > d, Vec128< TI, N > vec)
Definition arm_neon-inl.h:3973
hwy::HWY_NAMESPACE::SwapAdjacentBlocks
HWY_API Vec128< T, N > SwapAdjacentBlocks(Vec128< T, N > v)
Definition arm_neon-inl.h:4704
hwy::HWY_NAMESPACE::Reverse2
HWY_API Vec128< T, N > Reverse2(Simd< T, N, 0 > d, const Vec128< T, N > v)
Definition arm_neon-inl.h:4061
hwy::HWY_NAMESPACE::Set
svuint16_t Set(Simd< bfloat16_t, N, kPow2 > d, bfloat16_t arg)
Definition arm_sve-inl.h:322
hwy::HWY_NAMESPACE::Reverse8
HWY_API Vec128< T, N > Reverse8(Simd< T, N, 0 > d, const Vec128< T, N > v)
Definition arm_neon-inl.h:4113
hwy::HWY_NAMESPACE::SignBit
HWY_API Vec< D > SignBit(D d)
Definition generic_ops-inl.h:69
hwy::HWY_NAMESPACE::MaxOfLanes
HWY_API Vec128< T, N > MaxOfLanes(Simd< T, N, 0 >, const Vec128< T, N > v)
Definition arm_neon-inl.h:5342
hwy::HWY_NAMESPACE::Iota
Vec128< T, N > Iota(const Simd< T, N, 0 > d, const T2 first)
Definition arm_neon-inl.h:1049
hwy::HWY_NAMESPACE::ExclusiveNeither
HWY_API Mask128< T, N > ExclusiveNeither(const Mask128< T, N > a, Mask128< T, N > b)
Definition arm_neon-inl.h:2314
hwy::HWY_NAMESPACE::TFromV
typename V::PrivateT TFromV
Definition arm_neon-inl.h:845
hwy::HWY_NAMESPACE::CompressBits
HWY_INLINE Vec128< T, N > CompressBits(Vec128< T, N > v, const uint8_t *HWY_RESTRICT bits)
Definition arm_neon-inl.h:6234
hwy::HWY_NAMESPACE::LoadMaskBits
HWY_API Mask128< T, N > LoadMaskBits(Simd< T, N, 0 > d, const uint8_t *HWY_RESTRICT bits)
Definition arm_neon-inl.h:5407
hwy::HWY_NAMESPACE::ZeroIfNegative
HWY_API Vec128< T, N > ZeroIfNegative(Vec128< T, N > v)
Definition arm_neon-inl.h:2277
hwy::HWY_NAMESPACE::operator/
HWY_API Vec128< float, N > operator/(const Vec128< float, N > a, const Vec128< float, N > b)
Definition arm_neon-inl.h:1761
hwy::HWY_NAMESPACE::DemoteTo
HWY_API Vec64< uint16_t > DemoteTo(Full64< uint16_t >, const Vec128< int32_t > v)
Definition arm_neon-inl.h:3145
hwy::HWY_NAMESPACE::LoadU
HWY_API Vec128< uint8_t > LoadU(Full128< uint8_t >, const uint8_t *HWY_RESTRICT unaligned)
Definition arm_neon-inl.h:2591
hwy::HWY_NAMESPACE::OrAnd
HWY_API Vec128< T, N > OrAnd(Vec128< T, N > o, Vec128< T, N > a1, Vec128< T, N > a2)
Definition arm_neon-inl.h:2040
hwy::HWY_NAMESPACE::IfNegativeThenElse
HWY_API Vec128< T, N > IfNegativeThenElse(Vec128< T, N > v, Vec128< T, N > yes, Vec128< T, N > no)
Definition arm_neon-inl.h:2266
hwy::HWY_NAMESPACE::operator<<
HWY_API Vec128< uint8_t > operator<<(const Vec128< uint8_t > v, const Vec128< uint8_t > bits)
Definition arm_neon-inl.h:1462
hwy::HWY_NAMESPACE::operator*
HWY_API Vec128< uint16_t > operator*(const Vec128< uint16_t > a, const Vec128< uint16_t > b)
Definition arm_neon-inl.h:1642
hwy::HWY_NAMESPACE::BitCast
HWY_API Vec128< T, N > BitCast(Simd< T, N, 0 > d, Vec128< FromT, N *sizeof(T)/sizeof(FromT)> v)
Definition arm_neon-inl.h:997
hwy::HWY_NAMESPACE::AllFalse
HWY_API bool AllFalse(const Simd< T, N, 0 > d, const Mask128< T, N > m)
Definition arm_neon-inl.h:5710
hwy::HWY_NAMESPACE::ExtractLane
HWY_API T ExtractLane(const Vec128< T, 1 > v, size_t i)
Definition arm_neon-inl.h:1085
hwy::HWY_NAMESPACE::ScatterOffset
HWY_API void ScatterOffset(Vec128< T, N > v, Simd< T, N, 0 > d, T *HWY_RESTRICT base, const Vec128< Offset, N > offset)
Definition arm_neon-inl.h:4984
hwy::HWY_NAMESPACE::Undefined
HWY_API Vec128< T, N > Undefined(Simd< T, N, 0 >)
Definition arm_neon-inl.h:1040
hwy::HWY_NAMESPACE::ShiftRight
HWY_API Vec128< T, N > ShiftRight(Vec128< T, N > v)
Definition emu128-inl.h:386
hwy::HWY_NAMESPACE::RearrangeToOddPlusEven
HWY_API Vec128< float, N > RearrangeToOddPlusEven(const Vec128< float, N > sum0, const Vec128< float, N > sum1)
Definition arm_neon-inl.h:4412
hwy::HWY_NAMESPACE::Zero
HWY_API Vec128< T, N > Zero(Simd< T, N, 0 > d)
Definition arm_neon-inl.h:1020
hwy::HWY_NAMESPACE::operator>=
HWY_API Mask128< T, N > operator>=(Vec128< T, N > a, Vec128< T, N > b)
Definition arm_neon-inl.h:2449
hwy::HWY_NAMESPACE::ShiftRightSame
HWY_API Vec128< T, N > ShiftRightSame(const Vec128< T, N > v, int bits)
Definition arm_neon-inl.h:1635
hwy::HWY_NAMESPACE::GatherOffset
HWY_API Vec128< T, N > GatherOffset(const Simd< T, N, 0 > d, const T *HWY_RESTRICT base, const Vec128< Offset, N > offset)
Definition arm_neon-inl.h:5020
hwy::HWY_NAMESPACE::LoadInterleaved3
HWY_API void LoadInterleaved3(Simd< T, N, 0 >, const T *HWY_RESTRICT unaligned, Vec128< T, N > &v0, Vec128< T, N > &v1, Vec128< T, N > &v2)
Definition arm_neon-inl.h:6387
hwy::HWY_NAMESPACE::IfThenZeroElse
HWY_API Vec128< T, N > IfThenZeroElse(const Mask128< T, N > mask, const Vec128< T, N > no)
Definition arm_neon-inl.h:2260
hwy::HWY_NAMESPACE::Ceiling
V Ceiling(const V v)
Definition scalar-inl.h:825
hwy::HWY_NAMESPACE::operator!=
HWY_API Mask128< T, N > operator!=(const Vec128< T, N > a, const Vec128< T, N > b)
Definition emu128-inl.h:1148
hwy::HWY_NAMESPACE::Or
HWY_API Vec128< T, N > Or(const Vec128< T, N > a, const Vec128< T, N > b)
Definition arm_neon-inl.h:1986
hwy::HWY_NAMESPACE::NearestInt
HWY_API Vec128< int32_t, N > NearestInt(const Vec128< float, N > v)
Definition arm_neon-inl.h:3497
hwy::HWY_NAMESPACE::ApproximateReciprocal
HWY_API Vec128< float > ApproximateReciprocal(const Vec128< float > v)
Definition arm_neon-inl.h:1734
hwy::HWY_NAMESPACE::U8FromU32
HWY_API Vec32< uint8_t > U8FromU32(const Vec128< uint32_t > v)
Definition arm_neon-inl.h:3287
hwy::HWY_NAMESPACE::SetTableIndices
HWY_API Indices128< T, N > SetTableIndices(Simd< T, N, 0 > d, const TI *idx)
Definition arm_neon-inl.h:4013
hwy::HWY_NAMESPACE::GetLane
HWY_API TFromV< V > GetLane(const V v)
Definition arm_neon-inl.h:1076
hwy::HWY_NAMESPACE::ScatterIndex
HWY_API void ScatterIndex(Vec128< T, N > v, Simd< T, N, 0 > d, T *HWY_RESTRICT base, const Vec128< Index, N > index)
Definition arm_neon-inl.h:5002
hwy::HWY_NAMESPACE::NegMulAdd
HWY_API Vec128< float, N > NegMulAdd(const Vec128< float, N > mul, const Vec128< float, N > x, const Vec128< float, N > add)
Definition arm_neon-inl.h:1832
hwy::HWY_NAMESPACE::PromoteTo
HWY_API Vec128< uint16_t > PromoteTo(Full128< uint16_t >, const Vec64< uint8_t > v)
Definition arm_neon-inl.h:2965
hwy::HWY_NAMESPACE::operator<=
HWY_API Mask128< T, N > operator<=(const Vec128< T, N > a, const Vec128< T, N > b)
Definition emu128-inl.h:1180
hwy::HWY_NAMESPACE::Or3
HWY_API Vec128< T, N > Or3(Vec128< T, N > o1, Vec128< T, N > o2, Vec128< T, N > o3)
Definition arm_neon-inl.h:2033
hwy::HWY_NAMESPACE::VFromD
decltype(Zero(D())) VFromD
Definition arm_neon-inl.h:1030
hwy::HWY_NAMESPACE::LoadDup128
HWY_API Vec128< T, N > LoadDup128(Simd< T, N, 0 > d, const T *const HWY_RESTRICT p)
Definition arm_neon-inl.h:2765
hwy::HWY_NAMESPACE::OddEven
HWY_API Vec128< T, N > OddEven(const Vec128< T, N > a, const Vec128< T, N > b)
Definition arm_neon-inl.h:4678
hwy::HWY_NAMESPACE::MulFixedPoint15
HWY_API Vec128< int16_t > MulFixedPoint15(Vec128< int16_t > a, Vec128< int16_t > b)
Definition arm_neon-inl.h:1720
hwy::HWY_NAMESPACE::Trunc
HWY_API Vec128< float, N > Trunc(const Vec128< float, N > v)
Definition arm_neon-inl.h:3425
hwy::HWY_NAMESPACE::MinOfLanes
HWY_API Vec128< T, N > MinOfLanes(Simd< T, N, 0 >, const Vec128< T, N > v)
Definition arm_neon-inl.h:5338
hwy::HWY_NAMESPACE::CompressStore
HWY_API size_t CompressStore(Vec128< T, N > v, const Mask128< T, N > mask, Simd< T, N, 0 > d, T *HWY_RESTRICT unaligned)
Definition arm_neon-inl.h:6248
hwy::HWY_NAMESPACE::Abs
HWY_API Vec128< int8_t > Abs(const Vec128< int8_t > v)
Definition arm_neon-inl.h:2146
hwy::HWY_NAMESPACE::ConvertTo
HWY_API Vec128< float > ConvertTo(Full128< float >, const Vec128< int32_t > v)
Definition arm_neon-inl.h:3327
hwy::HWY_NAMESPACE::Sqrt
HWY_API Vec128< float, N > Sqrt(const Vec128< float, N > v)
Definition arm_neon-inl.h:1913
hwy::HWY_NAMESPACE::CompressBitsStore
HWY_API size_t CompressBitsStore(Vec128< T, N > v, const uint8_t *HWY_RESTRICT bits, Simd< T, N, 0 > d, T *HWY_RESTRICT unaligned)
Definition arm_neon-inl.h:6273
hwy::HWY_NAMESPACE::RotateRight
HWY_API Vec128< uint32_t, N > RotateRight(const Vec128< uint32_t, N > v)
Definition arm_neon-inl.h:1444
hwy::HWY_NAMESPACE::IsFinite
HWY_API Mask128< T, N > IsFinite(const Vec128< T, N > v)
Definition arm_neon-inl.h:3521
hwy::HWY_NAMESPACE::AndNot
HWY_API Vec128< T, N > AndNot(const Vec128< T, N > not_mask, const Vec128< T, N > mask)
Definition arm_neon-inl.h:1964
hwy::HWY_NAMESPACE::SumsOf8
HWY_API Vec128< uint64_t > SumsOf8(const Vec128< uint8_t > v)
Definition arm_neon-inl.h:1361
hwy::HWY_NAMESPACE::ApproximateReciprocalSqrt
HWY_API Vec128< float > ApproximateReciprocalSqrt(const Vec128< float > v)
Definition arm_neon-inl.h:1885
hwy::HWY_NAMESPACE::LoadInterleaved4
HWY_API void LoadInterleaved4(Simd< T, N, 0 >, const T *HWY_RESTRICT unaligned, Vec128< T, N > &v0, Vec128< T, N > &v1, Vec128< T, N > &v2, Vec128< T, N > &v3)
Definition arm_neon-inl.h:6428
hwy::HWY_NAMESPACE::ReverseBlocks
HWY_API Vec128< T > ReverseBlocks(Full128< T >, const Vec128< T > v)
Definition arm_neon-inl.h:4712
hwy::HWY_NAMESPACE::CompressBlendedStore
HWY_API size_t CompressBlendedStore(Vec128< T, N > v, Mask128< T, N > m, Simd< T, N, 0 > d, T *HWY_RESTRICT unaligned)
Definition arm_neon-inl.h:6257
hwy::HWY_NAMESPACE::Reverse4
HWY_API Vec128< T, N > Reverse4(Simd< T, N, 0 > d, const Vec128< T, N > v)
Definition arm_neon-inl.h:4090
hwy::HWY_NAMESPACE::FindKnownFirstTrue
HWY_API size_t FindKnownFirstTrue(const Simd< T, N, 0 > d, const Mask128< T, N > mask)
Definition arm_neon-inl.h:5683
hwy::HWY_NAMESPACE::operator+
HWY_API Vec128< T, N > operator+(Vec128< T, N > a, const Vec128< T, N > b)
Definition emu128-inl.h:580
hwy::HWY_NAMESPACE::StoreInterleaved2
HWY_API void StoreInterleaved2(const Vec128< T, N > v0, const Vec128< T, N > v1, Simd< T, N, 0 >, T *HWY_RESTRICT unaligned)
Definition arm_neon-inl.h:6517
hwy::HWY_NAMESPACE::Reverse
HWY_API Vec128< T, 1 > Reverse(Simd< T, 1, 0 >, const Vec128< T, 1 > v)
Definition arm_neon-inl.h:4030
hwy::HWY_NAMESPACE::operator>>
HWY_API Vec128< uint8_t > operator>>(const Vec128< uint8_t > v, const Vec128< uint8_t > bits)
Definition arm_neon-inl.h:1542
hwy::HWY_NAMESPACE::Store
HWY_API void Store(Vec128< T, N > v, Simd< T, N, 0 > d, T *HWY_RESTRICT aligned)
Definition arm_neon-inl.h:2934
hwy::HWY_NAMESPACE::InsertLane
HWY_API Vec128< T, 1 > InsertLane(const Vec128< T, 1 > v, size_t i, T t)
Definition arm_neon-inl.h:1225
hwy::HWY_NAMESPACE::SaturatedSub
HWY_API Vec128< T, N > SaturatedSub(Vec128< T, N > a, const Vec128< T, N > b)
Definition emu128-inl.h:608
hwy::HWY_NAMESPACE::ShiftLeft
HWY_API Vec128< T, N > ShiftLeft(Vec128< T, N > v)
Definition emu128-inl.h:376
hwy::HWY_NAMESPACE::Broadcast
HWY_API Vec128< uint16_t > Broadcast(const Vec128< uint16_t > v)
Definition arm_neon-inl.h:3885
hwy::HWY_NAMESPACE::v
const vfloat64m1_t v
Definition rvv-inl.h:1998
hwy::HWY_NAMESPACE::AbsDiff
HWY_API Vec128< float > AbsDiff(const Vec128< float > a, const Vec128< float > b)
Definition arm_neon-inl.h:1773
hwy::HWY_NAMESPACE::StoreInterleaved3
HWY_API void StoreInterleaved3(const Vec128< T, N > v0, const Vec128< T, N > v1, const Vec128< T, N > v2, Simd< T, N, 0 >, T *HWY_RESTRICT unaligned)
Definition arm_neon-inl.h:6549
hwy::HWY_NAMESPACE::TableLookupBytesOr0
HWY_API VI TableLookupBytesOr0(const V bytes, const VI from)
Definition arm_neon-inl.h:4977
hwy::HWY_NAMESPACE::Compress
HWY_API Vec128< T, 1 > Compress(Vec128< T, 1 > v, Mask128< T, 1 >)
Definition arm_neon-inl.h:6174
hwy::HWY_NAMESPACE::NegMulSub
HWY_API Vec128< float, N > NegMulSub(const Vec128< float, N > mul, const Vec128< float, N > x, const Vec128< float, N > sub)
Definition arm_neon-inl.h:1861
hwy
Definition aligned_allocator.h:27
hwy::F32FromBF16
HWY_API float F32FromBF16(bfloat16_t bf)
Definition base.h:975
hwy::BF16FromF32
HWY_API bfloat16_t BF16FromF32(float f)
Definition base.h:983
hwy::HighestValue< float >
constexpr float HighestValue< float >()
Definition base.h:688
hwy::CopySameSize
HWY_API void CopySameSize(const From *HWY_RESTRICT from, To *HWY_RESTRICT to)
Definition base.h:961
hwy::PopCount
HWY_API size_t PopCount(uint64_t x)
Definition base.h:865
hwy::LowestValue< float >
constexpr float LowestValue< float >()
Definition base.h:675
hwy::MakeUnsigned
typename detail::Relations< T >::Unsigned MakeUnsigned
Definition base.h:593
hwy::MakeSigned
typename detail::Relations< T >::Signed MakeSigned
Definition base.h:595
shared-inl.h
HWY_AFTER_NAMESPACE
HWY_AFTER_NAMESPACE()
HWY_BEFORE_NAMESPACE
HWY_BEFORE_NAMESPACE()
HWY_NAMESPACE
#define HWY_NAMESPACE
Definition set_macros-inl.h:82
hwy::HWY_NAMESPACE::CompressIsPartition::value
@ value
Definition arm_neon-inl.h:5730
hwy::HWY_NAMESPACE::Indices1
Definition scalar-inl.h:1365
hwy::HWY_NAMESPACE::Indices1::raw
MakeSigned< T > raw
Definition scalar-inl.h:1366
hwy::HWY_NAMESPACE::Simd
Definition ops/shared-inl.h:52
hwy::HWY_NAMESPACE::Vec1
Definition scalar-inl.h:35
hwy::HWY_NAMESPACE::Vec1::raw
T raw
Definition scalar-inl.h:66
hwy::HWY_NAMESPACE::Vec1::kPrivateN
static constexpr size_t kPrivateN
Definition scalar-inl.h:37
hwy::HWY_NAMESPACE::Vec1::operator*=
HWY_INLINE Vec1 & operator*=(const Vec1 other)
Definition scalar-inl.h:44
hwy::HWY_NAMESPACE::Vec1::Vec1
Vec1(const Vec1 &)=default
hwy::HWY_NAMESPACE::Vec1::Vec1
HWY_INLINE Vec1()=default
hwy::HWY_NAMESPACE::Vec1::operator^=
HWY_INLINE Vec1 & operator^=(const Vec1 other)
Definition scalar-inl.h:62
hwy::HWY_NAMESPACE::Vec1::Vec1
HWY_INLINE Vec1(const T t)
Definition scalar-inl.h:42
hwy::HWY_NAMESPACE::Vec1::operator&=
HWY_INLINE Vec1 & operator&=(const Vec1 other)
Definition scalar-inl.h:56
hwy::HWY_NAMESPACE::Vec1::PrivateT
T PrivateT
Definition scalar-inl.h:36
hwy::HWY_NAMESPACE::Vec1::operator=
Vec1 & operator=(const Vec1 &)=default
hwy::HWY_NAMESPACE::Vec1::operator-=
HWY_INLINE Vec1 & operator-=(const Vec1 other)
Definition scalar-inl.h:53
hwy::HWY_NAMESPACE::Vec1::operator+=
HWY_INLINE Vec1 & operator+=(const Vec1 other)
Definition scalar-inl.h:50
hwy::HWY_NAMESPACE::Vec1::operator|=
HWY_INLINE Vec1 & operator|=(const Vec1 other)
Definition scalar-inl.h:59
hwy::HWY_NAMESPACE::Vec1::operator/=
HWY_INLINE Vec1 & operator/=(const Vec1 other)
Definition scalar-inl.h:47
hwy::HWY_NAMESPACE::detail::RotateRight< 0 >::operator()
HWY_INLINE Vec1< T > operator()(const Vec1< T > v) const
Definition scalar-inl.h:400
hwy::HWY_NAMESPACE::detail::RotateRight
Definition emu128-inl.h:422
hwy::HWY_NAMESPACE::detail::RotateRight::operator()
HWY_INLINE Vec1< T > operator()(const Vec1< T > v) const
Definition scalar-inl.h:392