Loading...
Searching...
No Matches
volk_32f_binary_slicer_8i.h
Go to the documentation of this file.
/* -*- c++ -*- */
/*
* Copyright 2014 Free Software Foundation, Inc.
*
* This file is part of VOLK
*
* SPDX-License-Identifier: LGPL-3.0-or-later
*/
#ifndef INCLUDED_volk_32f_binary_slicer_8i_H
#define INCLUDED_volk_32f_binary_slicer_8i_H
#ifdef LV_HAVE_GENERIC
static inline void volk_32f_binary_slicer_8i_generic(int8_t* cVector,
const float* aVector,
unsigned int num_points)
{
int8_t* cPtr = cVector;
const float* aPtr = aVector;
unsigned int number = 0;
for (number = 0; number < num_points; number++) {
if (*aPtr++ >= 0) {
*cPtr++ = 1;
} else {
*cPtr++ = 0;
}
}
}
#endif /* LV_HAVE_GENERIC */
#ifdef LV_HAVE_GENERIC
static inline void volk_32f_binary_slicer_8i_generic_branchless(int8_t* cVector,
const float* aVector,
unsigned int num_points)
{
int8_t* cPtr = cVector;
const float* aPtr = aVector;
unsigned int number = 0;
for (number = 0; number < num_points; number++) {
*cPtr++ = (*aPtr++ >= 0);
}
}
#endif /* LV_HAVE_GENERIC */
#ifdef LV_HAVE_AVX2
#include <immintrin.h>
static inline void volk_32f_binary_slicer_8i_a_avx2(int8_t* cVector,
const float* aVector,
unsigned int num_points)
{
int8_t* cPtr = cVector;
const float* aPtr = aVector;
unsigned int number = 0;
unsigned int n32points = num_points / 32;
const __m256 zero_val = _mm256_set1_ps(0.0f);
__m256 a0_val, a1_val, a2_val, a3_val;
__m256 res0_f, res1_f, res2_f, res3_f;
__m256i res0_i, res1_i, res2_i, res3_i;
__m256i byte_shuffle = _mm256_set_epi8(15,
14,
13,
12,
7,
6,
5,
4,
11,
10,
9,
8,
3,
2,
1,
0,
15,
14,
13,
12,
7,
6,
5,
4,
11,
10,
9,
8,
3,
2,
1,
0);
for (number = 0; number < n32points; number++) {
a0_val = _mm256_load_ps(aPtr);
a1_val = _mm256_load_ps(aPtr + 8);
a2_val = _mm256_load_ps(aPtr + 16);
a3_val = _mm256_load_ps(aPtr + 24);
// compare >= 0; return float
res0_f = _mm256_cmp_ps(a0_val, zero_val, _CMP_GE_OS);
res1_f = _mm256_cmp_ps(a1_val, zero_val, _CMP_GE_OS);
res2_f = _mm256_cmp_ps(a2_val, zero_val, _CMP_GE_OS);
res3_f = _mm256_cmp_ps(a3_val, zero_val, _CMP_GE_OS);
// convert to 32i and >> 31
res0_i = _mm256_srli_epi32(_mm256_cvtps_epi32(res0_f), 31);
res1_i = _mm256_srli_epi32(_mm256_cvtps_epi32(res1_f), 31);
res2_i = _mm256_srli_epi32(_mm256_cvtps_epi32(res2_f), 31);
res3_i = _mm256_srli_epi32(_mm256_cvtps_epi32(res3_f), 31);
// pack in to 16-bit results
res0_i = _mm256_packs_epi32(res0_i, res1_i);
res2_i = _mm256_packs_epi32(res2_i, res3_i);
// pack in to 8-bit results
// res0: (after packs_epi32)
// a0, a1, a2, a3, b0, b1, b2, b3, a4, a5, a6, a7, b4, b5, b6, b7
// res2:
// c0, c1, c2, c3, d0, d1, d2, d3, c4, c5, c6, c7, d4, d5, d6, d7
res0_i = _mm256_packs_epi16(res0_i, res2_i);
// shuffle the lanes
// res0: (after packs_epi16)
// a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3
// a4, a5, a6, a7, b4, b5, b6, b7, c4, c5, c6, c7, d4, d5, d6, d7
// 0, 2, 1, 3 -> 11 01 10 00 (0xd8)
res0_i = _mm256_permute4x64_epi64(res0_i, 0xd8);
// shuffle bytes within lanes
// res0: (after shuffle_epi8)
// a0, a1, a2, a3, b0, b1, b2, b3, a4, a5, a6, a7, b4, b5, b6, b7
// c0, c1, c2, c3, d0, d1, d2, d3, c4, c5, c6, c7, d4, d5, d6, d7
res0_i = _mm256_shuffle_epi8(res0_i, byte_shuffle);
_mm256_store_si256((__m256i*)cPtr, res0_i);
aPtr += 32;
cPtr += 32;
}
for (number = n32points * 32; number < num_points; number++) {
if (*aPtr++ >= 0) {
*cPtr++ = 1;
} else {
*cPtr++ = 0;
}
}
}
#endif
#ifdef LV_HAVE_AVX2
#include <immintrin.h>
static inline void volk_32f_binary_slicer_8i_u_avx2(int8_t* cVector,
const float* aVector,
unsigned int num_points)
{
int8_t* cPtr = cVector;
const float* aPtr = aVector;
unsigned int number = 0;
unsigned int n32points = num_points / 32;
const __m256 zero_val = _mm256_set1_ps(0.0f);
__m256 a0_val, a1_val, a2_val, a3_val;
__m256 res0_f, res1_f, res2_f, res3_f;
__m256i res0_i, res1_i, res2_i, res3_i;
__m256i byte_shuffle = _mm256_set_epi8(15,
14,
13,
12,
7,
6,
5,
4,
11,
10,
9,
8,
3,
2,
1,
0,
15,
14,
13,
12,
7,
6,
5,
4,
11,
10,
9,
8,
3,
2,
1,
0);
for (number = 0; number < n32points; number++) {
a0_val = _mm256_loadu_ps(aPtr);
a1_val = _mm256_loadu_ps(aPtr + 8);
a2_val = _mm256_loadu_ps(aPtr + 16);
a3_val = _mm256_loadu_ps(aPtr + 24);
// compare >= 0; return float
res0_f = _mm256_cmp_ps(a0_val, zero_val, _CMP_GE_OS);
res1_f = _mm256_cmp_ps(a1_val, zero_val, _CMP_GE_OS);
res2_f = _mm256_cmp_ps(a2_val, zero_val, _CMP_GE_OS);
res3_f = _mm256_cmp_ps(a3_val, zero_val, _CMP_GE_OS);
// convert to 32i and >> 31
res0_i = _mm256_srli_epi32(_mm256_cvtps_epi32(res0_f), 31);
res1_i = _mm256_srli_epi32(_mm256_cvtps_epi32(res1_f), 31);
res2_i = _mm256_srli_epi32(_mm256_cvtps_epi32(res2_f), 31);
res3_i = _mm256_srli_epi32(_mm256_cvtps_epi32(res3_f), 31);
// pack in to 16-bit results
res0_i = _mm256_packs_epi32(res0_i, res1_i);
res2_i = _mm256_packs_epi32(res2_i, res3_i);
// pack in to 8-bit results
// res0: (after packs_epi32)
// a0, a1, a2, a3, b0, b1, b2, b3, a4, a5, a6, a7, b4, b5, b6, b7
// res2:
// c0, c1, c2, c3, d0, d1, d2, d3, c4, c5, c6, c7, d4, d5, d6, d7
res0_i = _mm256_packs_epi16(res0_i, res2_i);
// shuffle the lanes
// res0: (after packs_epi16)
// a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3
// a4, a5, a6, a7, b4, b5, b6, b7, c4, c5, c6, c7, d4, d5, d6, d7
// 0, 2, 1, 3 -> 11 01 10 00 (0xd8)
res0_i = _mm256_permute4x64_epi64(res0_i, 0xd8);
// shuffle bytes within lanes
// res0: (after shuffle_epi8)
// a0, a1, a2, a3, b0, b1, b2, b3, a4, a5, a6, a7, b4, b5, b6, b7
// c0, c1, c2, c3, d0, d1, d2, d3, c4, c5, c6, c7, d4, d5, d6, d7
res0_i = _mm256_shuffle_epi8(res0_i, byte_shuffle);
_mm256_storeu_si256((__m256i*)cPtr, res0_i);
aPtr += 32;
cPtr += 32;
}
for (number = n32points * 32; number < num_points; number++) {
if (*aPtr++ >= 0) {
*cPtr++ = 1;
} else {
*cPtr++ = 0;
}
}
}
#endif
#ifdef LV_HAVE_SSE2
#include <emmintrin.h>
static inline void volk_32f_binary_slicer_8i_a_sse2(int8_t* cVector,
const float* aVector,
unsigned int num_points)
{
int8_t* cPtr = cVector;
const float* aPtr = aVector;
unsigned int number = 0;
unsigned int n16points = num_points / 16;
__m128 a0_val, a1_val, a2_val, a3_val;
__m128 res0_f, res1_f, res2_f, res3_f;
__m128i res0_i, res1_i, res2_i, res3_i;
__m128 zero_val;
zero_val = _mm_set1_ps(0.0f);
for (number = 0; number < n16points; number++) {
a0_val = _mm_load_ps(aPtr);
a1_val = _mm_load_ps(aPtr + 4);
a2_val = _mm_load_ps(aPtr + 8);
a3_val = _mm_load_ps(aPtr + 12);
// compare >= 0; return float
res0_f = _mm_cmpge_ps(a0_val, zero_val);
res1_f = _mm_cmpge_ps(a1_val, zero_val);
res2_f = _mm_cmpge_ps(a2_val, zero_val);
res3_f = _mm_cmpge_ps(a3_val, zero_val);
// convert to 32i and >> 31
res0_i = _mm_srli_epi32(_mm_cvtps_epi32(res0_f), 31);
res1_i = _mm_srli_epi32(_mm_cvtps_epi32(res1_f), 31);
res2_i = _mm_srli_epi32(_mm_cvtps_epi32(res2_f), 31);
res3_i = _mm_srli_epi32(_mm_cvtps_epi32(res3_f), 31);
// pack into 16-bit results
res0_i = _mm_packs_epi32(res0_i, res1_i);
res2_i = _mm_packs_epi32(res2_i, res3_i);
// pack into 8-bit results
res0_i = _mm_packs_epi16(res0_i, res2_i);
_mm_store_si128((__m128i*)cPtr, res0_i);
cPtr += 16;
aPtr += 16;
}
for (number = n16points * 16; number < num_points; number++) {
if (*aPtr++ >= 0) {
*cPtr++ = 1;
} else {
*cPtr++ = 0;
}
}
}
#endif /* LV_HAVE_SSE2 */
#ifdef LV_HAVE_SSE2
#include <emmintrin.h>
static inline void volk_32f_binary_slicer_8i_u_sse2(int8_t* cVector,
const float* aVector,
unsigned int num_points)
{
int8_t* cPtr = cVector;
const float* aPtr = aVector;
unsigned int number = 0;
unsigned int n16points = num_points / 16;
__m128 a0_val, a1_val, a2_val, a3_val;
__m128 res0_f, res1_f, res2_f, res3_f;
__m128i res0_i, res1_i, res2_i, res3_i;
__m128 zero_val;
zero_val = _mm_set1_ps(0.0f);
for (number = 0; number < n16points; number++) {
a0_val = _mm_loadu_ps(aPtr);
a1_val = _mm_loadu_ps(aPtr + 4);
a2_val = _mm_loadu_ps(aPtr + 8);
a3_val = _mm_loadu_ps(aPtr + 12);
// compare >= 0; return float
res0_f = _mm_cmpge_ps(a0_val, zero_val);
res1_f = _mm_cmpge_ps(a1_val, zero_val);
res2_f = _mm_cmpge_ps(a2_val, zero_val);
res3_f = _mm_cmpge_ps(a3_val, zero_val);
// convert to 32i and >> 31
res0_i = _mm_srli_epi32(_mm_cvtps_epi32(res0_f), 31);
res1_i = _mm_srli_epi32(_mm_cvtps_epi32(res1_f), 31);
res2_i = _mm_srli_epi32(_mm_cvtps_epi32(res2_f), 31);
res3_i = _mm_srli_epi32(_mm_cvtps_epi32(res3_f), 31);
// pack into 16-bit results
res0_i = _mm_packs_epi32(res0_i, res1_i);
res2_i = _mm_packs_epi32(res2_i, res3_i);
// pack into 8-bit results
res0_i = _mm_packs_epi16(res0_i, res2_i);
_mm_storeu_si128((__m128i*)cPtr, res0_i);
cPtr += 16;
aPtr += 16;
}
for (number = n16points * 16; number < num_points; number++) {
if (*aPtr++ >= 0) {
*cPtr++ = 1;
} else {
*cPtr++ = 0;
}
}
}
#endif /* LV_HAVE_SSE2 */
#ifdef LV_HAVE_NEON
#include <arm_neon.h>
static inline void volk_32f_binary_slicer_8i_neon(int8_t* cVector,
const float* aVector,
unsigned int num_points)
{
int8_t* cPtr = cVector;
const float* aPtr = aVector;
unsigned int number = 0;
unsigned int n16points = num_points / 16;
float32x4x2_t input_val0, input_val1;
float32x4_t zero_val;
uint32x4x2_t res0_u32, res1_u32;
uint16x4x2_t res0_u16x4, res1_u16x4;
uint16x8x2_t res_u16x8;
uint8x8x2_t res_u8;
uint8x8_t one;
zero_val = vdupq_n_f32(0.0);
one = vdup_n_u8(0x01);
// TODO: this is a good candidate for asm because the vcombines
// can be eliminated simply by picking dst registers that are
// adjacent.
for (number = 0; number < n16points; number++) {
input_val0 = vld2q_f32(aPtr);
input_val1 = vld2q_f32(aPtr + 8);
// test against 0; return uint32
res0_u32.val[0] = vcgeq_f32(input_val0.val[0], zero_val);
res0_u32.val[1] = vcgeq_f32(input_val0.val[1], zero_val);
res1_u32.val[0] = vcgeq_f32(input_val1.val[0], zero_val);
res1_u32.val[1] = vcgeq_f32(input_val1.val[1], zero_val);
// narrow uint32 -> uint16 followed by combine to 8-element vectors
res0_u16x4.val[0] = vmovn_u32(res0_u32.val[0]);
res0_u16x4.val[1] = vmovn_u32(res0_u32.val[1]);
res1_u16x4.val[0] = vmovn_u32(res1_u32.val[0]);
res1_u16x4.val[1] = vmovn_u32(res1_u32.val[1]);
res_u16x8.val[0] = vcombine_u16(res0_u16x4.val[0], res1_u16x4.val[0]);
res_u16x8.val[1] = vcombine_u16(res0_u16x4.val[1], res1_u16x4.val[1]);
// narrow uint16x8 -> uint8x8
res_u8.val[0] = vmovn_u16(res_u16x8.val[0]);
res_u8.val[1] = vmovn_u16(res_u16x8.val[1]);
// we *could* load twice as much data and do another vcombine here
// to get a uint8x16x2 vector, still only do 2 vandqs and a single store
// but that turns out to be ~16% slower than this version on zc702
// it's possible register contention in GCC scheduler slows it down
// and a hand-written asm with quad-word u8 registers is much faster.
res_u8.val[0] = vand_u8(one, res_u8.val[0]);
res_u8.val[1] = vand_u8(one, res_u8.val[1]);
vst2_u8((unsigned char*)cPtr, res_u8);
cPtr += 16;
aPtr += 16;
}
for (number = n16points * 16; number < num_points; number++) {
if (*aPtr++ >= 0) {
*cPtr++ = 1;
} else {
*cPtr++ = 0;
}
}
}
#endif /* LV_HAVE_NEON */
#ifdef LV_HAVE_RVV
#include <riscv_vector.h>
static inline void volk_32f_binary_slicer_8i_rvv(int8_t* cVector,
const float* aVector,
unsigned int num_points)
{
size_t n = num_points;
vint8m2_t v0 = __riscv_vmv_v_x_i8m2(1, __riscv_vsetvlmax_e8m2());
for (size_t vl; n > 0; n -= vl, aVector += vl, cVector += vl) {
vl = __riscv_vsetvl_e32m8(n);
vfloat32m8_t v = __riscv_vle32_v_f32m8(aVector, vl);
vint8m2_t vn = __riscv_vmerge(v0, 0, __riscv_vmflt(v, 0, vl), vl);
__riscv_vse8(cVector, vn, vl);
}
}
#endif /*LV_HAVE_RVV*/
#endif /* INCLUDED_volk_32f_binary_slicer_8i_H */