SSE2 functions

Processor specific optimizations for SSE2 instructions (internals). More...

## Data Structures

class | bm::sse_empty_guard
| SSE2 reinitialization guard class. More...

## Functions

void | bm::sse2_xor_arr_2_mask (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src, const __m128i *BMRESTRICT src_end, bm::word_t mask) BMNOEXCEPT
| XOR array elements to specified mask dst = *src ^ mask.
void | bm::sse2_andnot_arr_2_mask (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src, const __m128i *BMRESTRICT src_end, bm::word_t mask) BMNOEXCEPT
| Inverts array elements and NOT them to specified mask dst = ~*src & mask.
unsigned | bm::sse2_and_block (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src) BMNOEXCEPT
| AND blocks2 dst &= *src.
unsigned | bm::sse2_and_arr_unal (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src, const __m128i *BMRESTRICT src_end) BMNOEXCEPT
| AND array elements against another array (unaligned) dst &= *src.
bool | bm::sse2_or_block (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src) BMNOEXCEPT
| OR array elements against another array dst |= *src.
bool | bm::sse2_or_arr_unal (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src, const __m128i *BMRESTRICT src_end) BMNOEXCEPT
| OR array elements against another array (unaligned) dst |= *src.
bool | bm::sse2_or_block_2way (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src1, const __m128i *BMRESTRICT src2) BMNOEXCEPT
| OR 2 blocks anc copy result to the destination dst = *src1 | src2.
bool | bm::sse2_or_block_3way (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src1, const __m128i *BMRESTRICT src2) BMNOEXCEPT
| OR array elements against another 2 arrays dst |= *src1 | src2.
bool | bm::sse2_or_block_5way (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src1, const __m128i *BMRESTRICT src2, const __m128i *BMRESTRICT src3, const __m128i *BMRESTRICT src4) BMNOEXCEPT
| OR array elements against another 2 arrays dst |= *src1 | src2 | src3 | src4.
unsigned | bm::sse2_xor_block (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src) BMNOEXCEPT
| XOR block against another dst ^= *src.
unsigned | bm::sse2_xor_block_2way (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src1, const __m128i *BMRESTRICT src2) BMNOEXCEPT
| 3 operand XOR dst = *src1 ^ src2
unsigned | bm::sse2_sub_block (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src) BMNOEXCEPT
| AND-NOT (SUB) array elements against another array dst &= ~*src.
void | bm::sse2_set_block (__m128i *BMRESTRICT dst, bm::word_t value) BMNOEXCEPT
| SSE2 block memset dst = value.
void | bm::sse2_copy_block (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src) BMNOEXCEPT
| SSE2 block copy dst = *src.
void | bm::sse2_copy_block_unalign (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src) BMNOEXCEPT
| SSE2 block copy (unaligned SRC) dst = *src.
void | bm::sse2_stream_block (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src) BMNOEXCEPT
| SSE2 block copy dst = *src.
void | bm::sse2_stream_block_unalign (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src) BMNOEXCEPT
| SSE2 block copy (unaligned src) dst = *src.
void | bm::sse2_invert_block (__m128i *BMRESTRICT dst) BMNOEXCEPT
| Invert bit block dst = ~*dst or dst ^= *dst.
unsigned | bm::sse2_lower_bound_scan_u32 (const unsigned *BMRESTRICT arr, unsigned target, unsigned from, unsigned to) BMNOEXCEPT
| lower bound (great or equal) linear scan in ascending order sorted array
bool | bm::sse2_is_all_zero (const __m128i *BMRESTRICT block) BMNOEXCEPT
| check if block is all zero bits
bool | bm::sse2_is_all_one (const __m128i *BMRESTRICT block) BMNOEXCEPT
| check if block is all ONE bits
BMFORCEINLINE bool | bm::sse2_is_digest_zero (const __m128i *BMRESTRICT block) BMNOEXCEPT
| check if digest stride is all zero bits
BMFORCEINLINE void | bm::sse2_block_set_digest (__m128i *dst, unsigned value) BMNOEXCEPT
| set digest stride to 0xFF.. or 0x0 value
void | bm::sse2_bit_block_xor (bm::word_t *target_block, const bm::word_t *block, const bm::word_t *xor_block, bm::id64_t digest) BMNOEXCEPT
| Build partial XOR product of 2 bit-blocks using digest mask.
void | bm::sse2_bit_block_xor_2way (bm::word_t *target_block, const bm::word_t *xor_block, bm::id64_t digest) BMNOEXCEPT
| Build partial XOR product of 2 bit-blocks using digest mask.
BMFORCEINLINE bool | bm::sse2_and_digest (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src) BMNOEXCEPT
| AND block digest stride dst &= *src.
BMFORCEINLINE bool | bm::sse2_and_or_digest_2way (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src1, const __m128i *BMRESTRICT src2) BMNOEXCEPT
| AND-OR block digest stride dst |= *src1 & src2.
bool | bm::sse2_and_digest_5way (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src1, const __m128i *BMRESTRICT src2, const __m128i *BMRESTRICT src3, const __m128i *BMRESTRICT src4) BMNOEXCEPT
| AND block digest stride.
bool | bm::sse2_and_digest_3way (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src1, const __m128i *BMRESTRICT src2) BMNOEXCEPT
| AND block digest stride.
BMFORCEINLINE bool | bm::sse2_and_digest_2way (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src1, const __m128i *BMRESTRICT src2) BMNOEXCEPT
| AND block digest stride dst = *src1 & src2.
BMFORCEINLINE bool | bm::sse2_sub_digest (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src) BMNOEXCEPT
| SUB (AND NOT) block digest stride dst &= ~*src.
BMFORCEINLINE bool | bm::sse2_sub_digest_2way (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src1, const __m128i *BMRESTRICT src2) BMNOEXCEPT
| 2-operand SUB (AND NOT) block digest stride dst = src1 & ~*src2
bool | bm::sse2_bit_find_first (const __m128i *BMRESTRICT block, unsigned off, unsigned *pos) BMNOEXCEPT
| Find first non-zero bit.
bool | bm::sse2_bit_find_first_diff (const __m128i *BMRESTRICT block1, const __m128i *BMRESTRICT block2, unsigned *pos) BMNOEXCEPT
| Find first bit which is different between two bit-blocks.
bool | bm::sse2_shift_r1 (__m128i *block, unsigned *empty_acc, unsigned co1) BMNOEXCEPT
| block shift right by 1
bool | bm::sse2_shift_l1 (__m128i *block, unsigned *empty_acc, unsigned co1) BMNOEXCEPT
| block shift left by 1
unsigned | bm::sse2_gap_bfind (const unsigned short *BMRESTRICT buf, unsigned pos, unsigned *BMRESTRICT is_set)
| Hybrid binary search, starts as binary, then switches to linear scan.
unsigned | bm::sse2_gap_test (const unsigned short *BMRESTRICT buf, unsigned pos)
| Hybrid binary search, starts as binary, then switches to scan.

## Detailed Description

Processor specific optimizations for SSE2 instructions (internals).

## Function Documentation

## sse2_and_arr_unal()

unsigned bm::sse2_and_arr_unal | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src,
|  | const __m128i *BMRESTRICT | src_end )

inline

AND array elements against another array (unaligned) dst &= *src.

Returns
  0 if no bits were set

Definition at line 259 of file bmsse_util.h.

References BM_ALIGN16, BM_ALIGN16ATTR, BMNOEXCEPT, and BMRESTRICT.

Referenced by bm::decoder::get_32_AND().

## sse2_and_block()

unsigned bm::sse2_and_block | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src )

inline

AND blocks2 dst &= *src.

Returns
  0 if no bits were set

Definition at line 126 of file bmsse_util.h.

References BM_ALIGN16, BM_ALIGN16ATTR, BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_and_digest()

BMFORCEINLINE bool bm::sse2_and_digest | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src )

AND block digest stride dst &= *src.

Returns
  true if stide is all zero

Definition at line 415 of file bmsse2.h.

References BMNOEXCEPT, and BMRESTRICT.

## sse2_and_digest_2way()

BMFORCEINLINE bool bm::sse2_and_digest_2way | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src1,
|  | const __m128i *BMRESTRICT | src2 )

AND block digest stride dst = *src1 & src2.

Returns
  true if stide is all zero

Definition at line 681 of file bmsse2.h.

References BMNOEXCEPT, and BMRESTRICT.

## sse2_and_digest_3way()

bool bm::sse2_and_digest_3way | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src1,
|  | const __m128i *BMRESTRICT | src2 )

inline

AND block digest stride.

Returns
  true if stide is all zero

Definition at line 599 of file bmsse2.h.

References BMNOEXCEPT, and BMRESTRICT.

## sse2_and_digest_5way()

bool bm::sse2_and_digest_5way | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src1,
|  | const __m128i *BMRESTRICT | src2,
|  | const __m128i *BMRESTRICT | src3,
|  | const __m128i *BMRESTRICT | src4 )

inline

AND block digest stride.

Returns
  true if stide is all zero

Definition at line 519 of file bmsse2.h.

References BMNOEXCEPT, and BMRESTRICT.

## sse2_and_or_digest_2way()

BMFORCEINLINE bool bm::sse2_and_or_digest_2way | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src1,
|  | const __m128i *BMRESTRICT | src2 )

AND-OR block digest stride dst |= *src1 & src2.

Returns
  true if stide is all zero

Definition at line 464 of file bmsse2.h.

References BMNOEXCEPT, and BMRESTRICT.

## sse2_andnot_arr_2_mask()

void bm::sse2_andnot_arr_2_mask | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src,
|  | const __m128i *BMRESTRICT | src_end,
|  | bm::word_t | mask )

inline

Inverts array elements and NOT them to specified mask dst = ~*src & mask.

Definition at line 102 of file bmsse_util.h.

References BMNOEXCEPT, and BMRESTRICT.

## sse2_bit_block_xor()

void bm::sse2_bit_block_xor | ( | bm::word_t * | target_block,
|  | const bm::word_t * | block,
|  | const bm::word_t * | xor_block,
|  | bm::id64_t | digest )

inline

Build partial XOR product of 2 bit-blocks using digest mask.

Parameters

  target_block | - target := block ^ xor_block
  block | - arg1
  xor_block | - arg2
  digest | - mask for each block wave to XOR (1) or just copy (0)

Definition at line 289 of file bmsse2.h.

References block_waves, BMNOEXCEPT, and set_block_digest_wave_size.

## sse2_bit_block_xor_2way()

void bm::sse2_bit_block_xor_2way | ( | bm::word_t * | target_block,
|  | const bm::word_t * | xor_block,
|  | bm::id64_t | digest )

inline

Build partial XOR product of 2 bit-blocks using digest mask.

Parameters

  target_block | - target ^= xor_block
  xor_block | - arg1
  digest | - mask for each block wave to XOR (if 1)

Definition at line 360 of file bmsse2.h.

References bmi_blsi_u64(), bmi_bslr_u64(), BMNOEXCEPT, set_block_digest_wave_size, and word_bitcount64().

## sse2_bit_find_first()

bool bm::sse2_bit_find_first | ( | const __m128i *BMRESTRICT | block,
|  | unsigned | off,
|  | unsigned * | pos )

inline

Find first non-zero bit.

Definition at line 992 of file bmsse2.h.

References bit_scan_forward32(), BM_ALIGN32, BM_ALIGN32ATTR, BM_ASSERT, BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_bit_find_first_diff()

bool bm::sse2_bit_find_first_diff | ( | const __m128i *BMRESTRICT | block1,
|  | const __m128i *BMRESTRICT | block2,
|  | unsigned * | pos )

inline

Find first bit which is different between two bit-blocks.

Definition at line 1048 of file bmsse2.h.

References bit_scan_forward32(), BM_ALIGN32, BM_ALIGN32ATTR, BM_ASSERT, BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_block_set_digest()

BMFORCEINLINE void bm::sse2_block_set_digest | ( | __m128i * | dst,
|  | unsigned | value )

set digest stride to 0xFF.. or 0x0 value

Definition at line 268 of file bmsse2.h.

References BMNOEXCEPT.

## sse2_copy_block()

void bm::sse2_copy_block | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src )

inline

SSE2 block copy dst = *src.

Definition at line 838 of file bmsse_util.h.

References BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_copy_block_unalign()

void bm::sse2_copy_block_unalign | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src )

inline

SSE2 block copy (unaligned SRC) dst = *src.

Definition at line 879 of file bmsse_util.h.

References BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_gap_bfind()

unsigned bm::sse2_gap_bfind | ( | const unsigned short *BMRESTRICT | buf,
|  | unsigned | pos,
|  | unsigned *BMRESTRICT | is_set )

inline

Hybrid binary search, starts as binary, then switches to linear scan.

Parameters

  buf | - GAP buffer pointer.
  pos | - index of the element.
  is_set | - output. GAP value (0 or 1).

Returns
  GAP index.

Definition at line 1445 of file bmsse2.h.

References BM_ASSERT, BMRESTRICT, and sse2_gap_find().

Referenced by sse2_gap_test().

## sse2_gap_test()

unsigned bm::sse2_gap_test | ( | const unsigned short *BMRESTRICT | buf,
|  | unsigned | pos )

inline

Hybrid binary search, starts as binary, then switches to scan.

Definition at line 1502 of file bmsse2.h.

References BMRESTRICT, and sse2_gap_bfind().

## sse2_invert_block()

void bm::sse2_invert_block | ( | __m128i *BMRESTRICT | dst | ) |

inline

Invert bit block dst = ~*dst or dst ^= *dst.

Definition at line 1006 of file bmsse_util.h.

References BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_is_all_one()

bool bm::sse2_is_all_one | ( | const __m128i *BMRESTRICT | block | ) |

inline

check if block is all ONE bits

Definition at line 219 of file bmsse2.h.

References BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_is_all_zero()

bool bm::sse2_is_all_zero | ( | const __m128i *BMRESTRICT | block | ) |

inline

check if block is all zero bits

Definition at line 194 of file bmsse2.h.

References BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_is_digest_zero()

BMFORCEINLINE bool bm::sse2_is_digest_zero | ( | const __m128i *BMRESTRICT | block | ) |

check if digest stride is all zero bits

Definition at line 244 of file bmsse2.h.

References BMNOEXCEPT, and BMRESTRICT.

## sse2_lower_bound_scan_u32()

unsigned bm::sse2_lower_bound_scan_u32 | ( | const unsigned *BMRESTRICT | arr,
|  | unsigned | target,
|  | unsigned | from,
|  | unsigned | to )

inline

lower bound (great or equal) linear scan in ascending order sorted array

Definition at line 1099 of file bmsse_util.h.

References bit_scan_forward32(), BMNOEXCEPT, and BMRESTRICT.

## sse2_or_arr_unal()

bool bm::sse2_or_arr_unal | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src,
|  | const __m128i *BMRESTRICT | src_end )

inline

OR array elements against another array (unaligned) dst |= *src.

Returns
  true if all bits are 1

Definition at line 426 of file bmsse_util.h.

References BMNOEXCEPT, and BMRESTRICT.

Referenced by bm::decoder::get_32_OR().

## sse2_or_block()

bool bm::sse2_or_block | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src )

inline

OR array elements against another array dst |= *src.

Returns
  true if all bits are 1

Definition at line 372 of file bmsse_util.h.

References BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_or_block_2way()

bool bm::sse2_or_block_2way | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src1,
|  | const __m128i *BMRESTRICT | src2 )

inline

OR 2 blocks anc copy result to the destination dst = *src1 | src2.

Returns
  true if all bits are 1

Definition at line 478 of file bmsse_util.h.

References BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_or_block_3way()

bool bm::sse2_or_block_3way | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src1,
|  | const __m128i *BMRESTRICT | src2 )

inline

OR array elements against another 2 arrays dst |= *src1 | src2.

Returns
  true if all bits are 1

Definition at line 524 of file bmsse_util.h.

References BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_or_block_5way()

bool bm::sse2_or_block_5way | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src1,
|  | const __m128i *BMRESTRICT | src2,
|  | const __m128i *BMRESTRICT | src3,
|  | const __m128i *BMRESTRICT | src4 )

inline

OR array elements against another 2 arrays dst |= *src1 | src2 | src3 | src4.

Returns
  true if all bits are 1

Definition at line 575 of file bmsse_util.h.

References BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_set_block()

void bm::sse2_set_block | ( | __m128i *BMRESTRICT | dst,
|  | bm::word_t | value )

inline

SSE2 block memset dst = value.

Definition at line 809 of file bmsse_util.h.

References BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_shift_l1()

bool bm::sse2_shift_l1 | ( | __m128i * | block,
|  | unsigned * | empty_acc,
|  | unsigned | co1 )

inline

block shift left by 1

Definition at line 1165 of file bmsse2.h.

References BMNOEXCEPT, and set_block_size.

## sse2_shift_r1()

bool bm::sse2_shift_r1 | ( | __m128i * | block,
|  | unsigned * | empty_acc,
|  | unsigned | co1 )

inline

block shift right by 1

Definition at line 1112 of file bmsse2.h.

References BMNOEXCEPT, and set_block_size.

## sse2_stream_block()

void bm::sse2_stream_block | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src )

inline

SSE2 block copy dst = *src.

Definition at line 921 of file bmsse_util.h.

References BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_stream_block_unalign()

void bm::sse2_stream_block_unalign | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src )

inline

SSE2 block copy (unaligned src) dst = *src.

Definition at line 962 of file bmsse_util.h.

References BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_sub_block()

unsigned bm::sse2_sub_block | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src )

inline

AND-NOT (SUB) array elements against another array dst &= ~*src.

Returns
  0 if no bits were set

Definition at line 744 of file bmsse_util.h.

References BM_ALIGN16, BM_ALIGN16ATTR, BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_sub_digest()

BMFORCEINLINE bool bm::sse2_sub_digest | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src )

SUB (AND NOT) block digest stride dst &= ~*src.

Returns
  true if stide is all zero

Definition at line 731 of file bmsse2.h.

References BMNOEXCEPT, and BMRESTRICT.

## sse2_sub_digest_2way()

BMFORCEINLINE bool bm::sse2_sub_digest_2way | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src1,
|  | const __m128i *BMRESTRICT | src2 )

2-operand SUB (AND NOT) block digest stride dst = src1 & ~*src2

Returns
  true if stide is all zero

Definition at line 780 of file bmsse2.h.

References BMNOEXCEPT, and BMRESTRICT.

## sse2_xor_arr_2_mask()

void bm::sse2_xor_arr_2_mask | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src,
|  | const __m128i *BMRESTRICT | src_end,
|  | bm::word_t | mask )

inline

XOR array elements to specified mask dst = *src ^ mask.

Definition at line 78 of file bmsse_util.h.

References BMNOEXCEPT, and BMRESTRICT.

## sse2_xor_block()

unsigned bm::sse2_xor_block | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src )

inline

XOR block against another dst ^= *src.

Returns
  0 if no bits were set

Definition at line 646 of file bmsse_util.h.

References BM_ALIGN16, BM_ALIGN16ATTR, BMNOEXCEPT, BMRESTRICT, and set_block_size.

## sse2_xor_block_2way()

unsigned bm::sse2_xor_block_2way | ( | __m128i *BMRESTRICT | dst,
|  | const __m128i *BMRESTRICT | src1,
|  | const __m128i *BMRESTRICT | src2 )

inline

3 operand XOR dst = *src1 ^ src2

Returns
  0 if no bits were set

Definition at line 693 of file bmsse_util.h.

References BM_ALIGN16, BM_ALIGN16ATTR, BMNOEXCEPT, BMRESTRICT, and set_block_size.